1 /*
   2  * Copyright (c) 2013, 2020, Red Hat, Inc. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "memory/allocation.hpp"
  27 #include "memory/universe.hpp"
  28 
  29 #include "gc/shared/gcArguments.hpp"
  30 #include "gc/shared/gcTimer.hpp"
  31 #include "gc/shared/gcTraceTime.inline.hpp"
  32 #include "gc/shared/locationPrinter.inline.hpp"
  33 #include "gc/shared/memAllocator.hpp"
  34 #include "gc/shared/oopStorageSet.hpp"
  35 #include "gc/shared/plab.hpp"
  36 
  37 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
  38 #include "gc/shenandoah/shenandoahClosures.inline.hpp"
  39 #include "gc/shenandoah/shenandoahCollectionSet.hpp"
  40 #include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
  41 #include "gc/shenandoah/shenandoahConcurrentMark.inline.hpp"
  42 #include "gc/shenandoah/shenandoahConcurrentRoots.hpp"
  43 #include "gc/shenandoah/shenandoahControlThread.hpp"
  44 #include "gc/shenandoah/shenandoahFreeSet.hpp"
  45 #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
  46 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
  47 #include "gc/shenandoah/shenandoahHeapRegion.inline.hpp"
  48 #include "gc/shenandoah/shenandoahHeapRegionSet.hpp"
  49 #include "gc/shenandoah/shenandoahInitLogger.hpp"
  50 #include "gc/shenandoah/shenandoahMarkCompact.hpp"
  51 #include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"
  52 #include "gc/shenandoah/shenandoahMemoryPool.hpp"
  53 #include "gc/shenandoah/shenandoahMetrics.hpp"
  54 #include "gc/shenandoah/shenandoahMonitoringSupport.hpp"
  55 #include "gc/shenandoah/shenandoahOopClosures.inline.hpp"
  56 #include "gc/shenandoah/shenandoahPacer.inline.hpp"
  57 #include "gc/shenandoah/shenandoahPadding.hpp"
  58 #include "gc/shenandoah/shenandoahParallelCleaning.inline.hpp"
  59 #include "gc/shenandoah/shenandoahRootProcessor.inline.hpp"
  60 #include "gc/shenandoah/shenandoahStringDedup.hpp"
  61 #include "gc/shenandoah/shenandoahTaskqueue.hpp"
  62 #include "gc/shenandoah/shenandoahUtils.hpp"
  63 #include "gc/shenandoah/shenandoahVerifier.hpp"
  64 #include "gc/shenandoah/shenandoahCodeRoots.hpp"
  65 #include "gc/shenandoah/shenandoahVMOperations.hpp"
  66 #include "gc/shenandoah/shenandoahWorkGroup.hpp"
  67 #include "gc/shenandoah/shenandoahWorkerPolicy.hpp"
  68 #include "gc/shenandoah/mode/shenandoahIUMode.hpp"
  69 #include "gc/shenandoah/mode/shenandoahPassiveMode.hpp"
  70 #include "gc/shenandoah/mode/shenandoahSATBMode.hpp"
  71 #if INCLUDE_JFR
  72 #include "gc/shenandoah/shenandoahJfrSupport.hpp"
  73 #endif
  74 
  75 #include "memory/metaspace.hpp"
  76 #include "oops/compressedOops.inline.hpp"
  77 #include "runtime/atomic.hpp"
  78 #include "runtime/globals.hpp"
  79 #include "runtime/interfaceSupport.inline.hpp"
  80 #include "runtime/orderAccess.hpp"
  81 #include "runtime/safepointMechanism.hpp"
  82 #include "runtime/vmThread.hpp"
  83 #include "services/mallocTracker.hpp"
  84 #include "utilities/powerOfTwo.hpp"
  85 
  86 ShenandoahHeap* ShenandoahHeap::_heap = NULL;
  87 
  88 #ifdef ASSERT
  89 template <class T>
  90 void ShenandoahAssertToSpaceClosure::do_oop_work(T* p) {
  91   T o = RawAccess<>::oop_load(p);
  92   if (! CompressedOops::is_null(o)) {
  93     oop obj = CompressedOops::decode_not_null(o);
  94     shenandoah_assert_not_forwarded(p, obj);
  95   }
  96 }
  97 
  98 void ShenandoahAssertToSpaceClosure::do_oop(narrowOop* p) { do_oop_work(p); }
  99 void ShenandoahAssertToSpaceClosure::do_oop(oop* p)       { do_oop_work(p); }
 100 #endif
 101 
 102 class ShenandoahPretouchHeapTask : public AbstractGangTask {
 103 private:
 104   ShenandoahRegionIterator _regions;
 105   const size_t _page_size;
 106 public:
 107   ShenandoahPretouchHeapTask(size_t page_size) :
 108     AbstractGangTask("Shenandoah Pretouch Heap"),
 109     _page_size(page_size) {}
 110 
 111   virtual void work(uint worker_id) {
 112     ShenandoahHeapRegion* r = _regions.next();
 113     while (r != NULL) {
 114       if (r->is_committed()) {
 115         os::pretouch_memory(r->bottom(), r->end(), _page_size);
 116       }
 117       r = _regions.next();
 118     }
 119   }
 120 };
 121 
 122 class ShenandoahPretouchBitmapTask : public AbstractGangTask {
 123 private:
 124   ShenandoahRegionIterator _regions;
 125   char* _bitmap_base;
 126   const size_t _bitmap_size;
 127   const size_t _page_size;
 128 public:
 129   ShenandoahPretouchBitmapTask(char* bitmap_base, size_t bitmap_size, size_t page_size) :
 130     AbstractGangTask("Shenandoah Pretouch Bitmap"),
 131     _bitmap_base(bitmap_base),
 132     _bitmap_size(bitmap_size),
 133     _page_size(page_size) {}
 134 
 135   virtual void work(uint worker_id) {
 136     ShenandoahHeapRegion* r = _regions.next();
 137     while (r != NULL) {
 138       size_t start = r->index()       * ShenandoahHeapRegion::region_size_bytes() / MarkBitMap::heap_map_factor();
 139       size_t end   = (r->index() + 1) * ShenandoahHeapRegion::region_size_bytes() / MarkBitMap::heap_map_factor();
 140       assert (end <= _bitmap_size, "end is sane: " SIZE_FORMAT " < " SIZE_FORMAT, end, _bitmap_size);
 141 
 142       if (r->is_committed()) {
 143         os::pretouch_memory(_bitmap_base + start, _bitmap_base + end, _page_size);
 144       }
 145 
 146       r = _regions.next();
 147     }
 148   }
 149 };
 150 
 151 jint ShenandoahHeap::initialize() {
 152   //
 153   // Figure out heap sizing
 154   //
 155 
 156   size_t init_byte_size = InitialHeapSize;
 157   size_t min_byte_size  = MinHeapSize;
 158   size_t max_byte_size  = MaxHeapSize;
 159   size_t heap_alignment = HeapAlignment;
 160 
 161   size_t reg_size_bytes = ShenandoahHeapRegion::region_size_bytes();
 162 
 163   Universe::check_alignment(max_byte_size,  reg_size_bytes, "Shenandoah heap");
 164   Universe::check_alignment(init_byte_size, reg_size_bytes, "Shenandoah heap");
 165 
 166   _num_regions = ShenandoahHeapRegion::region_count();
 167 
 168   // Now we know the number of regions, initialize the heuristics.
 169   initialize_heuristics();
 170 
 171   size_t num_committed_regions = init_byte_size / reg_size_bytes;
 172   num_committed_regions = MIN2(num_committed_regions, _num_regions);
 173   assert(num_committed_regions <= _num_regions, "sanity");
 174   _initial_size = num_committed_regions * reg_size_bytes;
 175 
 176   size_t num_min_regions = min_byte_size / reg_size_bytes;
 177   num_min_regions = MIN2(num_min_regions, _num_regions);
 178   assert(num_min_regions <= _num_regions, "sanity");
 179   _minimum_size = num_min_regions * reg_size_bytes;
 180 
 181   _committed = _initial_size;
 182 
 183   size_t heap_page_size   = UseLargePages ? (size_t)os::large_page_size() : (size_t)os::vm_page_size();
 184   size_t bitmap_page_size = UseLargePages ? (size_t)os::large_page_size() : (size_t)os::vm_page_size();
 185   size_t region_page_size = UseLargePages ? (size_t)os::large_page_size() : (size_t)os::vm_page_size();
 186 
 187   //
 188   // Reserve and commit memory for heap
 189   //
 190 
 191   ReservedHeapSpace heap_rs = Universe::reserve_heap(max_byte_size, heap_alignment);
 192   initialize_reserved_region(heap_rs);
 193   _heap_region = MemRegion((HeapWord*)heap_rs.base(), heap_rs.size() / HeapWordSize);
 194   _heap_region_special = heap_rs.special();
 195 
 196   assert((((size_t) base()) & ShenandoahHeapRegion::region_size_bytes_mask()) == 0,
 197          "Misaligned heap: " PTR_FORMAT, p2i(base()));
 198 
 199 #if SHENANDOAH_OPTIMIZED_OBJTASK
 200   // The optimized ObjArrayChunkedTask takes some bits away from the full object bits.
 201   // Fail if we ever attempt to address more than we can.
 202   if ((uintptr_t)heap_rs.end() >= ObjArrayChunkedTask::max_addressable()) {
 203     FormatBuffer<512> buf("Shenandoah reserved [" PTR_FORMAT ", " PTR_FORMAT") for the heap, \n"
 204                           "but max object address is " PTR_FORMAT ". Try to reduce heap size, or try other \n"
 205                           "VM options that allocate heap at lower addresses (HeapBaseMinAddress, AllocateHeapAt, etc).",
 206                 p2i(heap_rs.base()), p2i(heap_rs.end()), ObjArrayChunkedTask::max_addressable());
 207     vm_exit_during_initialization("Fatal Error", buf);
 208   }
 209 #endif
 210 
 211   ReservedSpace sh_rs = heap_rs.first_part(max_byte_size);
 212   if (!_heap_region_special) {
 213     os::commit_memory_or_exit(sh_rs.base(), _initial_size, heap_alignment, false,
 214                               "Cannot commit heap memory");
 215   }
 216 
 217   //
 218   // Reserve and commit memory for bitmap(s)
 219   //
 220 
 221   _bitmap_size = MarkBitMap::compute_size(heap_rs.size());
 222   _bitmap_size = align_up(_bitmap_size, bitmap_page_size);
 223 
 224   size_t bitmap_bytes_per_region = reg_size_bytes / MarkBitMap::heap_map_factor();
 225 
 226   guarantee(bitmap_bytes_per_region != 0,
 227             "Bitmap bytes per region should not be zero");
 228   guarantee(is_power_of_2(bitmap_bytes_per_region),
 229             "Bitmap bytes per region should be power of two: " SIZE_FORMAT, bitmap_bytes_per_region);
 230 
 231   if (bitmap_page_size > bitmap_bytes_per_region) {
 232     _bitmap_regions_per_slice = bitmap_page_size / bitmap_bytes_per_region;
 233     _bitmap_bytes_per_slice = bitmap_page_size;
 234   } else {
 235     _bitmap_regions_per_slice = 1;
 236     _bitmap_bytes_per_slice = bitmap_bytes_per_region;
 237   }
 238 
 239   guarantee(_bitmap_regions_per_slice >= 1,
 240             "Should have at least one region per slice: " SIZE_FORMAT,
 241             _bitmap_regions_per_slice);
 242 
 243   guarantee(((_bitmap_bytes_per_slice) % bitmap_page_size) == 0,
 244             "Bitmap slices should be page-granular: bps = " SIZE_FORMAT ", page size = " SIZE_FORMAT,
 245             _bitmap_bytes_per_slice, bitmap_page_size);
 246 
 247   ReservedSpace bitmap(_bitmap_size, bitmap_page_size);
 248   MemTracker::record_virtual_memory_type(bitmap.base(), mtGC);
 249   _bitmap_region = MemRegion((HeapWord*) bitmap.base(), bitmap.size() / HeapWordSize);
 250   _bitmap_region_special = bitmap.special();
 251 
 252   size_t bitmap_init_commit = _bitmap_bytes_per_slice *
 253                               align_up(num_committed_regions, _bitmap_regions_per_slice) / _bitmap_regions_per_slice;
 254   bitmap_init_commit = MIN2(_bitmap_size, bitmap_init_commit);
 255   if (!_bitmap_region_special) {
 256     os::commit_memory_or_exit((char *) _bitmap_region.start(), bitmap_init_commit, bitmap_page_size, false,
 257                               "Cannot commit bitmap memory");
 258   }
 259 
 260   _marking_context = new ShenandoahMarkingContext(_heap_region, _bitmap_region, _num_regions);
 261 
 262   if (ShenandoahVerify) {
 263     ReservedSpace verify_bitmap(_bitmap_size, bitmap_page_size);
 264     if (!verify_bitmap.special()) {
 265       os::commit_memory_or_exit(verify_bitmap.base(), verify_bitmap.size(), bitmap_page_size, false,
 266                                 "Cannot commit verification bitmap memory");
 267     }
 268     MemTracker::record_virtual_memory_type(verify_bitmap.base(), mtGC);
 269     MemRegion verify_bitmap_region = MemRegion((HeapWord *) verify_bitmap.base(), verify_bitmap.size() / HeapWordSize);
 270     _verification_bit_map.initialize(_heap_region, verify_bitmap_region);
 271     _verifier = new ShenandoahVerifier(this, &_verification_bit_map);
 272   }
 273 
 274   // Reserve aux bitmap for use in object_iterate(). We don't commit it here.
 275   ReservedSpace aux_bitmap(_bitmap_size, bitmap_page_size);
 276   MemTracker::record_virtual_memory_type(aux_bitmap.base(), mtGC);
 277   _aux_bitmap_region = MemRegion((HeapWord*) aux_bitmap.base(), aux_bitmap.size() / HeapWordSize);
 278   _aux_bitmap_region_special = aux_bitmap.special();
 279   _aux_bit_map.initialize(_heap_region, _aux_bitmap_region);
 280 
 281   //
 282   // Create regions and region sets
 283   //
 284   size_t region_align = align_up(sizeof(ShenandoahHeapRegion), SHENANDOAH_CACHE_LINE_SIZE);
 285   size_t region_storage_size = align_up(region_align * _num_regions, region_page_size);
 286   region_storage_size = align_up(region_storage_size, os::vm_allocation_granularity());
 287 
 288   ReservedSpace region_storage(region_storage_size, region_page_size);
 289   MemTracker::record_virtual_memory_type(region_storage.base(), mtGC);
 290   if (!region_storage.special()) {
 291     os::commit_memory_or_exit(region_storage.base(), region_storage_size, region_page_size, false,
 292                               "Cannot commit region memory");
 293   }
 294 
 295   // Try to fit the collection set bitmap at lower addresses. This optimizes code generation for cset checks.
 296   // Go up until a sensible limit (subject to encoding constraints) and try to reserve the space there.
 297   // If not successful, bite a bullet and allocate at whatever address.
 298   {
 299     size_t cset_align = MAX2<size_t>(os::vm_page_size(), os::vm_allocation_granularity());
 300     size_t cset_size = align_up(((size_t) sh_rs.base() + sh_rs.size()) >> ShenandoahHeapRegion::region_size_bytes_shift(), cset_align);
 301 
 302     uintptr_t min = round_up_power_of_2(cset_align);
 303     uintptr_t max = (1u << 30u);
 304 
 305     for (uintptr_t addr = min; addr <= max; addr <<= 1u) {
 306       char* req_addr = (char*)addr;
 307       assert(is_aligned(req_addr, cset_align), "Should be aligned");
 308       ReservedSpace cset_rs(cset_size, cset_align, false, req_addr);
 309       if (cset_rs.is_reserved()) {
 310         assert(cset_rs.base() == req_addr, "Allocated where requested: " PTR_FORMAT ", " PTR_FORMAT, p2i(cset_rs.base()), addr);
 311         _collection_set = new ShenandoahCollectionSet(this, cset_rs, sh_rs.base());
 312         break;
 313       }
 314     }
 315 
 316     if (_collection_set == NULL) {
 317       ReservedSpace cset_rs(cset_size, cset_align, false);
 318       _collection_set = new ShenandoahCollectionSet(this, cset_rs, sh_rs.base());
 319     }
 320   }
 321 
 322   _regions = NEW_C_HEAP_ARRAY(ShenandoahHeapRegion*, _num_regions, mtGC);
 323   _free_set = new ShenandoahFreeSet(this, _num_regions);
 324 
 325   {
 326     ShenandoahHeapLocker locker(lock());
 327 
 328     for (size_t i = 0; i < _num_regions; i++) {
 329       HeapWord* start = (HeapWord*)sh_rs.base() + ShenandoahHeapRegion::region_size_words() * i;
 330       bool is_committed = i < num_committed_regions;
 331       void* loc = region_storage.base() + i * region_align;
 332 
 333       ShenandoahHeapRegion* r = new (loc) ShenandoahHeapRegion(start, i, is_committed);
 334       assert(is_aligned(r, SHENANDOAH_CACHE_LINE_SIZE), "Sanity");
 335 
 336       _marking_context->initialize_top_at_mark_start(r);
 337       _regions[i] = r;
 338       assert(!collection_set()->is_in(i), "New region should not be in collection set");
 339     }
 340 
 341     // Initialize to complete
 342     _marking_context->mark_complete();
 343 
 344     _free_set->rebuild();
 345   }
 346 
 347   if (AlwaysPreTouch) {
 348     // For NUMA, it is important to pre-touch the storage under bitmaps with worker threads,
 349     // before initialize() below zeroes it with initializing thread. For any given region,
 350     // we touch the region and the corresponding bitmaps from the same thread.
 351     ShenandoahPushWorkerScope scope(workers(), _max_workers, false);
 352 
 353     _pretouch_heap_page_size = heap_page_size;
 354     _pretouch_bitmap_page_size = bitmap_page_size;
 355 
 356 #ifdef LINUX
 357     // UseTransparentHugePages would madvise that backing memory can be coalesced into huge
 358     // pages. But, the kernel needs to know that every small page is used, in order to coalesce
 359     // them into huge one. Therefore, we need to pretouch with smaller pages.
 360     if (UseTransparentHugePages) {
 361       _pretouch_heap_page_size = (size_t)os::vm_page_size();
 362       _pretouch_bitmap_page_size = (size_t)os::vm_page_size();
 363     }
 364 #endif
 365 
 366     // OS memory managers may want to coalesce back-to-back pages. Make their jobs
 367     // simpler by pre-touching continuous spaces (heap and bitmap) separately.
 368 
 369     ShenandoahPretouchBitmapTask bcl(bitmap.base(), _bitmap_size, _pretouch_bitmap_page_size);
 370     _workers->run_task(&bcl);
 371 
 372     ShenandoahPretouchHeapTask hcl(_pretouch_heap_page_size);
 373     _workers->run_task(&hcl);
 374   }
 375 
 376   //
 377   // Initialize the rest of GC subsystems
 378   //
 379 
 380   _liveness_cache = NEW_C_HEAP_ARRAY(ShenandoahLiveData*, _max_workers, mtGC);
 381   for (uint worker = 0; worker < _max_workers; worker++) {
 382     _liveness_cache[worker] = NEW_C_HEAP_ARRAY(ShenandoahLiveData, _num_regions, mtGC);
 383     Copy::fill_to_bytes(_liveness_cache[worker], _num_regions * sizeof(ShenandoahLiveData));
 384   }
 385 
 386   // There should probably be Shenandoah-specific options for these,
 387   // just as there are G1-specific options.
 388   {
 389     ShenandoahSATBMarkQueueSet& satbqs = ShenandoahBarrierSet::satb_mark_queue_set();
 390     satbqs.set_process_completed_buffers_threshold(20); // G1SATBProcessCompletedThreshold
 391     satbqs.set_buffer_enqueue_threshold_percentage(60); // G1SATBBufferEnqueueingThresholdPercent
 392   }
 393 
 394   _monitoring_support = new ShenandoahMonitoringSupport(this);
 395   _phase_timings = new ShenandoahPhaseTimings(max_workers());
 396   ShenandoahStringDedup::initialize();
 397   ShenandoahCodeRoots::initialize();
 398 
 399   if (ShenandoahPacing) {
 400     _pacer = new ShenandoahPacer(this);
 401     _pacer->setup_for_idle();
 402   } else {
 403     _pacer = NULL;
 404   }
 405 
 406   _control_thread = new ShenandoahControlThread();
 407 
 408   _ref_proc_mt_processing = ParallelRefProcEnabled && (ParallelGCThreads > 1);
 409   _ref_proc_mt_discovery = _max_workers > 1;
 410 
 411   ShenandoahInitLogger::print();
 412 
 413   return JNI_OK;
 414 }
 415 
 416 void ShenandoahHeap::initialize_heuristics() {
 417   if (ShenandoahGCMode != NULL) {
 418     if (strcmp(ShenandoahGCMode, "satb") == 0) {
 419       _gc_mode = new ShenandoahSATBMode();
 420     } else if (strcmp(ShenandoahGCMode, "iu") == 0) {
 421       _gc_mode = new ShenandoahIUMode();
 422     } else if (strcmp(ShenandoahGCMode, "passive") == 0) {
 423       _gc_mode = new ShenandoahPassiveMode();
 424     } else {
 425       vm_exit_during_initialization("Unknown -XX:ShenandoahGCMode option");
 426     }
 427   } else {
 428     ShouldNotReachHere();
 429   }
 430   _gc_mode->initialize_flags();
 431   if (_gc_mode->is_diagnostic() && !UnlockDiagnosticVMOptions) {
 432     vm_exit_during_initialization(
 433             err_msg("GC mode \"%s\" is diagnostic, and must be enabled via -XX:+UnlockDiagnosticVMOptions.",
 434                     _gc_mode->name()));
 435   }
 436   if (_gc_mode->is_experimental() && !UnlockExperimentalVMOptions) {
 437     vm_exit_during_initialization(
 438             err_msg("GC mode \"%s\" is experimental, and must be enabled via -XX:+UnlockExperimentalVMOptions.",
 439                     _gc_mode->name()));
 440   }
 441 
 442   _heuristics = _gc_mode->initialize_heuristics();
 443 
 444   if (_heuristics->is_diagnostic() && !UnlockDiagnosticVMOptions) {
 445     vm_exit_during_initialization(
 446             err_msg("Heuristics \"%s\" is diagnostic, and must be enabled via -XX:+UnlockDiagnosticVMOptions.",
 447                     _heuristics->name()));
 448   }
 449   if (_heuristics->is_experimental() && !UnlockExperimentalVMOptions) {
 450     vm_exit_during_initialization(
 451             err_msg("Heuristics \"%s\" is experimental, and must be enabled via -XX:+UnlockExperimentalVMOptions.",
 452                     _heuristics->name()));
 453   }
 454 }
 455 
 456 #ifdef _MSC_VER
 457 #pragma warning( push )
 458 #pragma warning( disable:4355 ) // 'this' : used in base member initializer list
 459 #endif
 460 
 461 ShenandoahHeap::ShenandoahHeap(ShenandoahCollectorPolicy* policy) :
 462   CollectedHeap(),
 463   _initial_size(0),
 464   _used(0),
 465   _committed(0),
 466   _bytes_allocated_since_gc_start(0),
 467   _max_workers(MAX2(ConcGCThreads, ParallelGCThreads)),
 468   _workers(NULL),
 469   _safepoint_workers(NULL),
 470   _heap_region_special(false),
 471   _num_regions(0),
 472   _regions(NULL),
 473   _update_refs_iterator(this),
 474   _control_thread(NULL),
 475   _shenandoah_policy(policy),
 476   _heuristics(NULL),
 477   _free_set(NULL),
 478   _scm(new ShenandoahConcurrentMark()),
 479   _full_gc(new ShenandoahMarkCompact()),
 480   _pacer(NULL),
 481   _verifier(NULL),
 482   _phase_timings(NULL),
 483   _monitoring_support(NULL),
 484   _memory_pool(NULL),
 485   _stw_memory_manager("Shenandoah Pauses", "end of GC pause"),
 486   _cycle_memory_manager("Shenandoah Cycles", "end of GC cycle"),
 487   _gc_timer(new (ResourceObj::C_HEAP, mtGC) ConcurrentGCTimer()),
 488   _soft_ref_policy(),
 489   _log_min_obj_alignment_in_bytes(LogMinObjAlignmentInBytes),
 490   _ref_processor(NULL),
 491   _marking_context(NULL),
 492   _bitmap_size(0),
 493   _bitmap_regions_per_slice(0),
 494   _bitmap_bytes_per_slice(0),
 495   _bitmap_region_special(false),
 496   _aux_bitmap_region_special(false),
 497   _liveness_cache(NULL),
 498   _collection_set(NULL)
 499 {
 500   _heap = this;
 501 
 502   BarrierSet::set_barrier_set(new ShenandoahBarrierSet(this));
 503 
 504   _max_workers = MAX2(_max_workers, 1U);
 505   _workers = new ShenandoahWorkGang("Shenandoah GC Threads", _max_workers,
 506                             /* are_GC_task_threads */ true,
 507                             /* are_ConcurrentGC_threads */ true);
 508   if (_workers == NULL) {
 509     vm_exit_during_initialization("Failed necessary allocation.");
 510   } else {
 511     _workers->initialize_workers();
 512   }
 513 
 514   if (ParallelGCThreads > 1) {
 515     _safepoint_workers = new ShenandoahWorkGang("Safepoint Cleanup Thread",
 516                                                 ParallelGCThreads,
 517                       /* are_GC_task_threads */ false,
 518                  /* are_ConcurrentGC_threads */ false);
 519     _safepoint_workers->initialize_workers();
 520   }
 521 }
 522 
 523 #ifdef _MSC_VER
 524 #pragma warning( pop )
 525 #endif
 526 
 527 class ShenandoahResetBitmapTask : public AbstractGangTask {
 528 private:
 529   ShenandoahRegionIterator _regions;
 530 
 531 public:
 532   ShenandoahResetBitmapTask() :
 533     AbstractGangTask("Parallel Reset Bitmap Task") {}
 534 
 535   void work(uint worker_id) {
 536     ShenandoahHeapRegion* region = _regions.next();
 537     ShenandoahHeap* heap = ShenandoahHeap::heap();
 538     ShenandoahMarkingContext* const ctx = heap->marking_context();
 539     while (region != NULL) {
 540       if (heap->is_bitmap_slice_committed(region)) {
 541         ctx->clear_bitmap(region);
 542       }
 543       region = _regions.next();
 544     }
 545   }
 546 };
 547 
 548 void ShenandoahHeap::reset_mark_bitmap() {
 549   assert_gc_workers(_workers->active_workers());
 550   mark_incomplete_marking_context();
 551 
 552   ShenandoahResetBitmapTask task;
 553   _workers->run_task(&task);
 554 }
 555 
 556 void ShenandoahHeap::print_on(outputStream* st) const {
 557   st->print_cr("Shenandoah Heap");
 558   st->print_cr(" " SIZE_FORMAT "%s total, " SIZE_FORMAT "%s committed, " SIZE_FORMAT "%s used",
 559                byte_size_in_proper_unit(max_capacity()), proper_unit_for_byte_size(max_capacity()),
 560                byte_size_in_proper_unit(committed()),    proper_unit_for_byte_size(committed()),
 561                byte_size_in_proper_unit(used()),         proper_unit_for_byte_size(used()));
 562   st->print_cr(" " SIZE_FORMAT " x " SIZE_FORMAT"%s regions",
 563                num_regions(),
 564                byte_size_in_proper_unit(ShenandoahHeapRegion::region_size_bytes()),
 565                proper_unit_for_byte_size(ShenandoahHeapRegion::region_size_bytes()));
 566 
 567   st->print("Status: ");
 568   if (has_forwarded_objects())                 st->print("has forwarded objects, ");
 569   if (is_concurrent_mark_in_progress())        st->print("marking, ");
 570   if (is_evacuation_in_progress())             st->print("evacuating, ");
 571   if (is_update_refs_in_progress())            st->print("updating refs, ");
 572   if (is_degenerated_gc_in_progress())         st->print("degenerated gc, ");
 573   if (is_full_gc_in_progress())                st->print("full gc, ");
 574   if (is_full_gc_move_in_progress())           st->print("full gc move, ");
 575   if (is_concurrent_weak_root_in_progress())   st->print("concurrent weak roots, ");
 576   if (is_concurrent_strong_root_in_progress() &&
 577       !is_concurrent_weak_root_in_progress())  st->print("concurrent strong roots, ");
 578 
 579   if (cancelled_gc()) {
 580     st->print("cancelled");
 581   } else {
 582     st->print("not cancelled");
 583   }
 584   st->cr();
 585 
 586   st->print_cr("Reserved region:");
 587   st->print_cr(" - [" PTR_FORMAT ", " PTR_FORMAT ") ",
 588                p2i(reserved_region().start()),
 589                p2i(reserved_region().end()));
 590 
 591   ShenandoahCollectionSet* cset = collection_set();
 592   st->print_cr("Collection set:");
 593   if (cset != NULL) {
 594     st->print_cr(" - map (vanilla): " PTR_FORMAT, p2i(cset->map_address()));
 595     st->print_cr(" - map (biased):  " PTR_FORMAT, p2i(cset->biased_map_address()));
 596   } else {
 597     st->print_cr(" (NULL)");
 598   }
 599 
 600   st->cr();
 601   MetaspaceUtils::print_on(st);
 602 
 603   if (Verbose) {
 604     print_heap_regions_on(st);
 605   }
 606 }
 607 
 608 class ShenandoahInitWorkerGCLABClosure : public ThreadClosure {
 609 public:
 610   void do_thread(Thread* thread) {
 611     assert(thread != NULL, "Sanity");
 612     assert(thread->is_Worker_thread(), "Only worker thread expected");
 613     ShenandoahThreadLocalData::initialize_gclab(thread);
 614   }
 615 };
 616 
 617 void ShenandoahHeap::post_initialize() {
 618   CollectedHeap::post_initialize();
 619   MutexLocker ml(Threads_lock);
 620 
 621   ShenandoahInitWorkerGCLABClosure init_gclabs;
 622   _workers->threads_do(&init_gclabs);
 623 
 624   // gclab can not be initialized early during VM startup, as it can not determinate its max_size.
 625   // Now, we will let WorkGang to initialize gclab when new worker is created.
 626   _workers->set_initialize_gclab();
 627 
 628   _scm->initialize(_max_workers);
 629   _full_gc->initialize(_gc_timer);
 630 
 631   ref_processing_init();
 632 
 633   _heuristics->initialize();
 634 
 635   JFR_ONLY(ShenandoahJFRSupport::register_jfr_type_serializers());
 636 }
 637 
 638 size_t ShenandoahHeap::used() const {
 639   return Atomic::load_acquire(&_used);
 640 }
 641 
 642 size_t ShenandoahHeap::committed() const {
 643   OrderAccess::acquire();
 644   return _committed;
 645 }
 646 
 647 void ShenandoahHeap::increase_committed(size_t bytes) {
 648   shenandoah_assert_heaplocked_or_safepoint();
 649   _committed += bytes;
 650 }
 651 
 652 void ShenandoahHeap::decrease_committed(size_t bytes) {
 653   shenandoah_assert_heaplocked_or_safepoint();
 654   _committed -= bytes;
 655 }
 656 
 657 void ShenandoahHeap::increase_used(size_t bytes) {
 658   Atomic::add(&_used, bytes);
 659 }
 660 
 661 void ShenandoahHeap::set_used(size_t bytes) {
 662   Atomic::release_store_fence(&_used, bytes);
 663 }
 664 
 665 void ShenandoahHeap::decrease_used(size_t bytes) {
 666   assert(used() >= bytes, "never decrease heap size by more than we've left");
 667   Atomic::sub(&_used, bytes);
 668 }
 669 
 670 void ShenandoahHeap::increase_allocated(size_t bytes) {
 671   Atomic::add(&_bytes_allocated_since_gc_start, bytes);
 672 }
 673 
 674 void ShenandoahHeap::notify_mutator_alloc_words(size_t words, bool waste) {
 675   size_t bytes = words * HeapWordSize;
 676   if (!waste) {
 677     increase_used(bytes);
 678   }
 679   increase_allocated(bytes);
 680   if (ShenandoahPacing) {
 681     control_thread()->pacing_notify_alloc(words);
 682     if (waste) {
 683       pacer()->claim_for_alloc(words, true);
 684     }
 685   }
 686 }
 687 
 688 size_t ShenandoahHeap::capacity() const {
 689   return committed();
 690 }
 691 
 692 size_t ShenandoahHeap::max_capacity() const {
 693   return _num_regions * ShenandoahHeapRegion::region_size_bytes();
 694 }
 695 
 696 size_t ShenandoahHeap::min_capacity() const {
 697   return _minimum_size;
 698 }
 699 
 700 size_t ShenandoahHeap::initial_capacity() const {
 701   return _initial_size;
 702 }
 703 
 704 bool ShenandoahHeap::is_in(const void* p) const {
 705   HeapWord* heap_base = (HeapWord*) base();
 706   HeapWord* last_region_end = heap_base + ShenandoahHeapRegion::region_size_words() * num_regions();
 707   return p >= heap_base && p < last_region_end;
 708 }
 709 
 710 void ShenandoahHeap::op_uncommit(double shrink_before) {
 711   assert (ShenandoahUncommit, "should be enabled");
 712 
 713   // Application allocates from the beginning of the heap, and GC allocates at
 714   // the end of it. It is more efficient to uncommit from the end, so that applications
 715   // could enjoy the near committed regions. GC allocations are much less frequent,
 716   // and therefore can accept the committing costs.
 717 
 718   size_t count = 0;
 719   for (size_t i = num_regions(); i > 0; i--) { // care about size_t underflow
 720     ShenandoahHeapRegion* r = get_region(i - 1);
 721     if (r->is_empty_committed() && (r->empty_time() < shrink_before)) {
 722       ShenandoahHeapLocker locker(lock());
 723       if (r->is_empty_committed()) {
 724         // Do not uncommit below minimal capacity
 725         if (committed() < min_capacity() + ShenandoahHeapRegion::region_size_bytes()) {
 726           break;
 727         }
 728 
 729         r->make_uncommitted();
 730         count++;
 731       }
 732     }
 733     SpinPause(); // allow allocators to take the lock
 734   }
 735 
 736   if (count > 0) {
 737     control_thread()->notify_heap_changed();
 738   }
 739 }
 740 
 741 HeapWord* ShenandoahHeap::allocate_from_gclab_slow(Thread* thread, size_t size) {
 742   // New object should fit the GCLAB size
 743   size_t min_size = MAX2(size, PLAB::min_size());
 744 
 745   // Figure out size of new GCLAB, looking back at heuristics. Expand aggressively.
 746   size_t new_size = ShenandoahThreadLocalData::gclab_size(thread) * 2;
 747   new_size = MIN2(new_size, PLAB::max_size());
 748   new_size = MAX2(new_size, PLAB::min_size());
 749 
 750   // Record new heuristic value even if we take any shortcut. This captures
 751   // the case when moderately-sized objects always take a shortcut. At some point,
 752   // heuristics should catch up with them.
 753   ShenandoahThreadLocalData::set_gclab_size(thread, new_size);
 754 
 755   if (new_size < size) {
 756     // New size still does not fit the object. Fall back to shared allocation.
 757     // This avoids retiring perfectly good GCLABs, when we encounter a large object.
 758     return NULL;
 759   }
 760 
 761   // Retire current GCLAB, and allocate a new one.
 762   PLAB* gclab = ShenandoahThreadLocalData::gclab(thread);
 763   gclab->retire();
 764 
 765   size_t actual_size = 0;
 766   HeapWord* gclab_buf = allocate_new_gclab(min_size, new_size, &actual_size);
 767   if (gclab_buf == NULL) {
 768     return NULL;
 769   }
 770 
 771   assert (size <= actual_size, "allocation should fit");
 772 
 773   if (ZeroTLAB) {
 774     // ..and clear it.
 775     Copy::zero_to_words(gclab_buf, actual_size);
 776   } else {
 777     // ...and zap just allocated object.
 778 #ifdef ASSERT
 779     // Skip mangling the space corresponding to the object header to
 780     // ensure that the returned space is not considered parsable by
 781     // any concurrent GC thread.
 782     size_t hdr_size = oopDesc::header_size();
 783     Copy::fill_to_words(gclab_buf + hdr_size, actual_size - hdr_size, badHeapWordVal);
 784 #endif // ASSERT
 785   }
 786   gclab->set_buf(gclab_buf, actual_size);
 787   return gclab->allocate(size);
 788 }
 789 
 790 HeapWord* ShenandoahHeap::allocate_new_tlab(size_t min_size,
 791                                             size_t requested_size,
 792                                             size_t* actual_size) {
 793   ShenandoahAllocRequest req = ShenandoahAllocRequest::for_tlab(min_size, requested_size);
 794   HeapWord* res = allocate_memory(req);
 795   if (res != NULL) {
 796     *actual_size = req.actual_size();
 797   } else {
 798     *actual_size = 0;
 799   }
 800   return res;
 801 }
 802 
 803 HeapWord* ShenandoahHeap::allocate_new_gclab(size_t min_size,
 804                                              size_t word_size,
 805                                              size_t* actual_size) {
 806   ShenandoahAllocRequest req = ShenandoahAllocRequest::for_gclab(min_size, word_size);
 807   HeapWord* res = allocate_memory(req);
 808   if (res != NULL) {
 809     *actual_size = req.actual_size();
 810   } else {
 811     *actual_size = 0;
 812   }
 813   return res;
 814 }
 815 
 816 HeapWord* ShenandoahHeap::allocate_memory(ShenandoahAllocRequest& req) {
 817   intptr_t pacer_epoch = 0;
 818   bool in_new_region = false;
 819   HeapWord* result = NULL;
 820 
 821   if (req.is_mutator_alloc()) {
 822     if (ShenandoahPacing) {
 823       pacer()->pace_for_alloc(req.size());
 824       pacer_epoch = pacer()->epoch();
 825     }
 826 
 827     if (!ShenandoahAllocFailureALot || !should_inject_alloc_failure()) {
 828       result = allocate_memory_under_lock(req, in_new_region);
 829     }
 830 
 831     // Allocation failed, block until control thread reacted, then retry allocation.
 832     //
 833     // It might happen that one of the threads requesting allocation would unblock
 834     // way later after GC happened, only to fail the second allocation, because
 835     // other threads have already depleted the free storage. In this case, a better
 836     // strategy is to try again, as long as GC makes progress.
 837     //
 838     // Then, we need to make sure the allocation was retried after at least one
 839     // Full GC, which means we want to try more than ShenandoahFullGCThreshold times.
 840 
 841     size_t tries = 0;
 842 
 843     while (result == NULL && _progress_last_gc.is_set()) {
 844       tries++;
 845       control_thread()->handle_alloc_failure(req);
 846       result = allocate_memory_under_lock(req, in_new_region);
 847     }
 848 
 849     while (result == NULL && tries <= ShenandoahFullGCThreshold) {
 850       tries++;
 851       control_thread()->handle_alloc_failure(req);
 852       result = allocate_memory_under_lock(req, in_new_region);
 853     }
 854 
 855   } else {
 856     assert(req.is_gc_alloc(), "Can only accept GC allocs here");
 857     result = allocate_memory_under_lock(req, in_new_region);
 858     // Do not call handle_alloc_failure() here, because we cannot block.
 859     // The allocation failure would be handled by the LRB slowpath with handle_alloc_failure_evac().
 860   }
 861 
 862   if (in_new_region) {
 863     control_thread()->notify_heap_changed();
 864   }
 865 
 866   if (result != NULL) {
 867     size_t requested = req.size();
 868     size_t actual = req.actual_size();
 869 
 870     assert (req.is_lab_alloc() || (requested == actual),
 871             "Only LAB allocations are elastic: %s, requested = " SIZE_FORMAT ", actual = " SIZE_FORMAT,
 872             ShenandoahAllocRequest::alloc_type_to_string(req.type()), requested, actual);
 873 
 874     if (req.is_mutator_alloc()) {
 875       notify_mutator_alloc_words(actual, false);
 876 
 877       // If we requested more than we were granted, give the rest back to pacer.
 878       // This only matters if we are in the same pacing epoch: do not try to unpace
 879       // over the budget for the other phase.
 880       if (ShenandoahPacing && (pacer_epoch > 0) && (requested > actual)) {
 881         pacer()->unpace_for_alloc(pacer_epoch, requested - actual);
 882       }
 883     } else {
 884       increase_used(actual*HeapWordSize);
 885     }
 886   }
 887 
 888   return result;
 889 }
 890 
 891 HeapWord* ShenandoahHeap::allocate_memory_under_lock(ShenandoahAllocRequest& req, bool& in_new_region) {
 892   ShenandoahHeapLocker locker(lock());
 893   return _free_set->allocate(req, in_new_region);
 894 }
 895 
 896 HeapWord* ShenandoahHeap::mem_allocate(size_t size,
 897                                         bool*  gc_overhead_limit_was_exceeded) {
 898   ShenandoahAllocRequest req = ShenandoahAllocRequest::for_shared(size);
 899   return allocate_memory(req);
 900 }
 901 
 902 MetaWord* ShenandoahHeap::satisfy_failed_metadata_allocation(ClassLoaderData* loader_data,
 903                                                              size_t size,
 904                                                              Metaspace::MetadataType mdtype) {
 905   MetaWord* result;
 906 
 907   // Inform metaspace OOM to GC heuristics if class unloading is possible.
 908   if (heuristics()->can_unload_classes()) {
 909     ShenandoahHeuristics* h = heuristics();
 910     h->record_metaspace_oom();
 911   }
 912 
 913   // Expand and retry allocation
 914   result = loader_data->metaspace_non_null()->expand_and_allocate(size, mdtype);
 915   if (result != NULL) {
 916     return result;
 917   }
 918 
 919   // Start full GC
 920   collect(GCCause::_metadata_GC_clear_soft_refs);
 921 
 922   // Retry allocation
 923   result = loader_data->metaspace_non_null()->allocate(size, mdtype);
 924   if (result != NULL) {
 925     return result;
 926   }
 927 
 928   // Expand and retry allocation
 929   result = loader_data->metaspace_non_null()->expand_and_allocate(size, mdtype);
 930   if (result != NULL) {
 931     return result;
 932   }
 933 
 934   // Out of memory
 935   return NULL;
 936 }
 937 
 938 class ShenandoahConcurrentEvacuateRegionObjectClosure : public ObjectClosure {
 939 private:
 940   ShenandoahHeap* const _heap;
 941   Thread* const _thread;
 942 public:
 943   ShenandoahConcurrentEvacuateRegionObjectClosure(ShenandoahHeap* heap) :
 944     _heap(heap), _thread(Thread::current()) {}
 945 
 946   void do_object(oop p) {
 947     shenandoah_assert_marked(NULL, p);
 948     if (!p->is_forwarded()) {
 949       _heap->evacuate_object(p, _thread);
 950     }
 951   }
 952 };
 953 
 954 class ShenandoahEvacuationTask : public AbstractGangTask {
 955 private:
 956   ShenandoahHeap* const _sh;
 957   ShenandoahCollectionSet* const _cs;
 958   bool _concurrent;
 959 public:
 960   ShenandoahEvacuationTask(ShenandoahHeap* sh,
 961                            ShenandoahCollectionSet* cs,
 962                            bool concurrent) :
 963     AbstractGangTask("Parallel Evacuation Task"),
 964     _sh(sh),
 965     _cs(cs),
 966     _concurrent(concurrent)
 967   {}
 968 
 969   void work(uint worker_id) {
 970     if (_concurrent) {
 971       ShenandoahConcurrentWorkerSession worker_session(worker_id);
 972       ShenandoahSuspendibleThreadSetJoiner stsj(ShenandoahSuspendibleWorkers);
 973       ShenandoahEvacOOMScope oom_evac_scope;
 974       do_work();
 975     } else {
 976       ShenandoahParallelWorkerSession worker_session(worker_id);
 977       ShenandoahEvacOOMScope oom_evac_scope;
 978       do_work();
 979     }
 980   }
 981 
 982 private:
 983   void do_work() {
 984     ShenandoahConcurrentEvacuateRegionObjectClosure cl(_sh);
 985     ShenandoahHeapRegion* r;
 986     while ((r =_cs->claim_next()) != NULL) {
 987       assert(r->has_live(), "Region " SIZE_FORMAT " should have been reclaimed early", r->index());
 988       _sh->marked_object_iterate(r, &cl);
 989 
 990       if (ShenandoahPacing) {
 991         _sh->pacer()->report_evac(r->used() >> LogHeapWordSize);
 992       }
 993 
 994       if (_sh->check_cancelled_gc_and_yield(_concurrent)) {
 995         break;
 996       }
 997     }
 998   }
 999 };
1000 
1001 void ShenandoahHeap::trash_cset_regions() {
1002   ShenandoahHeapLocker locker(lock());
1003 
1004   ShenandoahCollectionSet* set = collection_set();
1005   ShenandoahHeapRegion* r;
1006   set->clear_current_index();
1007   while ((r = set->next()) != NULL) {
1008     r->make_trash();
1009   }
1010   collection_set()->clear();
1011 }
1012 
1013 void ShenandoahHeap::print_heap_regions_on(outputStream* st) const {
1014   st->print_cr("Heap Regions:");
1015   st->print_cr("EU=empty-uncommitted, EC=empty-committed, R=regular, H=humongous start, HC=humongous continuation, CS=collection set, T=trash, P=pinned");
1016   st->print_cr("BTE=bottom/top/end, U=used, T=TLAB allocs, G=GCLAB allocs, S=shared allocs, L=live data");
1017   st->print_cr("R=root, CP=critical pins, TAMS=top-at-mark-start, UWM=update watermark");
1018   st->print_cr("SN=alloc sequence number");
1019 
1020   for (size_t i = 0; i < num_regions(); i++) {
1021     get_region(i)->print_on(st);
1022   }
1023 }
1024 
1025 void ShenandoahHeap::trash_humongous_region_at(ShenandoahHeapRegion* start) {
1026   assert(start->is_humongous_start(), "reclaim regions starting with the first one");
1027 
1028   oop humongous_obj = oop(start->bottom());
1029   size_t size = humongous_obj->size();
1030   size_t required_regions = ShenandoahHeapRegion::required_regions(size * HeapWordSize);
1031   size_t index = start->index() + required_regions - 1;
1032 
1033   assert(!start->has_live(), "liveness must be zero");
1034 
1035   for(size_t i = 0; i < required_regions; i++) {
1036     // Reclaim from tail. Otherwise, assertion fails when printing region to trace log,
1037     // as it expects that every region belongs to a humongous region starting with a humongous start region.
1038     ShenandoahHeapRegion* region = get_region(index --);
1039 
1040     assert(region->is_humongous(), "expect correct humongous start or continuation");
1041     assert(!region->is_cset(), "Humongous region should not be in collection set");
1042 
1043     region->make_trash_immediate();
1044   }
1045 }
1046 
1047 class ShenandoahRetireGCLABClosure : public ThreadClosure {
1048 public:
1049   void do_thread(Thread* thread) {
1050     PLAB* gclab = ShenandoahThreadLocalData::gclab(thread);
1051     assert(gclab != NULL, "GCLAB should be initialized for %s", thread->name());
1052     gclab->retire();
1053   }
1054 };
1055 
1056 void ShenandoahHeap::make_parsable(bool retire_tlabs) {
1057   if (UseTLAB) {
1058     CollectedHeap::ensure_parsability(retire_tlabs);
1059   }
1060   ShenandoahRetireGCLABClosure cl;
1061   for (JavaThreadIteratorWithHandle jtiwh; JavaThread *t = jtiwh.next(); ) {
1062     cl.do_thread(t);
1063   }
1064   workers()->threads_do(&cl);
1065 }
1066 
1067 void ShenandoahHeap::resize_tlabs() {
1068   CollectedHeap::resize_all_tlabs();
1069 }
1070 
1071 class ShenandoahEvacuateUpdateRootsTask : public AbstractGangTask {
1072 private:
1073   ShenandoahRootEvacuator* _rp;
1074 
1075 public:
1076   ShenandoahEvacuateUpdateRootsTask(ShenandoahRootEvacuator* rp) :
1077     AbstractGangTask("Shenandoah evacuate and update roots"),
1078     _rp(rp) {}
1079 
1080   void work(uint worker_id) {
1081     ShenandoahParallelWorkerSession worker_session(worker_id);
1082     ShenandoahEvacOOMScope oom_evac_scope;
1083     ShenandoahEvacuateUpdateRootsClosure<> cl;
1084     MarkingCodeBlobClosure blobsCl(&cl, CodeBlobToOopClosure::FixRelocations);
1085     _rp->roots_do(worker_id, &cl);
1086   }
1087 };
1088 
1089 void ShenandoahHeap::evacuate_and_update_roots() {
1090 #if COMPILER2_OR_JVMCI
1091   DerivedPointerTable::clear();
1092 #endif
1093   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Only iterate roots while world is stopped");
1094   {
1095     // Include concurrent roots if current cycle can not process those roots concurrently
1096     ShenandoahRootEvacuator rp(workers()->active_workers(),
1097                                ShenandoahPhaseTimings::init_evac,
1098                                !ShenandoahConcurrentRoots::should_do_concurrent_roots(),
1099                                !ShenandoahConcurrentRoots::should_do_concurrent_class_unloading());
1100     ShenandoahEvacuateUpdateRootsTask roots_task(&rp);
1101     workers()->run_task(&roots_task);
1102   }
1103 
1104 #if COMPILER2_OR_JVMCI
1105   DerivedPointerTable::update_pointers();
1106 #endif
1107 }
1108 
1109 // Returns size in bytes
1110 size_t ShenandoahHeap::unsafe_max_tlab_alloc(Thread *thread) const {
1111   if (ShenandoahElasticTLAB) {
1112     // With Elastic TLABs, return the max allowed size, and let the allocation path
1113     // figure out the safe size for current allocation.
1114     return ShenandoahHeapRegion::max_tlab_size_bytes();
1115   } else {
1116     return MIN2(_free_set->unsafe_peek_free(), ShenandoahHeapRegion::max_tlab_size_bytes());
1117   }
1118 }
1119 
1120 size_t ShenandoahHeap::max_tlab_size() const {
1121   // Returns size in words
1122   return ShenandoahHeapRegion::max_tlab_size_words();
1123 }
1124 
1125 class ShenandoahRetireAndResetGCLABClosure : public ThreadClosure {
1126 public:
1127   void do_thread(Thread* thread) {
1128     PLAB* gclab = ShenandoahThreadLocalData::gclab(thread);
1129     gclab->retire();
1130     if (ShenandoahThreadLocalData::gclab_size(thread) > 0) {
1131       ShenandoahThreadLocalData::set_gclab_size(thread, 0);
1132     }
1133   }
1134 };
1135 
1136 void ShenandoahHeap::retire_and_reset_gclabs() {
1137   ShenandoahRetireAndResetGCLABClosure cl;
1138   for (JavaThreadIteratorWithHandle jtiwh; JavaThread *t = jtiwh.next(); ) {
1139     cl.do_thread(t);
1140   }
1141   workers()->threads_do(&cl);
1142 }
1143 
1144 void ShenandoahHeap::collect(GCCause::Cause cause) {
1145   control_thread()->request_gc(cause);
1146 }
1147 
1148 void ShenandoahHeap::do_full_collection(bool clear_all_soft_refs) {
1149   //assert(false, "Shouldn't need to do full collections");
1150 }
1151 
1152 HeapWord* ShenandoahHeap::block_start(const void* addr) const {
1153   ShenandoahHeapRegion* r = heap_region_containing(addr);
1154   if (r != NULL) {
1155     return r->block_start(addr);
1156   }
1157   return NULL;
1158 }
1159 
1160 bool ShenandoahHeap::block_is_obj(const HeapWord* addr) const {
1161   ShenandoahHeapRegion* r = heap_region_containing(addr);
1162   return r->block_is_obj(addr);
1163 }
1164 
1165 bool ShenandoahHeap::print_location(outputStream* st, void* addr) const {
1166   return BlockLocationPrinter<ShenandoahHeap>::print_location(st, addr);
1167 }
1168 
1169 jlong ShenandoahHeap::millis_since_last_gc() {
1170   double v = heuristics()->time_since_last_gc() * 1000;
1171   assert(0 <= v && v <= max_jlong, "value should fit: %f", v);
1172   return (jlong)v;
1173 }
1174 
1175 void ShenandoahHeap::prepare_for_verify() {
1176   if (SafepointSynchronize::is_at_safepoint() || ! UseTLAB) {
1177     make_parsable(false);
1178   }
1179 }
1180 
1181 void ShenandoahHeap::print_gc_threads_on(outputStream* st) const {
1182   workers()->print_worker_threads_on(st);
1183   if (ShenandoahStringDedup::is_enabled()) {
1184     ShenandoahStringDedup::print_worker_threads_on(st);
1185   }
1186 }
1187 
1188 void ShenandoahHeap::gc_threads_do(ThreadClosure* tcl) const {
1189   workers()->threads_do(tcl);
1190   if (_safepoint_workers != NULL) {
1191     _safepoint_workers->threads_do(tcl);
1192   }
1193   if (ShenandoahStringDedup::is_enabled()) {
1194     ShenandoahStringDedup::threads_do(tcl);
1195   }
1196 }
1197 
1198 void ShenandoahHeap::print_tracing_info() const {
1199   LogTarget(Info, gc, stats) lt;
1200   if (lt.is_enabled()) {
1201     ResourceMark rm;
1202     LogStream ls(lt);
1203 
1204     phase_timings()->print_global_on(&ls);
1205 
1206     ls.cr();
1207     ls.cr();
1208 
1209     shenandoah_policy()->print_gc_stats(&ls);
1210 
1211     ls.cr();
1212     ls.cr();
1213 
1214     if (ShenandoahPacing) {
1215       pacer()->print_on(&ls);
1216     }
1217 
1218     ls.cr();
1219     ls.cr();
1220   }
1221 }
1222 
1223 void ShenandoahHeap::verify(VerifyOption vo) {
1224   if (ShenandoahSafepoint::is_at_shenandoah_safepoint()) {
1225     if (ShenandoahVerify) {
1226       verifier()->verify_generic(vo);
1227     } else {
1228       // TODO: Consider allocating verification bitmaps on demand,
1229       // and turn this on unconditionally.
1230     }
1231   }
1232 }
1233 size_t ShenandoahHeap::tlab_capacity(Thread *thr) const {
1234   return _free_set->capacity();
1235 }
1236 
1237 class ObjectIterateScanRootClosure : public BasicOopIterateClosure {
1238 private:
1239   MarkBitMap* _bitmap;
1240   Stack<oop,mtGC>* _oop_stack;
1241   ShenandoahHeap* const _heap;
1242   ShenandoahMarkingContext* const _marking_context;
1243 
1244   template <class T>
1245   void do_oop_work(T* p) {
1246     T o = RawAccess<>::oop_load(p);
1247     if (!CompressedOops::is_null(o)) {
1248       oop obj = CompressedOops::decode_not_null(o);
1249       if (_heap->is_concurrent_weak_root_in_progress() && !_marking_context->is_marked(obj)) {
1250         // There may be dead oops in weak roots in concurrent root phase, do not touch them.
1251         return;
1252       }
1253       obj = ShenandoahBarrierSet::resolve_forwarded_not_null(obj);
1254 
1255       assert(oopDesc::is_oop(obj), "must be a valid oop");
1256       if (!_bitmap->is_marked(obj)) {
1257         _bitmap->mark(obj);
1258         _oop_stack->push(obj);
1259       }
1260     }
1261   }
1262 public:
1263   ObjectIterateScanRootClosure(MarkBitMap* bitmap, Stack<oop,mtGC>* oop_stack) :
1264     _bitmap(bitmap), _oop_stack(oop_stack), _heap(ShenandoahHeap::heap()),
1265     _marking_context(_heap->marking_context()) {}
1266   void do_oop(oop* p)       { do_oop_work(p); }
1267   void do_oop(narrowOop* p) { do_oop_work(p); }
1268 };
1269 
1270 /*
1271  * This is public API, used in preparation of object_iterate().
1272  * Since we don't do linear scan of heap in object_iterate() (see comment below), we don't
1273  * need to make the heap parsable. For Shenandoah-internal linear heap scans that we can
1274  * control, we call SH::make_tlabs_parsable().
1275  */
1276 void ShenandoahHeap::ensure_parsability(bool retire_tlabs) {
1277   // No-op.
1278 }
1279 
1280 /*
1281  * Iterates objects in the heap. This is public API, used for, e.g., heap dumping.
1282  *
1283  * We cannot safely iterate objects by doing a linear scan at random points in time. Linear
1284  * scanning needs to deal with dead objects, which may have dead Klass* pointers (e.g.
1285  * calling oopDesc::size() would crash) or dangling reference fields (crashes) etc. Linear
1286  * scanning therefore depends on having a valid marking bitmap to support it. However, we only
1287  * have a valid marking bitmap after successful marking. In particular, we *don't* have a valid
1288  * marking bitmap during marking, after aborted marking or during/after cleanup (when we just
1289  * wiped the bitmap in preparation for next marking).
1290  *
1291  * For all those reasons, we implement object iteration as a single marking traversal, reporting
1292  * objects as we mark+traverse through the heap, starting from GC roots. JVMTI IterateThroughHeap
1293  * is allowed to report dead objects, but is not required to do so.
1294  */
1295 void ShenandoahHeap::object_iterate(ObjectClosure* cl) {
1296   assert(SafepointSynchronize::is_at_safepoint(), "safe iteration is only available during safepoints");
1297   if (!_aux_bitmap_region_special && !os::commit_memory((char*)_aux_bitmap_region.start(), _aux_bitmap_region.byte_size(), false)) {
1298     log_warning(gc)("Could not commit native memory for auxiliary marking bitmap for heap iteration");
1299     return;
1300   }
1301 
1302   // Reset bitmap
1303   _aux_bit_map.clear();
1304 
1305   Stack<oop,mtGC> oop_stack;
1306 
1307   // First, we process GC roots according to current GC cycle. This populates the work stack with initial objects.
1308   ShenandoahHeapIterationRootScanner rp;
1309   ObjectIterateScanRootClosure oops(&_aux_bit_map, &oop_stack);
1310 
1311   rp.roots_do(&oops);
1312 
1313   // Work through the oop stack to traverse heap.
1314   while (! oop_stack.is_empty()) {
1315     oop obj = oop_stack.pop();
1316     assert(oopDesc::is_oop(obj), "must be a valid oop");
1317     cl->do_object(obj);
1318     obj->oop_iterate(&oops);
1319   }
1320 
1321   assert(oop_stack.is_empty(), "should be empty");
1322 
1323   if (!_aux_bitmap_region_special && !os::uncommit_memory((char*)_aux_bitmap_region.start(), _aux_bitmap_region.byte_size())) {
1324     log_warning(gc)("Could not uncommit native memory for auxiliary marking bitmap for heap iteration");
1325   }
1326 }
1327 
1328 // Keep alive an object that was loaded with AS_NO_KEEPALIVE.
1329 void ShenandoahHeap::keep_alive(oop obj) {
1330   if (is_concurrent_mark_in_progress()) {
1331     ShenandoahBarrierSet::barrier_set()->enqueue(obj);
1332   }
1333 }
1334 
1335 void ShenandoahHeap::heap_region_iterate(ShenandoahHeapRegionClosure* blk) const {
1336   for (size_t i = 0; i < num_regions(); i++) {
1337     ShenandoahHeapRegion* current = get_region(i);
1338     blk->heap_region_do(current);
1339   }
1340 }
1341 
1342 class ShenandoahParallelHeapRegionTask : public AbstractGangTask {
1343 private:
1344   ShenandoahHeap* const _heap;
1345   ShenandoahHeapRegionClosure* const _blk;
1346 
1347   shenandoah_padding(0);
1348   volatile size_t _index;
1349   shenandoah_padding(1);
1350 
1351 public:
1352   ShenandoahParallelHeapRegionTask(ShenandoahHeapRegionClosure* blk) :
1353           AbstractGangTask("Parallel Region Task"),
1354           _heap(ShenandoahHeap::heap()), _blk(blk), _index(0) {}
1355 
1356   void work(uint worker_id) {
1357     ShenandoahParallelWorkerSession worker_session(worker_id);
1358     size_t stride = ShenandoahParallelRegionStride;
1359 
1360     size_t max = _heap->num_regions();
1361     while (_index < max) {
1362       size_t cur = Atomic::fetch_and_add(&_index, stride);
1363       size_t start = cur;
1364       size_t end = MIN2(cur + stride, max);
1365       if (start >= max) break;
1366 
1367       for (size_t i = cur; i < end; i++) {
1368         ShenandoahHeapRegion* current = _heap->get_region(i);
1369         _blk->heap_region_do(current);
1370       }
1371     }
1372   }
1373 };
1374 
1375 void ShenandoahHeap::parallel_heap_region_iterate(ShenandoahHeapRegionClosure* blk) const {
1376   assert(blk->is_thread_safe(), "Only thread-safe closures here");
1377   if (num_regions() > ShenandoahParallelRegionStride) {
1378     ShenandoahParallelHeapRegionTask task(blk);
1379     workers()->run_task(&task);
1380   } else {
1381     heap_region_iterate(blk);
1382   }
1383 }
1384 
1385 class ShenandoahInitMarkUpdateRegionStateClosure : public ShenandoahHeapRegionClosure {
1386 private:
1387   ShenandoahMarkingContext* const _ctx;
1388 public:
1389   ShenandoahInitMarkUpdateRegionStateClosure() : _ctx(ShenandoahHeap::heap()->marking_context()) {}
1390 
1391   void heap_region_do(ShenandoahHeapRegion* r) {
1392     assert(!r->has_live(), "Region " SIZE_FORMAT " should have no live data", r->index());
1393     if (r->is_active()) {
1394       // Check if region needs updating its TAMS. We have updated it already during concurrent
1395       // reset, so it is very likely we don't need to do another write here.
1396       if (_ctx->top_at_mark_start(r) != r->top()) {
1397         _ctx->capture_top_at_mark_start(r);
1398       }
1399     } else {
1400       assert(_ctx->top_at_mark_start(r) == r->top(),
1401              "Region " SIZE_FORMAT " should already have correct TAMS", r->index());
1402     }
1403   }
1404 
1405   bool is_thread_safe() { return true; }
1406 };
1407 
1408 void ShenandoahHeap::op_init_mark() {
1409   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Should be at safepoint");
1410   assert(Thread::current()->is_VM_thread(), "can only do this in VMThread");
1411 
1412   assert(marking_context()->is_bitmap_clear(), "need clear marking bitmap");
1413   assert(!marking_context()->is_complete(), "should not be complete");
1414   assert(!has_forwarded_objects(), "No forwarded objects on this path");
1415 
1416   if (ShenandoahVerify) {
1417     verifier()->verify_before_concmark();
1418   }
1419 
1420   if (VerifyBeforeGC) {
1421     Universe::verify();
1422   }
1423 
1424   set_concurrent_mark_in_progress(true);
1425   // We need to reset all TLABs because we'd lose marks on all objects allocated in them.
1426   {
1427     ShenandoahGCPhase phase(ShenandoahPhaseTimings::make_parsable);
1428     make_parsable(true);
1429   }
1430 
1431   {
1432     ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_update_region_states);
1433     ShenandoahInitMarkUpdateRegionStateClosure cl;
1434     parallel_heap_region_iterate(&cl);
1435   }
1436 
1437   // Make above changes visible to worker threads
1438   OrderAccess::fence();
1439 
1440   concurrent_mark()->mark_roots(ShenandoahPhaseTimings::scan_roots);
1441 
1442   if (UseTLAB) {
1443     ShenandoahGCPhase phase(ShenandoahPhaseTimings::resize_tlabs);
1444     resize_tlabs();
1445   }
1446 
1447   if (ShenandoahPacing) {
1448     pacer()->setup_for_mark();
1449   }
1450 
1451   // Arm nmethods for concurrent marking. When a nmethod is about to be executed,
1452   // we need to make sure that all its metadata are marked. alternative is to remark
1453   // thread roots at final mark pause, but it can be potential latency killer.
1454   if (ShenandoahConcurrentRoots::should_do_concurrent_class_unloading()) {
1455     ShenandoahCodeRoots::arm_nmethods();
1456   }
1457 }
1458 
1459 void ShenandoahHeap::op_mark() {
1460   concurrent_mark()->mark_from_roots();
1461 }
1462 
1463 class ShenandoahFinalMarkUpdateRegionStateClosure : public ShenandoahHeapRegionClosure {
1464 private:
1465   ShenandoahMarkingContext* const _ctx;
1466   ShenandoahHeapLock* const _lock;
1467 
1468 public:
1469   ShenandoahFinalMarkUpdateRegionStateClosure() :
1470     _ctx(ShenandoahHeap::heap()->complete_marking_context()), _lock(ShenandoahHeap::heap()->lock()) {}
1471 
1472   void heap_region_do(ShenandoahHeapRegion* r) {
1473     if (r->is_active()) {
1474       // All allocations past TAMS are implicitly live, adjust the region data.
1475       // Bitmaps/TAMS are swapped at this point, so we need to poll complete bitmap.
1476       HeapWord *tams = _ctx->top_at_mark_start(r);
1477       HeapWord *top = r->top();
1478       if (top > tams) {
1479         r->increase_live_data_alloc_words(pointer_delta(top, tams));
1480       }
1481 
1482       // We are about to select the collection set, make sure it knows about
1483       // current pinning status. Also, this allows trashing more regions that
1484       // now have their pinning status dropped.
1485       if (r->is_pinned()) {
1486         if (r->pin_count() == 0) {
1487           ShenandoahHeapLocker locker(_lock);
1488           r->make_unpinned();
1489         }
1490       } else {
1491         if (r->pin_count() > 0) {
1492           ShenandoahHeapLocker locker(_lock);
1493           r->make_pinned();
1494         }
1495       }
1496 
1497       // Remember limit for updating refs. It's guaranteed that we get no
1498       // from-space-refs written from here on.
1499       r->set_update_watermark_at_safepoint(r->top());
1500     } else {
1501       assert(!r->has_live(), "Region " SIZE_FORMAT " should have no live data", r->index());
1502       assert(_ctx->top_at_mark_start(r) == r->top(),
1503              "Region " SIZE_FORMAT " should have correct TAMS", r->index());
1504     }
1505   }
1506 
1507   bool is_thread_safe() { return true; }
1508 };
1509 
1510 void ShenandoahHeap::op_final_mark() {
1511   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Should be at safepoint");
1512   assert(!has_forwarded_objects(), "No forwarded objects on this path");
1513 
1514   // It is critical that we
1515   // evacuate roots right after finishing marking, so that we don't
1516   // get unmarked objects in the roots.
1517 
1518   if (!cancelled_gc()) {
1519     concurrent_mark()->finish_mark_from_roots(/* full_gc = */ false);
1520 
1521     // Marking is completed, deactivate SATB barrier
1522     set_concurrent_mark_in_progress(false);
1523     mark_complete_marking_context();
1524 
1525     parallel_cleaning(false /* full gc*/);
1526 
1527     if (ShenandoahVerify) {
1528       verifier()->verify_roots_no_forwarded();
1529     }
1530 
1531     {
1532       ShenandoahGCPhase phase(ShenandoahPhaseTimings::final_update_region_states);
1533       ShenandoahFinalMarkUpdateRegionStateClosure cl;
1534       parallel_heap_region_iterate(&cl);
1535 
1536       assert_pinned_region_status();
1537     }
1538 
1539     // Retire the TLABs, which will force threads to reacquire their TLABs after the pause.
1540     // This is needed for two reasons. Strong one: new allocations would be with new freeset,
1541     // which would be outside the collection set, so no cset writes would happen there.
1542     // Weaker one: new allocations would happen past update watermark, and so less work would
1543     // be needed for reference updates (would update the large filler instead).
1544     {
1545       ShenandoahGCPhase phase(ShenandoahPhaseTimings::retire_tlabs);
1546       make_parsable(true);
1547     }
1548 
1549     {
1550       ShenandoahGCPhase phase(ShenandoahPhaseTimings::choose_cset);
1551       ShenandoahHeapLocker locker(lock());
1552       _collection_set->clear();
1553       heuristics()->choose_collection_set(_collection_set);
1554     }
1555 
1556     {
1557       ShenandoahGCPhase phase(ShenandoahPhaseTimings::final_rebuild_freeset);
1558       ShenandoahHeapLocker locker(lock());
1559       _free_set->rebuild();
1560     }
1561 
1562     if (!is_degenerated_gc_in_progress()) {
1563       prepare_concurrent_roots();
1564       prepare_concurrent_unloading();
1565     }
1566 
1567     // If collection set has candidates, start evacuation.
1568     // Otherwise, bypass the rest of the cycle.
1569     if (!collection_set()->is_empty()) {
1570       ShenandoahGCPhase init_evac(ShenandoahPhaseTimings::init_evac);
1571 
1572       if (ShenandoahVerify) {
1573         verifier()->verify_before_evacuation();
1574       }
1575 
1576       set_evacuation_in_progress(true);
1577       // From here on, we need to update references.
1578       set_has_forwarded_objects(true);
1579 
1580       if (!is_degenerated_gc_in_progress()) {
1581         if (ShenandoahConcurrentRoots::should_do_concurrent_class_unloading()) {
1582           ShenandoahCodeRoots::arm_nmethods();
1583         }
1584         evacuate_and_update_roots();
1585       }
1586 
1587       if (ShenandoahPacing) {
1588         pacer()->setup_for_evac();
1589       }
1590 
1591       if (ShenandoahVerify) {
1592         ShenandoahRootVerifier::RootTypes types = ShenandoahRootVerifier::None;
1593         if (ShenandoahConcurrentRoots::should_do_concurrent_roots()) {
1594           types = ShenandoahRootVerifier::combine(ShenandoahRootVerifier::JNIHandleRoots, ShenandoahRootVerifier::WeakRoots);
1595           types = ShenandoahRootVerifier::combine(types, ShenandoahRootVerifier::CLDGRoots);
1596           types = ShenandoahRootVerifier::combine(types, ShenandoahRootVerifier::StringDedupRoots);
1597         }
1598 
1599         if (ShenandoahConcurrentRoots::should_do_concurrent_class_unloading()) {
1600           types = ShenandoahRootVerifier::combine(types, ShenandoahRootVerifier::CodeRoots);
1601         }
1602         verifier()->verify_roots_no_forwarded_except(types);
1603         verifier()->verify_during_evacuation();
1604       }
1605     } else {
1606       if (ShenandoahVerify) {
1607         verifier()->verify_after_concmark();
1608       }
1609 
1610       if (VerifyAfterGC) {
1611         Universe::verify();
1612       }
1613     }
1614 
1615   } else {
1616     // If this cycle was updating references, we need to keep the has_forwarded_objects
1617     // flag on, for subsequent phases to deal with it.
1618     concurrent_mark()->cancel();
1619     set_concurrent_mark_in_progress(false);
1620 
1621     if (process_references()) {
1622       // Abandon reference processing right away: pre-cleaning must have failed.
1623       ReferenceProcessor *rp = ref_processor();
1624       rp->disable_discovery();
1625       rp->abandon_partial_discovery();
1626       rp->verify_no_references_recorded();
1627     }
1628   }
1629 }
1630 
1631 void ShenandoahHeap::op_conc_evac() {
1632   ShenandoahEvacuationTask task(this, _collection_set, true);
1633   workers()->run_task(&task);
1634 }
1635 
1636 void ShenandoahHeap::op_stw_evac() {
1637   ShenandoahEvacuationTask task(this, _collection_set, false);
1638   workers()->run_task(&task);
1639 }
1640 
1641 void ShenandoahHeap::op_updaterefs() {
1642   update_heap_references(true);
1643 }
1644 
1645 void ShenandoahHeap::op_cleanup_early() {
1646   free_set()->recycle_trash();
1647 }
1648 
1649 void ShenandoahHeap::op_cleanup_complete() {
1650   free_set()->recycle_trash();
1651 }
1652 
1653 class ShenandoahConcurrentRootsEvacUpdateTask : public AbstractGangTask {
1654 private:
1655   ShenandoahVMRoots<true /*concurrent*/>        _vm_roots;
1656   ShenandoahClassLoaderDataRoots<true /*concurrent*/, false /*single threaded*/> _cld_roots;
1657   ShenandoahConcurrentStringDedupRoots          _dedup_roots;
1658 
1659 public:
1660   ShenandoahConcurrentRootsEvacUpdateTask(ShenandoahPhaseTimings::Phase phase) :
1661     AbstractGangTask("Shenandoah Evacuate/Update Concurrent Strong Roots Task"),
1662     _vm_roots(phase),
1663     _cld_roots(phase) {}
1664 
1665   void work(uint worker_id) {
1666     ShenandoahConcurrentWorkerSession worker_session(worker_id);
1667     ShenandoahEvacOOMScope oom;
1668     {
1669       // vm_roots and weak_roots are OopStorage backed roots, concurrent iteration
1670       // may race against OopStorage::release() calls.
1671       ShenandoahEvacUpdateOopStorageRootsClosure cl;
1672       _vm_roots.oops_do<ShenandoahEvacUpdateOopStorageRootsClosure>(&cl, worker_id);
1673     }
1674 
1675     {
1676       ShenandoahEvacuateUpdateRootsClosure<> cl;
1677       CLDToOopClosure clds(&cl, ClassLoaderData::_claim_strong);
1678       _cld_roots.cld_do(&clds, worker_id);
1679     }
1680 
1681     {
1682       ShenandoahForwardedIsAliveClosure is_alive;
1683       ShenandoahEvacuateUpdateRootsClosure<MO_RELEASE> keep_alive;
1684       _dedup_roots.oops_do(&is_alive, &keep_alive, worker_id);
1685     }
1686   }
1687 };
1688 
1689 class ShenandoahEvacUpdateCleanupOopStorageRootsClosure : public BasicOopIterateClosure {
1690 private:
1691   ShenandoahHeap* const _heap;
1692   ShenandoahMarkingContext* const _mark_context;
1693   bool  _evac_in_progress;
1694   Thread* const _thread;
1695   size_t  _dead_counter;
1696 
1697 public:
1698   ShenandoahEvacUpdateCleanupOopStorageRootsClosure();
1699   void do_oop(oop* p);
1700   void do_oop(narrowOop* p);
1701 
1702   size_t dead_counter() const;
1703   void reset_dead_counter();
1704 };
1705 
1706 ShenandoahEvacUpdateCleanupOopStorageRootsClosure::ShenandoahEvacUpdateCleanupOopStorageRootsClosure() :
1707   _heap(ShenandoahHeap::heap()),
1708   _mark_context(ShenandoahHeap::heap()->marking_context()),
1709   _evac_in_progress(ShenandoahHeap::heap()->is_evacuation_in_progress()),
1710   _thread(Thread::current()),
1711   _dead_counter(0) {
1712 }
1713 
1714 void ShenandoahEvacUpdateCleanupOopStorageRootsClosure::do_oop(oop* p) {
1715   const oop obj = RawAccess<>::oop_load(p);
1716   if (!CompressedOops::is_null(obj)) {
1717     if (!_mark_context->is_marked(obj)) {
1718       shenandoah_assert_correct(p, obj);
1719       oop old = Atomic::cmpxchg(p, obj, oop(NULL));
1720       if (obj == old) {
1721         _dead_counter ++;
1722       }
1723     } else if (_evac_in_progress && _heap->in_collection_set(obj)) {
1724       oop resolved = ShenandoahBarrierSet::resolve_forwarded_not_null(obj);
1725       if (resolved == obj) {
1726         resolved = _heap->evacuate_object(obj, _thread);
1727       }
1728       Atomic::cmpxchg(p, obj, resolved);
1729       assert(_heap->cancelled_gc() ||
1730              _mark_context->is_marked(resolved) && !_heap->in_collection_set(resolved),
1731              "Sanity");
1732     }
1733   }
1734 }
1735 
1736 void ShenandoahEvacUpdateCleanupOopStorageRootsClosure::do_oop(narrowOop* p) {
1737   ShouldNotReachHere();
1738 }
1739 
1740 size_t ShenandoahEvacUpdateCleanupOopStorageRootsClosure::dead_counter() const {
1741   return _dead_counter;
1742 }
1743 
1744 void ShenandoahEvacUpdateCleanupOopStorageRootsClosure::reset_dead_counter() {
1745   _dead_counter = 0;
1746 }
1747 
1748 class ShenandoahIsCLDAliveClosure : public CLDClosure {
1749 public:
1750   void do_cld(ClassLoaderData* cld) {
1751     cld->is_alive();
1752   }
1753 };
1754 
1755 class ShenandoahIsNMethodAliveClosure: public NMethodClosure {
1756 public:
1757   void do_nmethod(nmethod* n) {
1758     n->is_unloading();
1759   }
1760 };
1761 
1762 // This task not only evacuates/updates marked weak roots, but also "NULL"
1763 // dead weak roots.
1764 class ShenandoahConcurrentWeakRootsEvacUpdateTask : public AbstractGangTask {
1765 private:
1766   ShenandoahWeakRoot<true /*concurrent*/>  _jni_roots;
1767   ShenandoahWeakRoot<true /*concurrent*/>  _string_table_roots;
1768   ShenandoahWeakRoot<true /*concurrent*/>  _resolved_method_table_roots;
1769   ShenandoahWeakRoot<true /*concurrent*/>  _vm_roots;
1770 
1771   // Roots related to concurrent class unloading
1772   ShenandoahClassLoaderDataRoots<true /* concurrent */, false /* single thread*/>
1773                                            _cld_roots;
1774   ShenandoahConcurrentNMethodIterator      _nmethod_itr;
1775   bool                                     _concurrent_class_unloading;
1776 
1777 public:
1778   ShenandoahConcurrentWeakRootsEvacUpdateTask(ShenandoahPhaseTimings::Phase phase) :
1779     AbstractGangTask("Shenandoah Concurrent Weak Root Task"),
1780     _jni_roots(OopStorageSet::jni_weak(), phase, ShenandoahPhaseTimings::JNIWeakRoots),
1781     _string_table_roots(OopStorageSet::string_table_weak(), phase, ShenandoahPhaseTimings::StringTableRoots),
1782     _resolved_method_table_roots(OopStorageSet::resolved_method_table_weak(), phase, ShenandoahPhaseTimings::ResolvedMethodTableRoots),
1783     _vm_roots(OopStorageSet::vm_weak(), phase, ShenandoahPhaseTimings::VMWeakRoots),
1784     _cld_roots(phase),
1785     _nmethod_itr(ShenandoahCodeRoots::table()),
1786     _concurrent_class_unloading(ShenandoahConcurrentRoots::should_do_concurrent_class_unloading()) {
1787     StringTable::reset_dead_counter();
1788     ResolvedMethodTable::reset_dead_counter();
1789     if (_concurrent_class_unloading) {
1790       MutexLocker mu(CodeCache_lock, Mutex::_no_safepoint_check_flag);
1791       _nmethod_itr.nmethods_do_begin();
1792     }
1793   }
1794 
1795   ~ShenandoahConcurrentWeakRootsEvacUpdateTask() {
1796     StringTable::finish_dead_counter();
1797     ResolvedMethodTable::finish_dead_counter();
1798     if (_concurrent_class_unloading) {
1799       MutexLocker mu(CodeCache_lock, Mutex::_no_safepoint_check_flag);
1800       _nmethod_itr.nmethods_do_end();
1801     }
1802   }
1803 
1804   void work(uint worker_id) {
1805     ShenandoahConcurrentWorkerSession worker_session(worker_id);
1806     {
1807       ShenandoahEvacOOMScope oom;
1808       // jni_roots and weak_roots are OopStorage backed roots, concurrent iteration
1809       // may race against OopStorage::release() calls.
1810       ShenandoahEvacUpdateCleanupOopStorageRootsClosure cl;
1811       _jni_roots.oops_do(&cl, worker_id);
1812       _vm_roots.oops_do(&cl, worker_id);
1813 
1814       cl.reset_dead_counter();
1815       _string_table_roots.oops_do(&cl, worker_id);
1816       StringTable::inc_dead_counter(cl.dead_counter());
1817 
1818       cl.reset_dead_counter();
1819       _resolved_method_table_roots.oops_do(&cl, worker_id);
1820       ResolvedMethodTable::inc_dead_counter(cl.dead_counter());
1821     }
1822 
1823     // If we are going to perform concurrent class unloading later on, we need to
1824     // cleanup the weak oops in CLD and determinate nmethod's unloading state, so that we
1825     // can cleanup immediate garbage sooner.
1826     if (_concurrent_class_unloading) {
1827       // Applies ShenandoahIsCLDAlive closure to CLDs, native barrier will either NULL the
1828       // CLD's holder or evacuate it.
1829       ShenandoahIsCLDAliveClosure is_cld_alive;
1830       _cld_roots.cld_do(&is_cld_alive, worker_id);
1831 
1832       // Applies ShenandoahIsNMethodAliveClosure to registered nmethods.
1833       // The closure calls nmethod->is_unloading(). The is_unloading
1834       // state is cached, therefore, during concurrent class unloading phase,
1835       // we will not touch the metadata of unloading nmethods
1836       ShenandoahIsNMethodAliveClosure is_nmethod_alive;
1837       _nmethod_itr.nmethods_do(&is_nmethod_alive);
1838     }
1839   }
1840 };
1841 
1842 void ShenandoahHeap::op_weak_roots() {
1843   if (is_concurrent_weak_root_in_progress()) {
1844     // Concurrent weak root processing
1845     {
1846       ShenandoahTimingsTracker t(ShenandoahPhaseTimings::conc_weak_roots_work);
1847       ShenandoahGCWorkerPhase worker_phase(ShenandoahPhaseTimings::conc_weak_roots_work);
1848       ShenandoahConcurrentWeakRootsEvacUpdateTask task(ShenandoahPhaseTimings::conc_weak_roots_work);
1849       workers()->run_task(&task);
1850       if (!ShenandoahConcurrentRoots::should_do_concurrent_class_unloading()) {
1851         set_concurrent_weak_root_in_progress(false);
1852       }
1853     }
1854 
1855     // Perform handshake to flush out dead oops
1856     {
1857       ShenandoahTimingsTracker t(ShenandoahPhaseTimings::conc_weak_roots_rendezvous);
1858       ShenandoahRendezvousClosure cl;
1859       Handshake::execute(&cl);
1860     }
1861   }
1862 }
1863 
1864 void ShenandoahHeap::op_class_unloading() {
1865   assert (is_concurrent_weak_root_in_progress() &&
1866           ShenandoahConcurrentRoots::should_do_concurrent_class_unloading(),
1867           "Checked by caller");
1868   _unloader.unload();
1869   set_concurrent_weak_root_in_progress(false);
1870 }
1871 
1872 void ShenandoahHeap::op_strong_roots() {
1873   assert(is_concurrent_strong_root_in_progress(), "Checked by caller");
1874   ShenandoahConcurrentRootsEvacUpdateTask task(ShenandoahPhaseTimings::conc_strong_roots);
1875   workers()->run_task(&task);
1876   set_concurrent_strong_root_in_progress(false);
1877 }
1878 
1879 class ShenandoahResetUpdateRegionStateClosure : public ShenandoahHeapRegionClosure {
1880 private:
1881   ShenandoahMarkingContext* const _ctx;
1882 public:
1883   ShenandoahResetUpdateRegionStateClosure() : _ctx(ShenandoahHeap::heap()->marking_context()) {}
1884 
1885   void heap_region_do(ShenandoahHeapRegion* r) {
1886     if (r->is_active()) {
1887       // Reset live data and set TAMS optimistically. We would recheck these under the pause
1888       // anyway to capture any updates that happened since now.
1889       r->clear_live_data();
1890       _ctx->capture_top_at_mark_start(r);
1891     }
1892   }
1893 
1894   bool is_thread_safe() { return true; }
1895 };
1896 
1897 void ShenandoahHeap::op_reset() {
1898   if (ShenandoahPacing) {
1899     pacer()->setup_for_reset();
1900   }
1901   reset_mark_bitmap();
1902 
1903   ShenandoahResetUpdateRegionStateClosure cl;
1904   parallel_heap_region_iterate(&cl);
1905 }
1906 
1907 void ShenandoahHeap::op_preclean() {
1908   if (ShenandoahPacing) {
1909     pacer()->setup_for_preclean();
1910   }
1911   concurrent_mark()->preclean_weak_refs();
1912 }
1913 
1914 void ShenandoahHeap::op_full(GCCause::Cause cause) {
1915   ShenandoahMetricsSnapshot metrics;
1916   metrics.snap_before();
1917 
1918   full_gc()->do_it(cause);
1919   if (UseTLAB) {
1920     ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_resize_tlabs);
1921     resize_all_tlabs();
1922   }
1923 
1924   metrics.snap_after();
1925 
1926   if (metrics.is_good_progress()) {
1927     _progress_last_gc.set();
1928   } else {
1929     // Nothing to do. Tell the allocation path that we have failed to make
1930     // progress, and it can finally fail.
1931     _progress_last_gc.unset();
1932   }
1933 }
1934 
1935 void ShenandoahHeap::op_degenerated(ShenandoahDegenPoint point) {
1936   // Degenerated GC is STW, but it can also fail. Current mechanics communicates
1937   // GC failure via cancelled_concgc() flag. So, if we detect the failure after
1938   // some phase, we have to upgrade the Degenerate GC to Full GC.
1939 
1940   clear_cancelled_gc();
1941 
1942   ShenandoahMetricsSnapshot metrics;
1943   metrics.snap_before();
1944 
1945   switch (point) {
1946     // The cases below form the Duff's-like device: it describes the actual GC cycle,
1947     // but enters it at different points, depending on which concurrent phase had
1948     // degenerated.
1949 
1950     case _degenerated_outside_cycle:
1951       // We have degenerated from outside the cycle, which means something is bad with
1952       // the heap, most probably heavy humongous fragmentation, or we are very low on free
1953       // space. It makes little sense to wait for Full GC to reclaim as much as it can, when
1954       // we can do the most aggressive degen cycle, which includes processing references and
1955       // class unloading, unless those features are explicitly disabled.
1956       //
1957       // Note that we can only do this for "outside-cycle" degens, otherwise we would risk
1958       // changing the cycle parameters mid-cycle during concurrent -> degenerated handover.
1959       set_process_references(heuristics()->can_process_references());
1960       set_unload_classes(heuristics()->can_unload_classes());
1961 
1962       op_reset();
1963 
1964       op_init_mark();
1965       if (cancelled_gc()) {
1966         op_degenerated_fail();
1967         return;
1968       }
1969 
1970     case _degenerated_mark:
1971       op_final_mark();
1972       if (cancelled_gc()) {
1973         op_degenerated_fail();
1974         return;
1975       }
1976 
1977       if (!has_forwarded_objects() && ShenandoahConcurrentRoots::can_do_concurrent_class_unloading()) {
1978         // Disarm nmethods that armed for concurrent mark. On normal cycle, it would
1979         // be disarmed while conc-roots phase is running.
1980         // TODO: Call op_conc_roots() here instead
1981         ShenandoahCodeRoots::disarm_nmethods();
1982       }
1983 
1984       op_cleanup_early();
1985 
1986     case _degenerated_evac:
1987       // If heuristics thinks we should do the cycle, this flag would be set,
1988       // and we can do evacuation. Otherwise, it would be the shortcut cycle.
1989       if (is_evacuation_in_progress()) {
1990 
1991         // Degeneration under oom-evac protocol might have left some objects in
1992         // collection set un-evacuated. Restart evacuation from the beginning to
1993         // capture all objects. For all the objects that are already evacuated,
1994         // it would be a simple check, which is supposed to be fast. This is also
1995         // safe to do even without degeneration, as CSet iterator is at beginning
1996         // in preparation for evacuation anyway.
1997         //
1998         // Before doing that, we need to make sure we never had any cset-pinned
1999         // regions. This may happen if allocation failure happened when evacuating
2000         // the about-to-be-pinned object, oom-evac protocol left the object in
2001         // the collection set, and then the pin reached the cset region. If we continue
2002         // the cycle here, we would trash the cset and alive objects in it. To avoid
2003         // it, we fail degeneration right away and slide into Full GC to recover.
2004 
2005         {
2006           sync_pinned_region_status();
2007           collection_set()->clear_current_index();
2008 
2009           ShenandoahHeapRegion* r;
2010           while ((r = collection_set()->next()) != NULL) {
2011             if (r->is_pinned()) {
2012               cancel_gc(GCCause::_shenandoah_upgrade_to_full_gc);
2013               op_degenerated_fail();
2014               return;
2015             }
2016           }
2017 
2018           collection_set()->clear_current_index();
2019         }
2020 
2021         op_stw_evac();
2022         if (cancelled_gc()) {
2023           op_degenerated_fail();
2024           return;
2025         }
2026       }
2027 
2028       // If heuristics thinks we should do the cycle, this flag would be set,
2029       // and we need to do update-refs. Otherwise, it would be the shortcut cycle.
2030       if (has_forwarded_objects()) {
2031         op_init_updaterefs();
2032         if (cancelled_gc()) {
2033           op_degenerated_fail();
2034           return;
2035         }
2036       }
2037 
2038     case _degenerated_updaterefs:
2039       if (has_forwarded_objects()) {
2040         op_final_updaterefs();
2041         if (cancelled_gc()) {
2042           op_degenerated_fail();
2043           return;
2044         }
2045       }
2046 
2047       op_cleanup_complete();
2048       break;
2049 
2050     default:
2051       ShouldNotReachHere();
2052   }
2053 
2054   if (ShenandoahVerify) {
2055     verifier()->verify_after_degenerated();
2056   }
2057 
2058   if (VerifyAfterGC) {
2059     Universe::verify();
2060   }
2061 
2062   metrics.snap_after();
2063 
2064   // Check for futility and fail. There is no reason to do several back-to-back Degenerated cycles,
2065   // because that probably means the heap is overloaded and/or fragmented.
2066   if (!metrics.is_good_progress()) {
2067     _progress_last_gc.unset();
2068     cancel_gc(GCCause::_shenandoah_upgrade_to_full_gc);
2069     op_degenerated_futile();
2070   } else {
2071     _progress_last_gc.set();
2072   }
2073 }
2074 
2075 void ShenandoahHeap::op_degenerated_fail() {
2076   log_info(gc)("Cannot finish degeneration, upgrading to Full GC");
2077   shenandoah_policy()->record_degenerated_upgrade_to_full();
2078   op_full(GCCause::_shenandoah_upgrade_to_full_gc);
2079 }
2080 
2081 void ShenandoahHeap::op_degenerated_futile() {
2082   shenandoah_policy()->record_degenerated_upgrade_to_full();
2083   op_full(GCCause::_shenandoah_upgrade_to_full_gc);
2084 }
2085 
2086 void ShenandoahHeap::force_satb_flush_all_threads() {
2087   if (!is_concurrent_mark_in_progress()) {
2088     // No need to flush SATBs
2089     return;
2090   }
2091 
2092   for (JavaThreadIteratorWithHandle jtiwh; JavaThread *t = jtiwh.next(); ) {
2093     ShenandoahThreadLocalData::set_force_satb_flush(t, true);
2094   }
2095   // The threads are not "acquiring" their thread-local data, but it does not
2096   // hurt to "release" the updates here anyway.
2097   OrderAccess::fence();
2098 }
2099 
2100 void ShenandoahHeap::set_gc_state_all_threads(char state) {
2101   for (JavaThreadIteratorWithHandle jtiwh; JavaThread *t = jtiwh.next(); ) {
2102     ShenandoahThreadLocalData::set_gc_state(t, state);
2103   }
2104 }
2105 
2106 void ShenandoahHeap::set_gc_state_mask(uint mask, bool value) {
2107   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Should really be Shenandoah safepoint");
2108   _gc_state.set_cond(mask, value);
2109   set_gc_state_all_threads(_gc_state.raw_value());
2110 }
2111 
2112 void ShenandoahHeap::set_concurrent_mark_in_progress(bool in_progress) {
2113   if (has_forwarded_objects()) {
2114     set_gc_state_mask(MARKING | UPDATEREFS, in_progress);
2115   } else {
2116     set_gc_state_mask(MARKING, in_progress);
2117   }
2118   ShenandoahBarrierSet::satb_mark_queue_set().set_active_all_threads(in_progress, !in_progress);
2119 }
2120 
2121 void ShenandoahHeap::set_evacuation_in_progress(bool in_progress) {
2122   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Only call this at safepoint");
2123   set_gc_state_mask(EVACUATION, in_progress);
2124 }
2125 
2126 void ShenandoahHeap::set_concurrent_strong_root_in_progress(bool in_progress) {
2127   assert(ShenandoahConcurrentRoots::can_do_concurrent_roots(), "Why set the flag?");
2128   if (in_progress) {
2129     _concurrent_strong_root_in_progress.set();
2130   } else {
2131     _concurrent_strong_root_in_progress.unset();
2132   }
2133 }
2134 
2135 void ShenandoahHeap::set_concurrent_weak_root_in_progress(bool in_progress) {
2136   assert(ShenandoahConcurrentRoots::can_do_concurrent_roots(), "Why set the flag?");
2137   if (in_progress) {
2138     _concurrent_weak_root_in_progress.set();
2139   } else {
2140     _concurrent_weak_root_in_progress.unset();
2141   }
2142 }
2143 
2144 void ShenandoahHeap::ref_processing_init() {
2145   assert(_max_workers > 0, "Sanity");
2146 
2147   _ref_processor =
2148     new ReferenceProcessor(&_subject_to_discovery,  // is_subject_to_discovery
2149                            _ref_proc_mt_processing, // MT processing
2150                            _max_workers,            // Degree of MT processing
2151                            _ref_proc_mt_discovery,  // MT discovery
2152                            _max_workers,            // Degree of MT discovery
2153                            false,                   // Reference discovery is not atomic
2154                            NULL,                    // No closure, should be installed before use
2155                            true);                   // Scale worker threads
2156 
2157   shenandoah_assert_rp_isalive_not_installed();
2158 }
2159 
2160 GCTracer* ShenandoahHeap::tracer() {
2161   return shenandoah_policy()->tracer();
2162 }
2163 
2164 size_t ShenandoahHeap::tlab_used(Thread* thread) const {
2165   return _free_set->used();
2166 }
2167 
2168 bool ShenandoahHeap::try_cancel_gc() {
2169   while (true) {
2170     jbyte prev = _cancelled_gc.cmpxchg(CANCELLED, CANCELLABLE);
2171     if (prev == CANCELLABLE) return true;
2172     else if (prev == CANCELLED) return false;
2173     assert(ShenandoahSuspendibleWorkers, "should not get here when not using suspendible workers");
2174     assert(prev == NOT_CANCELLED, "must be NOT_CANCELLED");
2175     if (Thread::current()->is_Java_thread()) {
2176       // We need to provide a safepoint here, otherwise we might
2177       // spin forever if a SP is pending.
2178       ThreadBlockInVM sp(JavaThread::current());
2179       SpinPause();
2180     }
2181   }
2182 }
2183 
2184 void ShenandoahHeap::cancel_gc(GCCause::Cause cause) {
2185   if (try_cancel_gc()) {
2186     FormatBuffer<> msg("Cancelling GC: %s", GCCause::to_string(cause));
2187     log_info(gc)("%s", msg.buffer());
2188     Events::log(Thread::current(), "%s", msg.buffer());
2189   }
2190 }
2191 
2192 uint ShenandoahHeap::max_workers() {
2193   return _max_workers;
2194 }
2195 
2196 void ShenandoahHeap::stop() {
2197   // The shutdown sequence should be able to terminate when GC is running.
2198 
2199   // Step 0. Notify policy to disable event recording.
2200   _shenandoah_policy->record_shutdown();
2201 
2202   // Step 1. Notify control thread that we are in shutdown.
2203   // Note that we cannot do that with stop(), because stop() is blocking and waits for the actual shutdown.
2204   // Doing stop() here would wait for the normal GC cycle to complete, never falling through to cancel below.
2205   control_thread()->prepare_for_graceful_shutdown();
2206 
2207   // Step 2. Notify GC workers that we are cancelling GC.
2208   cancel_gc(GCCause::_shenandoah_stop_vm);
2209 
2210   // Step 3. Wait until GC worker exits normally.
2211   control_thread()->stop();
2212 
2213   // Step 4. Stop String Dedup thread if it is active
2214   if (ShenandoahStringDedup::is_enabled()) {
2215     ShenandoahStringDedup::stop();
2216   }
2217 }
2218 
2219 void ShenandoahHeap::stw_unload_classes(bool full_gc) {
2220   if (!unload_classes()) return;
2221 
2222   // Unload classes and purge SystemDictionary.
2223   {
2224     ShenandoahGCPhase phase(full_gc ?
2225                             ShenandoahPhaseTimings::full_gc_purge_class_unload :
2226                             ShenandoahPhaseTimings::purge_class_unload);
2227     bool purged_class = SystemDictionary::do_unloading(gc_timer());
2228 
2229     ShenandoahIsAliveSelector is_alive;
2230     uint num_workers = _workers->active_workers();
2231     ShenandoahClassUnloadingTask unlink_task(is_alive.is_alive_closure(), num_workers, purged_class);
2232     _workers->run_task(&unlink_task);
2233   }
2234 
2235   {
2236     ShenandoahGCPhase phase(full_gc ?
2237                             ShenandoahPhaseTimings::full_gc_purge_cldg :
2238                             ShenandoahPhaseTimings::purge_cldg);
2239     ClassLoaderDataGraph::purge();
2240   }
2241   // Resize and verify metaspace
2242   MetaspaceGC::compute_new_size();
2243   MetaspaceUtils::verify_metrics();
2244 }
2245 
2246 // Weak roots are either pre-evacuated (final mark) or updated (final updaterefs),
2247 // so they should not have forwarded oops.
2248 // However, we do need to "null" dead oops in the roots, if can not be done
2249 // in concurrent cycles.
2250 void ShenandoahHeap::stw_process_weak_roots(bool full_gc) {
2251   ShenandoahGCPhase root_phase(full_gc ?
2252                                ShenandoahPhaseTimings::full_gc_purge :
2253                                ShenandoahPhaseTimings::purge);
2254   uint num_workers = _workers->active_workers();
2255   ShenandoahPhaseTimings::Phase timing_phase = full_gc ?
2256                                                ShenandoahPhaseTimings::full_gc_purge_weak_par :
2257                                                ShenandoahPhaseTimings::purge_weak_par;
2258   ShenandoahGCPhase phase(timing_phase);
2259   ShenandoahGCWorkerPhase worker_phase(timing_phase);
2260 
2261   // Cleanup weak roots
2262   if (has_forwarded_objects()) {
2263     ShenandoahForwardedIsAliveClosure is_alive;
2264     ShenandoahUpdateRefsClosure keep_alive;
2265     ShenandoahParallelWeakRootsCleaningTask<ShenandoahForwardedIsAliveClosure, ShenandoahUpdateRefsClosure>
2266       cleaning_task(timing_phase, &is_alive, &keep_alive, num_workers, !ShenandoahConcurrentRoots::should_do_concurrent_class_unloading());
2267     _workers->run_task(&cleaning_task);
2268   } else {
2269     ShenandoahIsAliveClosure is_alive;
2270 #ifdef ASSERT
2271     ShenandoahAssertNotForwardedClosure verify_cl;
2272     ShenandoahParallelWeakRootsCleaningTask<ShenandoahIsAliveClosure, ShenandoahAssertNotForwardedClosure>
2273       cleaning_task(timing_phase, &is_alive, &verify_cl, num_workers, !ShenandoahConcurrentRoots::should_do_concurrent_class_unloading());
2274 #else
2275     ShenandoahParallelWeakRootsCleaningTask<ShenandoahIsAliveClosure, DoNothingClosure>
2276       cleaning_task(timing_phase, &is_alive, &do_nothing_cl, num_workers, !ShenandoahConcurrentRoots::should_do_concurrent_class_unloading());
2277 #endif
2278     _workers->run_task(&cleaning_task);
2279   }
2280 }
2281 
2282 void ShenandoahHeap::parallel_cleaning(bool full_gc) {
2283   assert(SafepointSynchronize::is_at_safepoint(), "Must be at a safepoint");
2284   stw_process_weak_roots(full_gc);
2285   if (!ShenandoahConcurrentRoots::should_do_concurrent_class_unloading()) {
2286     stw_unload_classes(full_gc);
2287   }
2288 }
2289 
2290 void ShenandoahHeap::set_has_forwarded_objects(bool cond) {
2291   set_gc_state_mask(HAS_FORWARDED, cond);
2292 }
2293 
2294 void ShenandoahHeap::set_process_references(bool pr) {
2295   _process_references.set_cond(pr);
2296 }
2297 
2298 void ShenandoahHeap::set_unload_classes(bool uc) {
2299   _unload_classes.set_cond(uc);
2300 }
2301 
2302 bool ShenandoahHeap::process_references() const {
2303   return _process_references.is_set();
2304 }
2305 
2306 bool ShenandoahHeap::unload_classes() const {
2307   return _unload_classes.is_set();
2308 }
2309 
2310 address ShenandoahHeap::in_cset_fast_test_addr() {
2311   ShenandoahHeap* heap = ShenandoahHeap::heap();
2312   assert(heap->collection_set() != NULL, "Sanity");
2313   return (address) heap->collection_set()->biased_map_address();
2314 }
2315 
2316 address ShenandoahHeap::cancelled_gc_addr() {
2317   return (address) ShenandoahHeap::heap()->_cancelled_gc.addr_of();
2318 }
2319 
2320 address ShenandoahHeap::gc_state_addr() {
2321   return (address) ShenandoahHeap::heap()->_gc_state.addr_of();
2322 }
2323 
2324 size_t ShenandoahHeap::bytes_allocated_since_gc_start() {
2325   return Atomic::load_acquire(&_bytes_allocated_since_gc_start);
2326 }
2327 
2328 void ShenandoahHeap::reset_bytes_allocated_since_gc_start() {
2329   Atomic::release_store_fence(&_bytes_allocated_since_gc_start, (size_t)0);
2330 }
2331 
2332 void ShenandoahHeap::set_degenerated_gc_in_progress(bool in_progress) {
2333   _degenerated_gc_in_progress.set_cond(in_progress);
2334 }
2335 
2336 void ShenandoahHeap::set_full_gc_in_progress(bool in_progress) {
2337   _full_gc_in_progress.set_cond(in_progress);
2338 }
2339 
2340 void ShenandoahHeap::set_full_gc_move_in_progress(bool in_progress) {
2341   assert (is_full_gc_in_progress(), "should be");
2342   _full_gc_move_in_progress.set_cond(in_progress);
2343 }
2344 
2345 void ShenandoahHeap::set_update_refs_in_progress(bool in_progress) {
2346   set_gc_state_mask(UPDATEREFS, in_progress);
2347 }
2348 
2349 void ShenandoahHeap::register_nmethod(nmethod* nm) {
2350   ShenandoahCodeRoots::register_nmethod(nm);
2351 }
2352 
2353 void ShenandoahHeap::unregister_nmethod(nmethod* nm) {
2354   ShenandoahCodeRoots::unregister_nmethod(nm);
2355 }
2356 
2357 void ShenandoahHeap::flush_nmethod(nmethod* nm) {
2358   ShenandoahCodeRoots::flush_nmethod(nm);
2359 }
2360 
2361 oop ShenandoahHeap::pin_object(JavaThread* thr, oop o) {
2362   heap_region_containing(o)->record_pin();
2363   return o;
2364 }
2365 
2366 void ShenandoahHeap::unpin_object(JavaThread* thr, oop o) {
2367   heap_region_containing(o)->record_unpin();
2368 }
2369 
2370 void ShenandoahHeap::sync_pinned_region_status() {
2371   ShenandoahHeapLocker locker(lock());
2372 
2373   for (size_t i = 0; i < num_regions(); i++) {
2374     ShenandoahHeapRegion *r = get_region(i);
2375     if (r->is_active()) {
2376       if (r->is_pinned()) {
2377         if (r->pin_count() == 0) {
2378           r->make_unpinned();
2379         }
2380       } else {
2381         if (r->pin_count() > 0) {
2382           r->make_pinned();
2383         }
2384       }
2385     }
2386   }
2387 
2388   assert_pinned_region_status();
2389 }
2390 
2391 #ifdef ASSERT
2392 void ShenandoahHeap::assert_pinned_region_status() {
2393   for (size_t i = 0; i < num_regions(); i++) {
2394     ShenandoahHeapRegion* r = get_region(i);
2395     assert((r->is_pinned() && r->pin_count() > 0) || (!r->is_pinned() && r->pin_count() == 0),
2396            "Region " SIZE_FORMAT " pinning status is inconsistent", i);
2397   }
2398 }
2399 #endif
2400 
2401 ConcurrentGCTimer* ShenandoahHeap::gc_timer() const {
2402   return _gc_timer;
2403 }
2404 
2405 void ShenandoahHeap::prepare_concurrent_roots() {
2406   assert(SafepointSynchronize::is_at_safepoint(), "Must be at a safepoint");
2407   if (ShenandoahConcurrentRoots::should_do_concurrent_roots()) {
2408     set_concurrent_strong_root_in_progress(!collection_set()->is_empty());
2409     set_concurrent_weak_root_in_progress(true);
2410   }
2411 }
2412 
2413 void ShenandoahHeap::prepare_concurrent_unloading() {
2414   assert(SafepointSynchronize::is_at_safepoint(), "Must be at a safepoint");
2415   if (ShenandoahConcurrentRoots::should_do_concurrent_class_unloading()) {
2416     _unloader.prepare();
2417   }
2418 }
2419 
2420 void ShenandoahHeap::finish_concurrent_unloading() {
2421   assert(SafepointSynchronize::is_at_safepoint(), "Must be at a safepoint");
2422   if (ShenandoahConcurrentRoots::should_do_concurrent_class_unloading()) {
2423     _unloader.finish();
2424   }
2425 }
2426 
2427 #ifdef ASSERT
2428 void ShenandoahHeap::assert_gc_workers(uint nworkers) {
2429   assert(nworkers > 0 && nworkers <= max_workers(), "Sanity");
2430 
2431   if (ShenandoahSafepoint::is_at_shenandoah_safepoint()) {
2432     if (UseDynamicNumberOfGCThreads) {
2433       assert(nworkers <= ParallelGCThreads, "Cannot use more than it has");
2434     } else {
2435       // Use ParallelGCThreads inside safepoints
2436       assert(nworkers == ParallelGCThreads, "Use ParallelGCThreads within safepoints");
2437     }
2438   } else {
2439     if (UseDynamicNumberOfGCThreads) {
2440       assert(nworkers <= ConcGCThreads, "Cannot use more than it has");
2441     } else {
2442       // Use ConcGCThreads outside safepoints
2443       assert(nworkers == ConcGCThreads, "Use ConcGCThreads outside safepoints");
2444     }
2445   }
2446 }
2447 #endif
2448 
2449 ShenandoahVerifier* ShenandoahHeap::verifier() {
2450   guarantee(ShenandoahVerify, "Should be enabled");
2451   assert (_verifier != NULL, "sanity");
2452   return _verifier;
2453 }
2454 
2455 template<class T>
2456 class ShenandoahUpdateHeapRefsTask : public AbstractGangTask {
2457 private:
2458   T cl;
2459   ShenandoahHeap* _heap;
2460   ShenandoahRegionIterator* _regions;
2461   bool _concurrent;
2462 public:
2463   ShenandoahUpdateHeapRefsTask(ShenandoahRegionIterator* regions, bool concurrent) :
2464     AbstractGangTask("Concurrent Update References Task"),
2465     cl(T()),
2466     _heap(ShenandoahHeap::heap()),
2467     _regions(regions),
2468     _concurrent(concurrent) {
2469   }
2470 
2471   void work(uint worker_id) {
2472     if (_concurrent) {
2473       ShenandoahConcurrentWorkerSession worker_session(worker_id);
2474       ShenandoahSuspendibleThreadSetJoiner stsj(ShenandoahSuspendibleWorkers);
2475       do_work();
2476     } else {
2477       ShenandoahParallelWorkerSession worker_session(worker_id);
2478       do_work();
2479     }
2480   }
2481 
2482 private:
2483   void do_work() {
2484     ShenandoahHeapRegion* r = _regions->next();
2485     ShenandoahMarkingContext* const ctx = _heap->complete_marking_context();
2486     while (r != NULL) {
2487       HeapWord* update_watermark = r->get_update_watermark();
2488       assert (update_watermark >= r->bottom(), "sanity");
2489       if (r->is_active() && !r->is_cset()) {
2490         _heap->marked_object_oop_iterate(r, &cl, update_watermark);
2491       }
2492       if (ShenandoahPacing) {
2493         _heap->pacer()->report_updaterefs(pointer_delta(update_watermark, r->bottom()));
2494       }
2495       if (_heap->check_cancelled_gc_and_yield(_concurrent)) {
2496         return;
2497       }
2498       r = _regions->next();
2499     }
2500   }
2501 };
2502 
2503 void ShenandoahHeap::update_heap_references(bool concurrent) {
2504   ShenandoahUpdateHeapRefsTask<ShenandoahUpdateHeapRefsClosure> task(&_update_refs_iterator, concurrent);
2505   workers()->run_task(&task);
2506 }
2507 
2508 void ShenandoahHeap::op_init_updaterefs() {
2509   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "must be at safepoint");
2510 
2511   set_evacuation_in_progress(false);
2512 
2513   {
2514     ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_update_refs_retire_gclabs);
2515     retire_and_reset_gclabs();
2516   }
2517 
2518   if (ShenandoahVerify) {
2519     if (!is_degenerated_gc_in_progress()) {
2520       verifier()->verify_roots_in_to_space_except(ShenandoahRootVerifier::ThreadRoots);
2521     }
2522     verifier()->verify_before_updaterefs();
2523   }
2524 
2525   set_update_refs_in_progress(true);
2526 
2527   _update_refs_iterator.reset();
2528 
2529   if (ShenandoahPacing) {
2530     pacer()->setup_for_updaterefs();
2531   }
2532 }
2533 
2534 class ShenandoahFinalUpdateRefsUpdateRegionStateClosure : public ShenandoahHeapRegionClosure {
2535 private:
2536   ShenandoahHeapLock* const _lock;
2537 
2538 public:
2539   ShenandoahFinalUpdateRefsUpdateRegionStateClosure() : _lock(ShenandoahHeap::heap()->lock()) {}
2540 
2541   void heap_region_do(ShenandoahHeapRegion* r) {
2542     // Drop unnecessary "pinned" state from regions that does not have CP marks
2543     // anymore, as this would allow trashing them.
2544 
2545     if (r->is_active()) {
2546       if (r->is_pinned()) {
2547         if (r->pin_count() == 0) {
2548           ShenandoahHeapLocker locker(_lock);
2549           r->make_unpinned();
2550         }
2551       } else {
2552         if (r->pin_count() > 0) {
2553           ShenandoahHeapLocker locker(_lock);
2554           r->make_pinned();
2555         }
2556       }
2557     }
2558   }
2559 
2560   bool is_thread_safe() { return true; }
2561 };
2562 
2563 void ShenandoahHeap::op_final_updaterefs() {
2564   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "must be at safepoint");
2565 
2566   finish_concurrent_unloading();
2567 
2568   // Check if there is left-over work, and finish it
2569   if (_update_refs_iterator.has_next()) {
2570     ShenandoahGCPhase phase(ShenandoahPhaseTimings::final_update_refs_finish_work);
2571 
2572     // Finish updating references where we left off.
2573     clear_cancelled_gc();
2574     update_heap_references(false);
2575   }
2576 
2577   // Clear cancelled GC, if set. On cancellation path, the block before would handle
2578   // everything. On degenerated paths, cancelled gc would not be set anyway.
2579   if (cancelled_gc()) {
2580     clear_cancelled_gc();
2581   }
2582   assert(!cancelled_gc(), "Should have been done right before");
2583 
2584   if (ShenandoahVerify && !is_degenerated_gc_in_progress()) {
2585     verifier()->verify_roots_in_to_space_except(ShenandoahRootVerifier::ThreadRoots);
2586   }
2587 
2588   if (is_degenerated_gc_in_progress()) {
2589     concurrent_mark()->update_roots(ShenandoahPhaseTimings::degen_gc_update_roots);
2590   } else {
2591     concurrent_mark()->update_thread_roots(ShenandoahPhaseTimings::final_update_refs_roots);
2592   }
2593 
2594   // Has to be done before cset is clear
2595   if (ShenandoahVerify) {
2596     verifier()->verify_roots_in_to_space();
2597   }
2598 
2599   {
2600     ShenandoahGCPhase phase(ShenandoahPhaseTimings::final_update_refs_update_region_states);
2601     ShenandoahFinalUpdateRefsUpdateRegionStateClosure cl;
2602     parallel_heap_region_iterate(&cl);
2603 
2604     assert_pinned_region_status();
2605   }
2606 
2607   {
2608     ShenandoahGCPhase phase(ShenandoahPhaseTimings::final_update_refs_trash_cset);
2609     trash_cset_regions();
2610   }
2611 
2612   set_has_forwarded_objects(false);
2613   set_update_refs_in_progress(false);
2614 
2615   if (ShenandoahVerify) {
2616     verifier()->verify_after_updaterefs();
2617   }
2618 
2619   if (VerifyAfterGC) {
2620     Universe::verify();
2621   }
2622 
2623   {
2624     ShenandoahGCPhase phase(ShenandoahPhaseTimings::final_update_refs_rebuild_freeset);
2625     ShenandoahHeapLocker locker(lock());
2626     _free_set->rebuild();
2627   }
2628 }
2629 
2630 void ShenandoahHeap::print_extended_on(outputStream *st) const {
2631   print_on(st);
2632   print_heap_regions_on(st);
2633 }
2634 
2635 bool ShenandoahHeap::is_bitmap_slice_committed(ShenandoahHeapRegion* r, bool skip_self) {
2636   size_t slice = r->index() / _bitmap_regions_per_slice;
2637 
2638   size_t regions_from = _bitmap_regions_per_slice * slice;
2639   size_t regions_to   = MIN2(num_regions(), _bitmap_regions_per_slice * (slice + 1));
2640   for (size_t g = regions_from; g < regions_to; g++) {
2641     assert (g / _bitmap_regions_per_slice == slice, "same slice");
2642     if (skip_self && g == r->index()) continue;
2643     if (get_region(g)->is_committed()) {
2644       return true;
2645     }
2646   }
2647   return false;
2648 }
2649 
2650 bool ShenandoahHeap::commit_bitmap_slice(ShenandoahHeapRegion* r) {
2651   shenandoah_assert_heaplocked();
2652 
2653   // Bitmaps in special regions do not need commits
2654   if (_bitmap_region_special) {
2655     return true;
2656   }
2657 
2658   if (is_bitmap_slice_committed(r, true)) {
2659     // Some other region from the group is already committed, meaning the bitmap
2660     // slice is already committed, we exit right away.
2661     return true;
2662   }
2663 
2664   // Commit the bitmap slice:
2665   size_t slice = r->index() / _bitmap_regions_per_slice;
2666   size_t off = _bitmap_bytes_per_slice * slice;
2667   size_t len = _bitmap_bytes_per_slice;
2668   char* start = (char*) _bitmap_region.start() + off;
2669 
2670   if (!os::commit_memory(start, len, false)) {
2671     return false;
2672   }
2673 
2674   if (AlwaysPreTouch) {
2675     os::pretouch_memory(start, start + len, _pretouch_bitmap_page_size);
2676   }
2677 
2678   return true;
2679 }
2680 
2681 bool ShenandoahHeap::uncommit_bitmap_slice(ShenandoahHeapRegion *r) {
2682   shenandoah_assert_heaplocked();
2683 
2684   // Bitmaps in special regions do not need uncommits
2685   if (_bitmap_region_special) {
2686     return true;
2687   }
2688 
2689   if (is_bitmap_slice_committed(r, true)) {
2690     // Some other region from the group is still committed, meaning the bitmap
2691     // slice is should stay committed, exit right away.
2692     return true;
2693   }
2694 
2695   // Uncommit the bitmap slice:
2696   size_t slice = r->index() / _bitmap_regions_per_slice;
2697   size_t off = _bitmap_bytes_per_slice * slice;
2698   size_t len = _bitmap_bytes_per_slice;
2699   if (!os::uncommit_memory((char*)_bitmap_region.start() + off, len)) {
2700     return false;
2701   }
2702   return true;
2703 }
2704 
2705 void ShenandoahHeap::safepoint_synchronize_begin() {
2706   if (ShenandoahSuspendibleWorkers || UseStringDeduplication) {
2707     SuspendibleThreadSet::synchronize();
2708   }
2709 }
2710 
2711 void ShenandoahHeap::safepoint_synchronize_end() {
2712   if (ShenandoahSuspendibleWorkers || UseStringDeduplication) {
2713     SuspendibleThreadSet::desynchronize();
2714   }
2715 }
2716 
2717 void ShenandoahHeap::vmop_entry_init_mark() {
2718   TraceCollectorStats tcs(monitoring_support()->stw_collection_counters());
2719   ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::init_mark_gross);
2720 
2721   try_inject_alloc_failure();
2722   VM_ShenandoahInitMark op;
2723   VMThread::execute(&op); // jump to entry_init_mark() under safepoint
2724 }
2725 
2726 void ShenandoahHeap::vmop_entry_final_mark() {
2727   TraceCollectorStats tcs(monitoring_support()->stw_collection_counters());
2728   ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_mark_gross);
2729 
2730   try_inject_alloc_failure();
2731   VM_ShenandoahFinalMarkStartEvac op;
2732   VMThread::execute(&op); // jump to entry_final_mark under safepoint
2733 }
2734 
2735 void ShenandoahHeap::vmop_entry_init_updaterefs() {
2736   TraceCollectorStats tcs(monitoring_support()->stw_collection_counters());
2737   ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::init_update_refs_gross);
2738 
2739   try_inject_alloc_failure();
2740   VM_ShenandoahInitUpdateRefs op;
2741   VMThread::execute(&op);
2742 }
2743 
2744 void ShenandoahHeap::vmop_entry_final_updaterefs() {
2745   TraceCollectorStats tcs(monitoring_support()->stw_collection_counters());
2746   ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_update_refs_gross);
2747 
2748   try_inject_alloc_failure();
2749   VM_ShenandoahFinalUpdateRefs op;
2750   VMThread::execute(&op);
2751 }
2752 
2753 void ShenandoahHeap::vmop_entry_full(GCCause::Cause cause) {
2754   TraceCollectorStats tcs(monitoring_support()->full_stw_collection_counters());
2755   ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::full_gc_gross);
2756 
2757   try_inject_alloc_failure();
2758   VM_ShenandoahFullGC op(cause);
2759   VMThread::execute(&op);
2760 }
2761 
2762 void ShenandoahHeap::vmop_degenerated(ShenandoahDegenPoint point) {
2763   TraceCollectorStats tcs(monitoring_support()->full_stw_collection_counters());
2764   ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::degen_gc_gross);
2765 
2766   VM_ShenandoahDegeneratedGC degenerated_gc((int)point);
2767   VMThread::execute(&degenerated_gc);
2768 }
2769 
2770 void ShenandoahHeap::entry_init_mark() {
2771   const char* msg = init_mark_event_message();
2772   ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::init_mark);
2773   EventMark em("%s", msg);
2774 
2775   ShenandoahWorkerScope scope(workers(),
2776                               ShenandoahWorkerPolicy::calc_workers_for_init_marking(),
2777                               "init marking");
2778 
2779   op_init_mark();
2780 }
2781 
2782 void ShenandoahHeap::entry_final_mark() {
2783   const char* msg = final_mark_event_message();
2784   ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::final_mark);
2785   EventMark em("%s", msg);
2786 
2787   ShenandoahWorkerScope scope(workers(),
2788                               ShenandoahWorkerPolicy::calc_workers_for_final_marking(),
2789                               "final marking");
2790 
2791   op_final_mark();
2792 }
2793 
2794 void ShenandoahHeap::entry_init_updaterefs() {
2795   static const char* msg = "Pause Init Update Refs";
2796   ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::init_update_refs);
2797   EventMark em("%s", msg);
2798 
2799   // No workers used in this phase, no setup required
2800 
2801   op_init_updaterefs();
2802 }
2803 
2804 void ShenandoahHeap::entry_final_updaterefs() {
2805   static const char* msg = "Pause Final Update Refs";
2806   ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::final_update_refs);
2807   EventMark em("%s", msg);
2808 
2809   ShenandoahWorkerScope scope(workers(),
2810                               ShenandoahWorkerPolicy::calc_workers_for_final_update_ref(),
2811                               "final reference update");
2812 
2813   op_final_updaterefs();
2814 }
2815 
2816 void ShenandoahHeap::entry_full(GCCause::Cause cause) {
2817   static const char* msg = "Pause Full";
2818   ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::full_gc, true /* log_heap_usage */);
2819   EventMark em("%s", msg);
2820 
2821   ShenandoahWorkerScope scope(workers(),
2822                               ShenandoahWorkerPolicy::calc_workers_for_fullgc(),
2823                               "full gc");
2824 
2825   op_full(cause);
2826 }
2827 
2828 void ShenandoahHeap::entry_degenerated(int point) {
2829   ShenandoahDegenPoint dpoint = (ShenandoahDegenPoint)point;
2830   const char* msg = degen_event_message(dpoint);
2831   ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::degen_gc, true /* log_heap_usage */);
2832   EventMark em("%s", msg);
2833 
2834   ShenandoahWorkerScope scope(workers(),
2835                               ShenandoahWorkerPolicy::calc_workers_for_stw_degenerated(),
2836                               "stw degenerated gc");
2837 
2838   set_degenerated_gc_in_progress(true);
2839   op_degenerated(dpoint);
2840   set_degenerated_gc_in_progress(false);
2841 }
2842 
2843 void ShenandoahHeap::entry_mark() {
2844   TraceCollectorStats tcs(monitoring_support()->concurrent_collection_counters());
2845 
2846   const char* msg = conc_mark_event_message();
2847   ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_mark);
2848   EventMark em("%s", msg);
2849 
2850   ShenandoahWorkerScope scope(workers(),
2851                               ShenandoahWorkerPolicy::calc_workers_for_conc_marking(),
2852                               "concurrent marking");
2853 
2854   try_inject_alloc_failure();
2855   op_mark();
2856 }
2857 
2858 void ShenandoahHeap::entry_evac() {
2859   TraceCollectorStats tcs(monitoring_support()->concurrent_collection_counters());
2860 
2861   static const char* msg = "Concurrent evacuation";
2862   ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_evac);
2863   EventMark em("%s", msg);
2864 
2865   ShenandoahWorkerScope scope(workers(),
2866                               ShenandoahWorkerPolicy::calc_workers_for_conc_evac(),
2867                               "concurrent evacuation");
2868 
2869   try_inject_alloc_failure();
2870   op_conc_evac();
2871 }
2872 
2873 void ShenandoahHeap::entry_updaterefs() {
2874   static const char* msg = "Concurrent update references";
2875   ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_update_refs);
2876   EventMark em("%s", msg);
2877 
2878   ShenandoahWorkerScope scope(workers(),
2879                               ShenandoahWorkerPolicy::calc_workers_for_conc_update_ref(),
2880                               "concurrent reference update");
2881 
2882   try_inject_alloc_failure();
2883   op_updaterefs();
2884 }
2885 
2886 void ShenandoahHeap::entry_weak_roots() {
2887   static const char* msg = "Concurrent weak roots";
2888   ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_weak_roots);
2889   EventMark em("%s", msg);
2890 
2891   ShenandoahWorkerScope scope(workers(),
2892                               ShenandoahWorkerPolicy::calc_workers_for_conc_root_processing(),
2893                               "concurrent weak root");
2894 
2895   try_inject_alloc_failure();
2896   op_weak_roots();
2897 }
2898 
2899 void ShenandoahHeap::entry_class_unloading() {
2900   static const char* msg = "Concurrent class unloading";
2901   ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_class_unload);
2902   EventMark em("%s", msg);
2903 
2904   ShenandoahWorkerScope scope(workers(),
2905                               ShenandoahWorkerPolicy::calc_workers_for_conc_root_processing(),
2906                               "concurrent class unloading");
2907 
2908   try_inject_alloc_failure();
2909   op_class_unloading();
2910 }
2911 
2912 void ShenandoahHeap::entry_strong_roots() {
2913   static const char* msg = "Concurrent strong roots";
2914   ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_strong_roots);
2915   EventMark em("%s", msg);
2916 
2917   ShenandoahGCWorkerPhase worker_phase(ShenandoahPhaseTimings::conc_strong_roots);
2918 
2919   ShenandoahWorkerScope scope(workers(),
2920                               ShenandoahWorkerPolicy::calc_workers_for_conc_root_processing(),
2921                               "concurrent strong root");
2922 
2923   try_inject_alloc_failure();
2924   op_strong_roots();
2925 }
2926 
2927 void ShenandoahHeap::entry_cleanup_early() {
2928   static const char* msg = "Concurrent cleanup";
2929   ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_cleanup_early, true /* log_heap_usage */);
2930   EventMark em("%s", msg);
2931 
2932   // This phase does not use workers, no need for setup
2933 
2934   try_inject_alloc_failure();
2935   op_cleanup_early();
2936 }
2937 
2938 void ShenandoahHeap::entry_cleanup_complete() {
2939   static const char* msg = "Concurrent cleanup";
2940   ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_cleanup_complete, true /* log_heap_usage */);
2941   EventMark em("%s", msg);
2942 
2943   // This phase does not use workers, no need for setup
2944 
2945   try_inject_alloc_failure();
2946   op_cleanup_complete();
2947 }
2948 
2949 void ShenandoahHeap::entry_reset() {
2950   static const char* msg = "Concurrent reset";
2951   ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_reset);
2952   EventMark em("%s", msg);
2953 
2954   ShenandoahWorkerScope scope(workers(),
2955                               ShenandoahWorkerPolicy::calc_workers_for_conc_reset(),
2956                               "concurrent reset");
2957 
2958   try_inject_alloc_failure();
2959   op_reset();
2960 }
2961 
2962 void ShenandoahHeap::entry_preclean() {
2963   if (ShenandoahPreclean && process_references()) {
2964     static const char* msg = "Concurrent precleaning";
2965     ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_preclean);
2966     EventMark em("%s", msg);
2967 
2968     ShenandoahWorkerScope scope(workers(),
2969                                 ShenandoahWorkerPolicy::calc_workers_for_conc_preclean(),
2970                                 "concurrent preclean",
2971                                 /* check_workers = */ false);
2972 
2973     try_inject_alloc_failure();
2974     op_preclean();
2975   }
2976 }
2977 
2978 void ShenandoahHeap::entry_uncommit(double shrink_before) {
2979   static const char *msg = "Concurrent uncommit";
2980   ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_uncommit, true /* log_heap_usage */);
2981   EventMark em("%s", msg);
2982 
2983   op_uncommit(shrink_before);
2984 }
2985 
2986 void ShenandoahHeap::try_inject_alloc_failure() {
2987   if (ShenandoahAllocFailureALot && !cancelled_gc() && ((os::random() % 1000) > 950)) {
2988     _inject_alloc_failure.set();
2989     os::naked_short_sleep(1);
2990     if (cancelled_gc()) {
2991       log_info(gc)("Allocation failure was successfully injected");
2992     }
2993   }
2994 }
2995 
2996 bool ShenandoahHeap::should_inject_alloc_failure() {
2997   return _inject_alloc_failure.is_set() && _inject_alloc_failure.try_unset();
2998 }
2999 
3000 void ShenandoahHeap::initialize_serviceability() {
3001   _memory_pool = new ShenandoahMemoryPool(this);
3002   _cycle_memory_manager.add_pool(_memory_pool);
3003   _stw_memory_manager.add_pool(_memory_pool);
3004 }
3005 
3006 GrowableArray<GCMemoryManager*> ShenandoahHeap::memory_managers() {
3007   GrowableArray<GCMemoryManager*> memory_managers(2);
3008   memory_managers.append(&_cycle_memory_manager);
3009   memory_managers.append(&_stw_memory_manager);
3010   return memory_managers;
3011 }
3012 
3013 GrowableArray<MemoryPool*> ShenandoahHeap::memory_pools() {
3014   GrowableArray<MemoryPool*> memory_pools(1);
3015   memory_pools.append(_memory_pool);
3016   return memory_pools;
3017 }
3018 
3019 MemoryUsage ShenandoahHeap::memory_usage() {
3020   return _memory_pool->get_memory_usage();
3021 }
3022 
3023 void ShenandoahHeap::enter_evacuation() {
3024   _oom_evac_handler.enter_evacuation();
3025 }
3026 
3027 void ShenandoahHeap::leave_evacuation() {
3028   _oom_evac_handler.leave_evacuation();
3029 }
3030 
3031 ShenandoahRegionIterator::ShenandoahRegionIterator() :
3032   _heap(ShenandoahHeap::heap()),
3033   _index(0) {}
3034 
3035 ShenandoahRegionIterator::ShenandoahRegionIterator(ShenandoahHeap* heap) :
3036   _heap(heap),
3037   _index(0) {}
3038 
3039 void ShenandoahRegionIterator::reset() {
3040   _index = 0;
3041 }
3042 
3043 bool ShenandoahRegionIterator::has_next() const {
3044   return _index < _heap->num_regions();
3045 }
3046 
3047 char ShenandoahHeap::gc_state() const {
3048   return _gc_state.raw_value();
3049 }
3050 
3051 void ShenandoahHeap::deduplicate_string(oop str) {
3052   assert(java_lang_String::is_instance(str), "invariant");
3053 
3054   if (ShenandoahStringDedup::is_enabled()) {
3055     ShenandoahStringDedup::deduplicate(str);
3056   }
3057 }
3058 
3059 const char* ShenandoahHeap::init_mark_event_message() const {
3060   assert(!has_forwarded_objects(), "Should not have forwarded objects here");
3061 
3062   bool proc_refs = process_references();
3063   bool unload_cls = unload_classes();
3064 
3065   if (proc_refs && unload_cls) {
3066     return "Pause Init Mark (process weakrefs) (unload classes)";
3067   } else if (proc_refs) {
3068     return "Pause Init Mark (process weakrefs)";
3069   } else if (unload_cls) {
3070     return "Pause Init Mark (unload classes)";
3071   } else {
3072     return "Pause Init Mark";
3073   }
3074 }
3075 
3076 const char* ShenandoahHeap::final_mark_event_message() const {
3077   assert(!has_forwarded_objects(), "Should not have forwarded objects here");
3078 
3079   bool proc_refs = process_references();
3080   bool unload_cls = unload_classes();
3081 
3082   if (proc_refs && unload_cls) {
3083     return "Pause Final Mark (process weakrefs) (unload classes)";
3084   } else if (proc_refs) {
3085     return "Pause Final Mark (process weakrefs)";
3086   } else if (unload_cls) {
3087     return "Pause Final Mark (unload classes)";
3088   } else {
3089     return "Pause Final Mark";
3090   }
3091 }
3092 
3093 const char* ShenandoahHeap::conc_mark_event_message() const {
3094   assert(!has_forwarded_objects(), "Should not have forwarded objects here");
3095 
3096   bool proc_refs = process_references();
3097   bool unload_cls = unload_classes();
3098 
3099   if (proc_refs && unload_cls) {
3100     return "Concurrent marking (process weakrefs) (unload classes)";
3101   } else if (proc_refs) {
3102     return "Concurrent marking (process weakrefs)";
3103   } else if (unload_cls) {
3104     return "Concurrent marking (unload classes)";
3105   } else {
3106     return "Concurrent marking";
3107   }
3108 }
3109 
3110 const char* ShenandoahHeap::degen_event_message(ShenandoahDegenPoint point) const {
3111   switch (point) {
3112     case _degenerated_unset:
3113       return "Pause Degenerated GC (<UNSET>)";
3114     case _degenerated_outside_cycle:
3115       return "Pause Degenerated GC (Outside of Cycle)";
3116     case _degenerated_mark:
3117       return "Pause Degenerated GC (Mark)";
3118     case _degenerated_evac:
3119       return "Pause Degenerated GC (Evacuation)";
3120     case _degenerated_updaterefs:
3121       return "Pause Degenerated GC (Update Refs)";
3122     default:
3123       ShouldNotReachHere();
3124       return "ERROR";
3125   }
3126 }
3127 
3128 ShenandoahLiveData* ShenandoahHeap::get_liveness_cache(uint worker_id) {
3129 #ifdef ASSERT
3130   assert(_liveness_cache != NULL, "sanity");
3131   assert(worker_id < _max_workers, "sanity");
3132   for (uint i = 0; i < num_regions(); i++) {
3133     assert(_liveness_cache[worker_id][i] == 0, "liveness cache should be empty");
3134   }
3135 #endif
3136   return _liveness_cache[worker_id];
3137 }
3138 
3139 void ShenandoahHeap::flush_liveness_cache(uint worker_id) {
3140   assert(worker_id < _max_workers, "sanity");
3141   assert(_liveness_cache != NULL, "sanity");
3142   ShenandoahLiveData* ld = _liveness_cache[worker_id];
3143   for (uint i = 0; i < num_regions(); i++) {
3144     ShenandoahLiveData live = ld[i];
3145     if (live > 0) {
3146       ShenandoahHeapRegion* r = get_region(i);
3147       r->increase_live_data_gc_words(live);
3148       ld[i] = 0;
3149     }
3150   }
3151 }