1 /*
   2  * Copyright (c) 2013, 2020, Red Hat, Inc. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "memory/allocation.hpp"
  27 #include "memory/universe.hpp"
  28 
  29 #include "gc/shared/gcArguments.hpp"
  30 #include "gc/shared/gcTimer.hpp"
  31 #include "gc/shared/gcTraceTime.inline.hpp"
  32 #include "gc/shared/locationPrinter.inline.hpp"
  33 #include "gc/shared/memAllocator.hpp"
  34 #include "gc/shared/oopStorageSet.hpp"
  35 #include "gc/shared/plab.hpp"
  36 
  37 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
  38 #include "gc/shenandoah/shenandoahClosures.inline.hpp"
  39 #include "gc/shenandoah/shenandoahCollectionSet.hpp"
  40 #include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
  41 #include "gc/shenandoah/shenandoahConcurrentMark.inline.hpp"
  42 #include "gc/shenandoah/shenandoahConcurrentRoots.hpp"
  43 #include "gc/shenandoah/shenandoahControlThread.hpp"
  44 #include "gc/shenandoah/shenandoahFreeSet.hpp"
  45 #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
  46 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
  47 #include "gc/shenandoah/shenandoahHeapRegion.inline.hpp"
  48 #include "gc/shenandoah/shenandoahHeapRegionSet.hpp"
  49 #include "gc/shenandoah/shenandoahIUMode.hpp"
  50 #include "gc/shenandoah/shenandoahMarkCompact.hpp"
  51 #include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"
  52 #include "gc/shenandoah/shenandoahMemoryPool.hpp"
  53 #include "gc/shenandoah/shenandoahMetrics.hpp"
  54 #include "gc/shenandoah/shenandoahMonitoringSupport.hpp"
  55 #include "gc/shenandoah/shenandoahNormalMode.hpp"
  56 #include "gc/shenandoah/shenandoahOopClosures.inline.hpp"
  57 #include "gc/shenandoah/shenandoahPacer.inline.hpp"
  58 #include "gc/shenandoah/shenandoahPadding.hpp"
  59 #include "gc/shenandoah/shenandoahParallelCleaning.inline.hpp"
  60 #include "gc/shenandoah/shenandoahPassiveMode.hpp"
  61 #include "gc/shenandoah/shenandoahRootProcessor.inline.hpp"
  62 #include "gc/shenandoah/shenandoahStringDedup.hpp"
  63 #include "gc/shenandoah/shenandoahTaskqueue.hpp"
  64 #include "gc/shenandoah/shenandoahUtils.hpp"
  65 #include "gc/shenandoah/shenandoahVerifier.hpp"
  66 #include "gc/shenandoah/shenandoahCodeRoots.hpp"
  67 #include "gc/shenandoah/shenandoahVMOperations.hpp"
  68 #include "gc/shenandoah/shenandoahWorkGroup.hpp"
  69 #include "gc/shenandoah/shenandoahWorkerPolicy.hpp"
  70 #if INCLUDE_JFR
  71 #include "gc/shenandoah/shenandoahJfrSupport.hpp"
  72 #endif
  73 
  74 #include "memory/metaspace.hpp"
  75 #include "oops/compressedOops.inline.hpp"
  76 #include "runtime/atomic.hpp"
  77 #include "runtime/globals.hpp"
  78 #include "runtime/interfaceSupport.inline.hpp"
  79 #include "runtime/orderAccess.hpp"
  80 #include "runtime/safepointMechanism.hpp"
  81 #include "runtime/vmThread.hpp"
  82 #include "services/mallocTracker.hpp"
  83 #include "utilities/powerOfTwo.hpp"
  84 
  85 ShenandoahHeap* ShenandoahHeap::_heap = NULL;
  86 
  87 #ifdef ASSERT
  88 template <class T>
  89 void ShenandoahAssertToSpaceClosure::do_oop_work(T* p) {
  90   T o = RawAccess<>::oop_load(p);
  91   if (! CompressedOops::is_null(o)) {
  92     oop obj = CompressedOops::decode_not_null(o);
  93     shenandoah_assert_not_forwarded(p, obj);
  94   }
  95 }
  96 
  97 void ShenandoahAssertToSpaceClosure::do_oop(narrowOop* p) { do_oop_work(p); }
  98 void ShenandoahAssertToSpaceClosure::do_oop(oop* p)       { do_oop_work(p); }
  99 #endif
 100 
 101 class ShenandoahPretouchHeapTask : public AbstractGangTask {
 102 private:
 103   ShenandoahRegionIterator _regions;
 104   const size_t _page_size;
 105 public:
 106   ShenandoahPretouchHeapTask(size_t page_size) :
 107     AbstractGangTask("Shenandoah Pretouch Heap"),
 108     _page_size(page_size) {}
 109 
 110   virtual void work(uint worker_id) {
 111     ShenandoahHeapRegion* r = _regions.next();
 112     while (r != NULL) {
 113       os::pretouch_memory(r->bottom(), r->end(), _page_size);
 114       r = _regions.next();
 115     }
 116   }
 117 };
 118 
 119 class ShenandoahPretouchBitmapTask : public AbstractGangTask {
 120 private:
 121   ShenandoahRegionIterator _regions;
 122   char* _bitmap_base;
 123   const size_t _bitmap_size;
 124   const size_t _page_size;
 125 public:
 126   ShenandoahPretouchBitmapTask(char* bitmap_base, size_t bitmap_size, size_t page_size) :
 127     AbstractGangTask("Shenandoah Pretouch Bitmap"),
 128     _bitmap_base(bitmap_base),
 129     _bitmap_size(bitmap_size),
 130     _page_size(page_size) {}
 131 
 132   virtual void work(uint worker_id) {
 133     ShenandoahHeapRegion* r = _regions.next();
 134     while (r != NULL) {
 135       size_t start = r->index()       * ShenandoahHeapRegion::region_size_bytes() / MarkBitMap::heap_map_factor();
 136       size_t end   = (r->index() + 1) * ShenandoahHeapRegion::region_size_bytes() / MarkBitMap::heap_map_factor();
 137       assert (end <= _bitmap_size, "end is sane: " SIZE_FORMAT " < " SIZE_FORMAT, end, _bitmap_size);
 138 
 139       os::pretouch_memory(_bitmap_base + start, _bitmap_base + end, _page_size);
 140 
 141       r = _regions.next();
 142     }
 143   }
 144 };
 145 
 146 jint ShenandoahHeap::initialize() {
 147   //
 148   // Figure out heap sizing
 149   //
 150 
 151   size_t init_byte_size = InitialHeapSize;
 152   size_t min_byte_size  = MinHeapSize;
 153   size_t max_byte_size  = MaxHeapSize;
 154   size_t heap_alignment = HeapAlignment;
 155 
 156   size_t reg_size_bytes = ShenandoahHeapRegion::region_size_bytes();
 157 
 158   if (ShenandoahAlwaysPreTouch) {
 159     // Enabled pre-touch means the entire heap is committed right away.
 160     init_byte_size = max_byte_size;
 161   }
 162 
 163   Universe::check_alignment(max_byte_size,  reg_size_bytes, "Shenandoah heap");
 164   Universe::check_alignment(init_byte_size, reg_size_bytes, "Shenandoah heap");
 165 
 166   _num_regions = ShenandoahHeapRegion::region_count();
 167 
 168   // Now we know the number of regions, initialize the heuristics.
 169   initialize_heuristics();
 170 
 171   size_t num_committed_regions = init_byte_size / reg_size_bytes;
 172   num_committed_regions = MIN2(num_committed_regions, _num_regions);
 173   assert(num_committed_regions <= _num_regions, "sanity");
 174   _initial_size = num_committed_regions * reg_size_bytes;
 175 
 176   size_t num_min_regions = min_byte_size / reg_size_bytes;
 177   num_min_regions = MIN2(num_min_regions, _num_regions);
 178   assert(num_min_regions <= _num_regions, "sanity");
 179   _minimum_size = num_min_regions * reg_size_bytes;
 180 
 181   _committed = _initial_size;
 182 
 183   size_t heap_page_size   = UseLargePages ? (size_t)os::large_page_size() : (size_t)os::vm_page_size();
 184   size_t bitmap_page_size = UseLargePages ? (size_t)os::large_page_size() : (size_t)os::vm_page_size();
 185   size_t region_page_size = UseLargePages ? (size_t)os::large_page_size() : (size_t)os::vm_page_size();
 186 
 187   //
 188   // Reserve and commit memory for heap
 189   //
 190 
 191   ReservedHeapSpace heap_rs = Universe::reserve_heap(max_byte_size, heap_alignment);
 192   initialize_reserved_region(heap_rs);
 193   _heap_region = MemRegion((HeapWord*)heap_rs.base(), heap_rs.size() / HeapWordSize);
 194   _heap_region_special = heap_rs.special();
 195 
 196   assert((((size_t) base()) & ShenandoahHeapRegion::region_size_bytes_mask()) == 0,
 197          "Misaligned heap: " PTR_FORMAT, p2i(base()));
 198 
 199 #if SHENANDOAH_OPTIMIZED_OBJTASK
 200   // The optimized ObjArrayChunkedTask takes some bits away from the full object bits.
 201   // Fail if we ever attempt to address more than we can.
 202   if ((uintptr_t)heap_rs.end() >= ObjArrayChunkedTask::max_addressable()) {
 203     FormatBuffer<512> buf("Shenandoah reserved [" PTR_FORMAT ", " PTR_FORMAT") for the heap, \n"
 204                           "but max object address is " PTR_FORMAT ". Try to reduce heap size, or try other \n"
 205                           "VM options that allocate heap at lower addresses (HeapBaseMinAddress, AllocateHeapAt, etc).",
 206                 p2i(heap_rs.base()), p2i(heap_rs.end()), ObjArrayChunkedTask::max_addressable());
 207     vm_exit_during_initialization("Fatal Error", buf);
 208   }
 209 #endif
 210 
 211   ReservedSpace sh_rs = heap_rs.first_part(max_byte_size);
 212   if (!_heap_region_special) {
 213     os::commit_memory_or_exit(sh_rs.base(), _initial_size, heap_alignment, false,
 214                               "Cannot commit heap memory");
 215   }
 216 
 217   //
 218   // Reserve and commit memory for bitmap(s)
 219   //
 220 
 221   _bitmap_size = MarkBitMap::compute_size(heap_rs.size());
 222   _bitmap_size = align_up(_bitmap_size, bitmap_page_size);
 223 
 224   size_t bitmap_bytes_per_region = reg_size_bytes / MarkBitMap::heap_map_factor();
 225 
 226   guarantee(bitmap_bytes_per_region != 0,
 227             "Bitmap bytes per region should not be zero");
 228   guarantee(is_power_of_2(bitmap_bytes_per_region),
 229             "Bitmap bytes per region should be power of two: " SIZE_FORMAT, bitmap_bytes_per_region);
 230 
 231   if (bitmap_page_size > bitmap_bytes_per_region) {
 232     _bitmap_regions_per_slice = bitmap_page_size / bitmap_bytes_per_region;
 233     _bitmap_bytes_per_slice = bitmap_page_size;
 234   } else {
 235     _bitmap_regions_per_slice = 1;
 236     _bitmap_bytes_per_slice = bitmap_bytes_per_region;
 237   }
 238 
 239   guarantee(_bitmap_regions_per_slice >= 1,
 240             "Should have at least one region per slice: " SIZE_FORMAT,
 241             _bitmap_regions_per_slice);
 242 
 243   guarantee(((_bitmap_bytes_per_slice) % bitmap_page_size) == 0,
 244             "Bitmap slices should be page-granular: bps = " SIZE_FORMAT ", page size = " SIZE_FORMAT,
 245             _bitmap_bytes_per_slice, bitmap_page_size);
 246 
 247   ReservedSpace bitmap(_bitmap_size, bitmap_page_size);
 248   MemTracker::record_virtual_memory_type(bitmap.base(), mtGC);
 249   _bitmap_region = MemRegion((HeapWord*) bitmap.base(), bitmap.size() / HeapWordSize);
 250   _bitmap_region_special = bitmap.special();
 251 
 252   size_t bitmap_init_commit = _bitmap_bytes_per_slice *
 253                               align_up(num_committed_regions, _bitmap_regions_per_slice) / _bitmap_regions_per_slice;
 254   bitmap_init_commit = MIN2(_bitmap_size, bitmap_init_commit);
 255   if (!_bitmap_region_special) {
 256     os::commit_memory_or_exit((char *) _bitmap_region.start(), bitmap_init_commit, bitmap_page_size, false,
 257                               "Cannot commit bitmap memory");
 258   }
 259 
 260   _marking_context = new ShenandoahMarkingContext(_heap_region, _bitmap_region, _num_regions);
 261 
 262   if (ShenandoahVerify) {
 263     ReservedSpace verify_bitmap(_bitmap_size, bitmap_page_size);
 264     if (!verify_bitmap.special()) {
 265       os::commit_memory_or_exit(verify_bitmap.base(), verify_bitmap.size(), bitmap_page_size, false,
 266                                 "Cannot commit verification bitmap memory");
 267     }
 268     MemTracker::record_virtual_memory_type(verify_bitmap.base(), mtGC);
 269     MemRegion verify_bitmap_region = MemRegion((HeapWord *) verify_bitmap.base(), verify_bitmap.size() / HeapWordSize);
 270     _verification_bit_map.initialize(_heap_region, verify_bitmap_region);
 271     _verifier = new ShenandoahVerifier(this, &_verification_bit_map);
 272   }
 273 
 274   // Reserve aux bitmap for use in object_iterate(). We don't commit it here.
 275   ReservedSpace aux_bitmap(_bitmap_size, bitmap_page_size);
 276   MemTracker::record_virtual_memory_type(aux_bitmap.base(), mtGC);
 277   _aux_bitmap_region = MemRegion((HeapWord*) aux_bitmap.base(), aux_bitmap.size() / HeapWordSize);
 278   _aux_bitmap_region_special = aux_bitmap.special();
 279   _aux_bit_map.initialize(_heap_region, _aux_bitmap_region);
 280 
 281   //
 282   // Create regions and region sets
 283   //
 284   size_t region_align = align_up(sizeof(ShenandoahHeapRegion), SHENANDOAH_CACHE_LINE_SIZE);
 285   size_t region_storage_size = align_up(region_align * _num_regions, region_page_size);
 286   region_storage_size = align_up(region_storage_size, os::vm_allocation_granularity());
 287 
 288   ReservedSpace region_storage(region_storage_size, region_page_size);
 289   MemTracker::record_virtual_memory_type(region_storage.base(), mtGC);
 290   if (!region_storage.special()) {
 291     os::commit_memory_or_exit(region_storage.base(), region_storage_size, region_page_size, false,
 292                               "Cannot commit region memory");
 293   }
 294 
 295   _regions = NEW_C_HEAP_ARRAY(ShenandoahHeapRegion*, _num_regions, mtGC);
 296   _free_set = new ShenandoahFreeSet(this, _num_regions);
 297   _collection_set = new ShenandoahCollectionSet(this, sh_rs.base(), sh_rs.size());
 298 
 299   {
 300     ShenandoahHeapLocker locker(lock());
 301 
 302     for (size_t i = 0; i < _num_regions; i++) {
 303       HeapWord* start = (HeapWord*)sh_rs.base() + ShenandoahHeapRegion::region_size_words() * i;
 304       bool is_committed = i < num_committed_regions;
 305       void* loc = region_storage.base() + i * region_align;
 306 
 307       ShenandoahHeapRegion* r = new (loc) ShenandoahHeapRegion(start, i, is_committed);
 308       assert(is_aligned(r, SHENANDOAH_CACHE_LINE_SIZE), "Sanity");
 309 
 310       _marking_context->initialize_top_at_mark_start(r);
 311       _regions[i] = r;
 312       assert(!collection_set()->is_in(i), "New region should not be in collection set");
 313     }
 314 
 315     // Initialize to complete
 316     _marking_context->mark_complete();
 317 
 318     _free_set->rebuild();
 319   }
 320 
 321   if (ShenandoahAlwaysPreTouch) {
 322     assert(!AlwaysPreTouch, "Should have been overridden");
 323 
 324     // For NUMA, it is important to pre-touch the storage under bitmaps with worker threads,
 325     // before initialize() below zeroes it with initializing thread. For any given region,
 326     // we touch the region and the corresponding bitmaps from the same thread.
 327     ShenandoahPushWorkerScope scope(workers(), _max_workers, false);
 328 
 329     size_t pretouch_heap_page_size = heap_page_size;
 330     size_t pretouch_bitmap_page_size = bitmap_page_size;
 331 
 332 #ifdef LINUX
 333     // UseTransparentHugePages would madvise that backing memory can be coalesced into huge
 334     // pages. But, the kernel needs to know that every small page is used, in order to coalesce
 335     // them into huge one. Therefore, we need to pretouch with smaller pages.
 336     if (UseTransparentHugePages) {
 337       pretouch_heap_page_size = (size_t)os::vm_page_size();
 338       pretouch_bitmap_page_size = (size_t)os::vm_page_size();
 339     }
 340 #endif
 341 
 342     // OS memory managers may want to coalesce back-to-back pages. Make their jobs
 343     // simpler by pre-touching continuous spaces (heap and bitmap) separately.
 344 
 345     log_info(gc, init)("Pretouch bitmap: " SIZE_FORMAT " regions, " SIZE_FORMAT " bytes page",
 346                        _num_regions, pretouch_bitmap_page_size);
 347     ShenandoahPretouchBitmapTask bcl(bitmap.base(), _bitmap_size, pretouch_bitmap_page_size);
 348     _workers->run_task(&bcl);
 349 
 350     log_info(gc, init)("Pretouch heap: " SIZE_FORMAT " regions, " SIZE_FORMAT " bytes page",
 351                        _num_regions, pretouch_heap_page_size);
 352     ShenandoahPretouchHeapTask hcl(pretouch_heap_page_size);
 353     _workers->run_task(&hcl);
 354   }
 355 
 356   //
 357   // Initialize the rest of GC subsystems
 358   //
 359 
 360   _liveness_cache = NEW_C_HEAP_ARRAY(ShenandoahLiveData*, _max_workers, mtGC);
 361   for (uint worker = 0; worker < _max_workers; worker++) {
 362     _liveness_cache[worker] = NEW_C_HEAP_ARRAY(ShenandoahLiveData, _num_regions, mtGC);
 363     Copy::fill_to_bytes(_liveness_cache[worker], _num_regions * sizeof(ShenandoahLiveData));
 364   }
 365 
 366   // There should probably be Shenandoah-specific options for these,
 367   // just as there are G1-specific options.
 368   {
 369     ShenandoahSATBMarkQueueSet& satbqs = ShenandoahBarrierSet::satb_mark_queue_set();
 370     satbqs.set_process_completed_buffers_threshold(20); // G1SATBProcessCompletedThreshold
 371     satbqs.set_buffer_enqueue_threshold_percentage(60); // G1SATBBufferEnqueueingThresholdPercent
 372   }
 373 
 374   _monitoring_support = new ShenandoahMonitoringSupport(this);
 375   _phase_timings = new ShenandoahPhaseTimings();
 376   ShenandoahStringDedup::initialize();
 377   ShenandoahCodeRoots::initialize();
 378 
 379   if (ShenandoahPacing) {
 380     _pacer = new ShenandoahPacer(this);
 381     _pacer->setup_for_idle();
 382   } else {
 383     _pacer = NULL;
 384   }
 385 
 386   _control_thread = new ShenandoahControlThread();
 387 
 388   log_info(gc, init)("Initialize Shenandoah heap: " SIZE_FORMAT "%s initial, " SIZE_FORMAT "%s min, " SIZE_FORMAT "%s max",
 389                      byte_size_in_proper_unit(_initial_size),  proper_unit_for_byte_size(_initial_size),
 390                      byte_size_in_proper_unit(_minimum_size),  proper_unit_for_byte_size(_minimum_size),
 391                      byte_size_in_proper_unit(max_capacity()), proper_unit_for_byte_size(max_capacity())
 392   );
 393 
 394   log_info(gc, init)("Safepointing mechanism: thread-local poll");
 395 
 396   return JNI_OK;
 397 }
 398 
 399 void ShenandoahHeap::initialize_heuristics() {
 400   if (ShenandoahGCMode != NULL) {
 401     if (strcmp(ShenandoahGCMode, "normal") == 0) {
 402       _gc_mode = new ShenandoahNormalMode();
 403     } else if (strcmp(ShenandoahGCMode, "iu") == 0) {
 404       _gc_mode = new ShenandoahIUMode();
 405     } else if (strcmp(ShenandoahGCMode, "passive") == 0) {
 406       _gc_mode = new ShenandoahPassiveMode();
 407     } else {
 408       vm_exit_during_initialization("Unknown -XX:ShenandoahGCMode option");
 409     }
 410   } else {
 411     ShouldNotReachHere();
 412   }
 413   _gc_mode->initialize_flags();
 414   if (_gc_mode->is_diagnostic() && !UnlockDiagnosticVMOptions) {
 415     vm_exit_during_initialization(
 416             err_msg("GC mode \"%s\" is diagnostic, and must be enabled via -XX:+UnlockDiagnosticVMOptions.",
 417                     _gc_mode->name()));
 418   }
 419   if (_gc_mode->is_experimental() && !UnlockExperimentalVMOptions) {
 420     vm_exit_during_initialization(
 421             err_msg("GC mode \"%s\" is experimental, and must be enabled via -XX:+UnlockExperimentalVMOptions.",
 422                     _gc_mode->name()));
 423   }
 424   log_info(gc, init)("Shenandoah GC mode: %s",
 425                      _gc_mode->name());
 426 
 427   _heuristics = _gc_mode->initialize_heuristics();
 428 
 429   if (_heuristics->is_diagnostic() && !UnlockDiagnosticVMOptions) {
 430     vm_exit_during_initialization(
 431             err_msg("Heuristics \"%s\" is diagnostic, and must be enabled via -XX:+UnlockDiagnosticVMOptions.",
 432                     _heuristics->name()));
 433   }
 434   if (_heuristics->is_experimental() && !UnlockExperimentalVMOptions) {
 435     vm_exit_during_initialization(
 436             err_msg("Heuristics \"%s\" is experimental, and must be enabled via -XX:+UnlockExperimentalVMOptions.",
 437                     _heuristics->name()));
 438   }
 439   log_info(gc, init)("Shenandoah heuristics: %s",
 440                      _heuristics->name());
 441 }
 442 
 443 #ifdef _MSC_VER
 444 #pragma warning( push )
 445 #pragma warning( disable:4355 ) // 'this' : used in base member initializer list
 446 #endif
 447 
 448 ShenandoahHeap::ShenandoahHeap(ShenandoahCollectorPolicy* policy) :
 449   CollectedHeap(),
 450   _initial_size(0),
 451   _used(0),
 452   _committed(0),
 453   _bytes_allocated_since_gc_start(0),
 454   _max_workers(MAX2(ConcGCThreads, ParallelGCThreads)),
 455   _workers(NULL),
 456   _safepoint_workers(NULL),
 457   _heap_region_special(false),
 458   _num_regions(0),
 459   _regions(NULL),
 460   _update_refs_iterator(this),
 461   _control_thread(NULL),
 462   _shenandoah_policy(policy),
 463   _heuristics(NULL),
 464   _free_set(NULL),
 465   _scm(new ShenandoahConcurrentMark()),
 466   _full_gc(new ShenandoahMarkCompact()),
 467   _pacer(NULL),
 468   _verifier(NULL),
 469   _phase_timings(NULL),
 470   _monitoring_support(NULL),
 471   _memory_pool(NULL),
 472   _stw_memory_manager("Shenandoah Pauses", "end of GC pause"),
 473   _cycle_memory_manager("Shenandoah Cycles", "end of GC cycle"),
 474   _gc_timer(new (ResourceObj::C_HEAP, mtGC) ConcurrentGCTimer()),
 475   _soft_ref_policy(),
 476   _log_min_obj_alignment_in_bytes(LogMinObjAlignmentInBytes),
 477   _ref_processor(NULL),
 478   _marking_context(NULL),
 479   _bitmap_size(0),
 480   _bitmap_regions_per_slice(0),
 481   _bitmap_bytes_per_slice(0),
 482   _bitmap_region_special(false),
 483   _aux_bitmap_region_special(false),
 484   _liveness_cache(NULL),
 485   _collection_set(NULL)
 486 {
 487   _heap = this;
 488 
 489   log_info(gc, init)("GC threads: " UINT32_FORMAT " parallel, " UINT32_FORMAT " concurrent", ParallelGCThreads, ConcGCThreads);
 490 
 491   BarrierSet::set_barrier_set(new ShenandoahBarrierSet(this));
 492 
 493   _max_workers = MAX2(_max_workers, 1U);
 494   _workers = new ShenandoahWorkGang("Shenandoah GC Threads", _max_workers,
 495                             /* are_GC_task_threads */ true,
 496                             /* are_ConcurrentGC_threads */ true);
 497   if (_workers == NULL) {
 498     vm_exit_during_initialization("Failed necessary allocation.");
 499   } else {
 500     _workers->initialize_workers();
 501   }
 502 
 503   if (ParallelGCThreads > 1) {
 504     _safepoint_workers = new ShenandoahWorkGang("Safepoint Cleanup Thread",
 505                                                 ParallelGCThreads,
 506                       /* are_GC_task_threads */ false,
 507                  /* are_ConcurrentGC_threads */ false);
 508     _safepoint_workers->initialize_workers();
 509   }
 510 }
 511 
 512 #ifdef _MSC_VER
 513 #pragma warning( pop )
 514 #endif
 515 
 516 class ShenandoahResetBitmapTask : public AbstractGangTask {
 517 private:
 518   ShenandoahRegionIterator _regions;
 519 
 520 public:
 521   ShenandoahResetBitmapTask() :
 522     AbstractGangTask("Parallel Reset Bitmap Task") {}
 523 
 524   void work(uint worker_id) {
 525     ShenandoahHeapRegion* region = _regions.next();
 526     ShenandoahHeap* heap = ShenandoahHeap::heap();
 527     ShenandoahMarkingContext* const ctx = heap->marking_context();
 528     while (region != NULL) {
 529       if (heap->is_bitmap_slice_committed(region)) {
 530         ctx->clear_bitmap(region);
 531       }
 532       region = _regions.next();
 533     }
 534   }
 535 };
 536 
 537 void ShenandoahHeap::reset_mark_bitmap() {
 538   assert_gc_workers(_workers->active_workers());
 539   mark_incomplete_marking_context();
 540 
 541   ShenandoahResetBitmapTask task;
 542   _workers->run_task(&task);
 543 }
 544 
 545 void ShenandoahHeap::print_on(outputStream* st) const {
 546   st->print_cr("Shenandoah Heap");
 547   st->print_cr(" " SIZE_FORMAT "%s total, " SIZE_FORMAT "%s committed, " SIZE_FORMAT "%s used",
 548                byte_size_in_proper_unit(max_capacity()), proper_unit_for_byte_size(max_capacity()),
 549                byte_size_in_proper_unit(committed()),    proper_unit_for_byte_size(committed()),
 550                byte_size_in_proper_unit(used()),         proper_unit_for_byte_size(used()));
 551   st->print_cr(" " SIZE_FORMAT " x " SIZE_FORMAT"%s regions",
 552                num_regions(),
 553                byte_size_in_proper_unit(ShenandoahHeapRegion::region_size_bytes()),
 554                proper_unit_for_byte_size(ShenandoahHeapRegion::region_size_bytes()));
 555 
 556   st->print("Status: ");
 557   if (has_forwarded_objects())                 st->print("has forwarded objects, ");
 558   if (is_concurrent_mark_in_progress())        st->print("marking, ");
 559   if (is_evacuation_in_progress())             st->print("evacuating, ");
 560   if (is_update_refs_in_progress())            st->print("updating refs, ");
 561   if (is_degenerated_gc_in_progress())         st->print("degenerated gc, ");
 562   if (is_full_gc_in_progress())                st->print("full gc, ");
 563   if (is_full_gc_move_in_progress())           st->print("full gc move, ");
 564   if (is_concurrent_weak_root_in_progress())   st->print("concurrent weak roots, ");
 565   if (is_concurrent_strong_root_in_progress() &&
 566       !is_concurrent_weak_root_in_progress())  st->print("concurrent strong roots, ");
 567 
 568   if (cancelled_gc()) {
 569     st->print("cancelled");
 570   } else {
 571     st->print("not cancelled");
 572   }
 573   st->cr();
 574 
 575   st->print_cr("Reserved region:");
 576   st->print_cr(" - [" PTR_FORMAT ", " PTR_FORMAT ") ",
 577                p2i(reserved_region().start()),
 578                p2i(reserved_region().end()));
 579 
 580   ShenandoahCollectionSet* cset = collection_set();
 581   st->print_cr("Collection set:");
 582   if (cset != NULL) {
 583     st->print_cr(" - map (vanilla): " PTR_FORMAT, p2i(cset->map_address()));
 584     st->print_cr(" - map (biased):  " PTR_FORMAT, p2i(cset->biased_map_address()));
 585   } else {
 586     st->print_cr(" (NULL)");
 587   }
 588 
 589   st->cr();
 590   MetaspaceUtils::print_on(st);
 591 
 592   if (Verbose) {
 593     print_heap_regions_on(st);
 594   }
 595 }
 596 
 597 class ShenandoahInitWorkerGCLABClosure : public ThreadClosure {
 598 public:
 599   void do_thread(Thread* thread) {
 600     assert(thread != NULL, "Sanity");
 601     assert(thread->is_Worker_thread(), "Only worker thread expected");
 602     ShenandoahThreadLocalData::initialize_gclab(thread);
 603   }
 604 };
 605 
 606 void ShenandoahHeap::post_initialize() {
 607   CollectedHeap::post_initialize();
 608   MutexLocker ml(Threads_lock);
 609 
 610   ShenandoahInitWorkerGCLABClosure init_gclabs;
 611   _workers->threads_do(&init_gclabs);
 612 
 613   // gclab can not be initialized early during VM startup, as it can not determinate its max_size.
 614   // Now, we will let WorkGang to initialize gclab when new worker is created.
 615   _workers->set_initialize_gclab();
 616 
 617   _scm->initialize(_max_workers);
 618   _full_gc->initialize(_gc_timer);
 619 
 620   ref_processing_init();
 621 
 622   _heuristics->initialize();
 623 
 624   JFR_ONLY(ShenandoahJFRSupport::register_jfr_type_serializers());
 625 }
 626 
 627 size_t ShenandoahHeap::used() const {
 628   return Atomic::load_acquire(&_used);
 629 }
 630 
 631 size_t ShenandoahHeap::committed() const {
 632   OrderAccess::acquire();
 633   return _committed;
 634 }
 635 
 636 void ShenandoahHeap::increase_committed(size_t bytes) {
 637   shenandoah_assert_heaplocked_or_safepoint();
 638   _committed += bytes;
 639 }
 640 
 641 void ShenandoahHeap::decrease_committed(size_t bytes) {
 642   shenandoah_assert_heaplocked_or_safepoint();
 643   _committed -= bytes;
 644 }
 645 
 646 void ShenandoahHeap::increase_used(size_t bytes) {
 647   Atomic::add(&_used, bytes);
 648 }
 649 
 650 void ShenandoahHeap::set_used(size_t bytes) {
 651   Atomic::release_store_fence(&_used, bytes);
 652 }
 653 
 654 void ShenandoahHeap::decrease_used(size_t bytes) {
 655   assert(used() >= bytes, "never decrease heap size by more than we've left");
 656   Atomic::sub(&_used, bytes);
 657 }
 658 
 659 void ShenandoahHeap::increase_allocated(size_t bytes) {
 660   Atomic::add(&_bytes_allocated_since_gc_start, bytes);
 661 }
 662 
 663 void ShenandoahHeap::notify_mutator_alloc_words(size_t words, bool waste) {
 664   size_t bytes = words * HeapWordSize;
 665   if (!waste) {
 666     increase_used(bytes);
 667   }
 668   increase_allocated(bytes);
 669   if (ShenandoahPacing) {
 670     control_thread()->pacing_notify_alloc(words);
 671     if (waste) {
 672       pacer()->claim_for_alloc(words, true);
 673     }
 674   }
 675 }
 676 
 677 size_t ShenandoahHeap::capacity() const {
 678   return committed();
 679 }
 680 
 681 size_t ShenandoahHeap::max_capacity() const {
 682   return _num_regions * ShenandoahHeapRegion::region_size_bytes();
 683 }
 684 
 685 size_t ShenandoahHeap::min_capacity() const {
 686   return _minimum_size;
 687 }
 688 
 689 size_t ShenandoahHeap::initial_capacity() const {
 690   return _initial_size;
 691 }
 692 
 693 bool ShenandoahHeap::is_in(const void* p) const {
 694   HeapWord* heap_base = (HeapWord*) base();
 695   HeapWord* last_region_end = heap_base + ShenandoahHeapRegion::region_size_words() * num_regions();
 696   return p >= heap_base && p < last_region_end;
 697 }
 698 
 699 void ShenandoahHeap::op_uncommit(double shrink_before) {
 700   assert (ShenandoahUncommit, "should be enabled");
 701 
 702   // Application allocates from the beginning of the heap, and GC allocates at
 703   // the end of it. It is more efficient to uncommit from the end, so that applications
 704   // could enjoy the near committed regions. GC allocations are much less frequent,
 705   // and therefore can accept the committing costs.
 706 
 707   size_t count = 0;
 708   for (size_t i = num_regions(); i > 0; i--) { // care about size_t underflow
 709     ShenandoahHeapRegion* r = get_region(i - 1);
 710     if (r->is_empty_committed() && (r->empty_time() < shrink_before)) {
 711       ShenandoahHeapLocker locker(lock());
 712       if (r->is_empty_committed()) {
 713         // Do not uncommit below minimal capacity
 714         if (committed() < min_capacity() + ShenandoahHeapRegion::region_size_bytes()) {
 715           break;
 716         }
 717 
 718         r->make_uncommitted();
 719         count++;
 720       }
 721     }
 722     SpinPause(); // allow allocators to take the lock
 723   }
 724 
 725   if (count > 0) {
 726     control_thread()->notify_heap_changed();
 727   }
 728 }
 729 
 730 HeapWord* ShenandoahHeap::allocate_from_gclab_slow(Thread* thread, size_t size) {
 731   // New object should fit the GCLAB size
 732   size_t min_size = MAX2(size, PLAB::min_size());
 733 
 734   // Figure out size of new GCLAB, looking back at heuristics. Expand aggressively.
 735   size_t new_size = ShenandoahThreadLocalData::gclab_size(thread) * 2;
 736   new_size = MIN2(new_size, PLAB::max_size());
 737   new_size = MAX2(new_size, PLAB::min_size());
 738 
 739   // Record new heuristic value even if we take any shortcut. This captures
 740   // the case when moderately-sized objects always take a shortcut. At some point,
 741   // heuristics should catch up with them.
 742   ShenandoahThreadLocalData::set_gclab_size(thread, new_size);
 743 
 744   if (new_size < size) {
 745     // New size still does not fit the object. Fall back to shared allocation.
 746     // This avoids retiring perfectly good GCLABs, when we encounter a large object.
 747     return NULL;
 748   }
 749 
 750   // Retire current GCLAB, and allocate a new one.
 751   PLAB* gclab = ShenandoahThreadLocalData::gclab(thread);
 752   gclab->retire();
 753 
 754   size_t actual_size = 0;
 755   HeapWord* gclab_buf = allocate_new_gclab(min_size, new_size, &actual_size);
 756   if (gclab_buf == NULL) {
 757     return NULL;
 758   }
 759 
 760   assert (size <= actual_size, "allocation should fit");
 761 
 762   if (ZeroTLAB) {
 763     // ..and clear it.
 764     Copy::zero_to_words(gclab_buf, actual_size);
 765   } else {
 766     // ...and zap just allocated object.
 767 #ifdef ASSERT
 768     // Skip mangling the space corresponding to the object header to
 769     // ensure that the returned space is not considered parsable by
 770     // any concurrent GC thread.
 771     size_t hdr_size = oopDesc::header_size();
 772     Copy::fill_to_words(gclab_buf + hdr_size, actual_size - hdr_size, badHeapWordVal);
 773 #endif // ASSERT
 774   }
 775   gclab->set_buf(gclab_buf, actual_size);
 776   return gclab->allocate(size);
 777 }
 778 
 779 HeapWord* ShenandoahHeap::allocate_new_tlab(size_t min_size,
 780                                             size_t requested_size,
 781                                             size_t* actual_size) {
 782   ShenandoahAllocRequest req = ShenandoahAllocRequest::for_tlab(min_size, requested_size);
 783   HeapWord* res = allocate_memory(req);
 784   if (res != NULL) {
 785     *actual_size = req.actual_size();
 786   } else {
 787     *actual_size = 0;
 788   }
 789   return res;
 790 }
 791 
 792 HeapWord* ShenandoahHeap::allocate_new_gclab(size_t min_size,
 793                                              size_t word_size,
 794                                              size_t* actual_size) {
 795   ShenandoahAllocRequest req = ShenandoahAllocRequest::for_gclab(min_size, word_size);
 796   HeapWord* res = allocate_memory(req);
 797   if (res != NULL) {
 798     *actual_size = req.actual_size();
 799   } else {
 800     *actual_size = 0;
 801   }
 802   return res;
 803 }
 804 
 805 HeapWord* ShenandoahHeap::allocate_memory(ShenandoahAllocRequest& req) {
 806   intptr_t pacer_epoch = 0;
 807   bool in_new_region = false;
 808   HeapWord* result = NULL;
 809 
 810   if (req.is_mutator_alloc()) {
 811     if (ShenandoahPacing) {
 812       pacer()->pace_for_alloc(req.size());
 813       pacer_epoch = pacer()->epoch();
 814     }
 815 
 816     if (!ShenandoahAllocFailureALot || !should_inject_alloc_failure()) {
 817       result = allocate_memory_under_lock(req, in_new_region);
 818     }
 819 
 820     // Allocation failed, block until control thread reacted, then retry allocation.
 821     //
 822     // It might happen that one of the threads requesting allocation would unblock
 823     // way later after GC happened, only to fail the second allocation, because
 824     // other threads have already depleted the free storage. In this case, a better
 825     // strategy is to try again, as long as GC makes progress.
 826     //
 827     // Then, we need to make sure the allocation was retried after at least one
 828     // Full GC, which means we want to try more than ShenandoahFullGCThreshold times.
 829 
 830     size_t tries = 0;
 831 
 832     while (result == NULL && _progress_last_gc.is_set()) {
 833       tries++;
 834       control_thread()->handle_alloc_failure(req);
 835       result = allocate_memory_under_lock(req, in_new_region);
 836     }
 837 
 838     while (result == NULL && tries <= ShenandoahFullGCThreshold) {
 839       tries++;
 840       control_thread()->handle_alloc_failure(req);
 841       result = allocate_memory_under_lock(req, in_new_region);
 842     }
 843 
 844   } else {
 845     assert(req.is_gc_alloc(), "Can only accept GC allocs here");
 846     result = allocate_memory_under_lock(req, in_new_region);
 847     // Do not call handle_alloc_failure() here, because we cannot block.
 848     // The allocation failure would be handled by the LRB slowpath with handle_alloc_failure_evac().
 849   }
 850 
 851   if (in_new_region) {
 852     control_thread()->notify_heap_changed();
 853   }
 854 
 855   if (result != NULL) {
 856     size_t requested = req.size();
 857     size_t actual = req.actual_size();
 858 
 859     assert (req.is_lab_alloc() || (requested == actual),
 860             "Only LAB allocations are elastic: %s, requested = " SIZE_FORMAT ", actual = " SIZE_FORMAT,
 861             ShenandoahAllocRequest::alloc_type_to_string(req.type()), requested, actual);
 862 
 863     if (req.is_mutator_alloc()) {
 864       notify_mutator_alloc_words(actual, false);
 865 
 866       // If we requested more than we were granted, give the rest back to pacer.
 867       // This only matters if we are in the same pacing epoch: do not try to unpace
 868       // over the budget for the other phase.
 869       if (ShenandoahPacing && (pacer_epoch > 0) && (requested > actual)) {
 870         pacer()->unpace_for_alloc(pacer_epoch, requested - actual);
 871       }
 872     } else {
 873       increase_used(actual*HeapWordSize);
 874     }
 875   }
 876 
 877   return result;
 878 }
 879 
 880 HeapWord* ShenandoahHeap::allocate_memory_under_lock(ShenandoahAllocRequest& req, bool& in_new_region) {
 881   ShenandoahHeapLocker locker(lock());
 882   return _free_set->allocate(req, in_new_region);
 883 }
 884 
 885 HeapWord* ShenandoahHeap::mem_allocate(size_t size,
 886                                         bool*  gc_overhead_limit_was_exceeded) {
 887   ShenandoahAllocRequest req = ShenandoahAllocRequest::for_shared(size);
 888   return allocate_memory(req);
 889 }
 890 
 891 MetaWord* ShenandoahHeap::satisfy_failed_metadata_allocation(ClassLoaderData* loader_data,
 892                                                              size_t size,
 893                                                              Metaspace::MetadataType mdtype) {
 894   MetaWord* result;
 895 
 896   // Inform metaspace OOM to GC heuristics if class unloading is possible.
 897   if (heuristics()->can_unload_classes()) {
 898     ShenandoahHeuristics* h = heuristics();
 899     h->record_metaspace_oom();
 900   }
 901 
 902   // Expand and retry allocation
 903   result = loader_data->metaspace_non_null()->expand_and_allocate(size, mdtype);
 904   if (result != NULL) {
 905     return result;
 906   }
 907 
 908   // Start full GC
 909   collect(GCCause::_metadata_GC_clear_soft_refs);
 910 
 911   // Retry allocation
 912   result = loader_data->metaspace_non_null()->allocate(size, mdtype);
 913   if (result != NULL) {
 914     return result;
 915   }
 916 
 917   // Expand and retry allocation
 918   result = loader_data->metaspace_non_null()->expand_and_allocate(size, mdtype);
 919   if (result != NULL) {
 920     return result;
 921   }
 922 
 923   // Out of memory
 924   return NULL;
 925 }
 926 
 927 class ShenandoahConcurrentEvacuateRegionObjectClosure : public ObjectClosure {
 928 private:
 929   ShenandoahHeap* const _heap;
 930   Thread* const _thread;
 931 public:
 932   ShenandoahConcurrentEvacuateRegionObjectClosure(ShenandoahHeap* heap) :
 933     _heap(heap), _thread(Thread::current()) {}
 934 
 935   void do_object(oop p) {
 936     shenandoah_assert_marked(NULL, p);
 937     if (!p->is_forwarded()) {
 938       _heap->evacuate_object(p, _thread);
 939     }
 940   }
 941 };
 942 
 943 class ShenandoahEvacuationTask : public AbstractGangTask {
 944 private:
 945   ShenandoahHeap* const _sh;
 946   ShenandoahCollectionSet* const _cs;
 947   bool _concurrent;
 948 public:
 949   ShenandoahEvacuationTask(ShenandoahHeap* sh,
 950                            ShenandoahCollectionSet* cs,
 951                            bool concurrent) :
 952     AbstractGangTask("Parallel Evacuation Task"),
 953     _sh(sh),
 954     _cs(cs),
 955     _concurrent(concurrent)
 956   {}
 957 
 958   void work(uint worker_id) {
 959     if (_concurrent) {
 960       ShenandoahConcurrentWorkerSession worker_session(worker_id);
 961       ShenandoahSuspendibleThreadSetJoiner stsj(ShenandoahSuspendibleWorkers);
 962       ShenandoahEvacOOMScope oom_evac_scope;
 963       do_work();
 964     } else {
 965       ShenandoahParallelWorkerSession worker_session(worker_id);
 966       ShenandoahEvacOOMScope oom_evac_scope;
 967       do_work();
 968     }
 969   }
 970 
 971 private:
 972   void do_work() {
 973     ShenandoahConcurrentEvacuateRegionObjectClosure cl(_sh);
 974     ShenandoahHeapRegion* r;
 975     while ((r =_cs->claim_next()) != NULL) {
 976       assert(r->has_live(), "Region " SIZE_FORMAT " should have been reclaimed early", r->index());
 977       _sh->marked_object_iterate(r, &cl);
 978 
 979       if (ShenandoahPacing) {
 980         _sh->pacer()->report_evac(r->used() >> LogHeapWordSize);
 981       }
 982 
 983       if (_sh->check_cancelled_gc_and_yield(_concurrent)) {
 984         break;
 985       }
 986     }
 987   }
 988 };
 989 
 990 void ShenandoahHeap::trash_cset_regions() {
 991   ShenandoahHeapLocker locker(lock());
 992 
 993   ShenandoahCollectionSet* set = collection_set();
 994   ShenandoahHeapRegion* r;
 995   set->clear_current_index();
 996   while ((r = set->next()) != NULL) {
 997     r->make_trash();
 998   }
 999   collection_set()->clear();
1000 }
1001 
1002 void ShenandoahHeap::print_heap_regions_on(outputStream* st) const {
1003   st->print_cr("Heap Regions:");
1004   st->print_cr("EU=empty-uncommitted, EC=empty-committed, R=regular, H=humongous start, HC=humongous continuation, CS=collection set, T=trash, P=pinned");
1005   st->print_cr("BTE=bottom/top/end, U=used, T=TLAB allocs, G=GCLAB allocs, S=shared allocs, L=live data");
1006   st->print_cr("R=root, CP=critical pins, TAMS=top-at-mark-start, UWM=update watermark");
1007   st->print_cr("SN=alloc sequence number");
1008 
1009   for (size_t i = 0; i < num_regions(); i++) {
1010     get_region(i)->print_on(st);
1011   }
1012 }
1013 
1014 void ShenandoahHeap::trash_humongous_region_at(ShenandoahHeapRegion* start) {
1015   assert(start->is_humongous_start(), "reclaim regions starting with the first one");
1016 
1017   oop humongous_obj = oop(start->bottom());
1018   size_t size = humongous_obj->size();
1019   size_t required_regions = ShenandoahHeapRegion::required_regions(size * HeapWordSize);
1020   size_t index = start->index() + required_regions - 1;
1021 
1022   assert(!start->has_live(), "liveness must be zero");
1023 
1024   for(size_t i = 0; i < required_regions; i++) {
1025     // Reclaim from tail. Otherwise, assertion fails when printing region to trace log,
1026     // as it expects that every region belongs to a humongous region starting with a humongous start region.
1027     ShenandoahHeapRegion* region = get_region(index --);
1028 
1029     assert(region->is_humongous(), "expect correct humongous start or continuation");
1030     assert(!region->is_cset(), "Humongous region should not be in collection set");
1031 
1032     region->make_trash_immediate();
1033   }
1034 }
1035 
1036 class ShenandoahRetireGCLABClosure : public ThreadClosure {
1037 public:
1038   void do_thread(Thread* thread) {
1039     PLAB* gclab = ShenandoahThreadLocalData::gclab(thread);
1040     assert(gclab != NULL, "GCLAB should be initialized for %s", thread->name());
1041     gclab->retire();
1042   }
1043 };
1044 
1045 void ShenandoahHeap::make_parsable(bool retire_tlabs) {
1046   if (UseTLAB) {
1047     CollectedHeap::ensure_parsability(retire_tlabs);
1048   }
1049   ShenandoahRetireGCLABClosure cl;
1050   for (JavaThreadIteratorWithHandle jtiwh; JavaThread *t = jtiwh.next(); ) {
1051     cl.do_thread(t);
1052   }
1053   workers()->threads_do(&cl);
1054 }
1055 
1056 void ShenandoahHeap::resize_tlabs() {
1057   CollectedHeap::resize_all_tlabs();
1058 }
1059 
1060 class ShenandoahEvacuateUpdateRootsTask : public AbstractGangTask {
1061 private:
1062   ShenandoahRootEvacuator* _rp;
1063 
1064 public:
1065   ShenandoahEvacuateUpdateRootsTask(ShenandoahRootEvacuator* rp) :
1066     AbstractGangTask("Shenandoah evacuate and update roots"),
1067     _rp(rp) {}
1068 
1069   void work(uint worker_id) {
1070     ShenandoahParallelWorkerSession worker_session(worker_id);
1071     ShenandoahEvacOOMScope oom_evac_scope;
1072     ShenandoahEvacuateUpdateRootsClosure<> cl;
1073     MarkingCodeBlobClosure blobsCl(&cl, CodeBlobToOopClosure::FixRelocations);
1074     _rp->roots_do(worker_id, &cl);
1075   }
1076 };
1077 
1078 void ShenandoahHeap::evacuate_and_update_roots() {
1079 #if COMPILER2_OR_JVMCI
1080   DerivedPointerTable::clear();
1081 #endif
1082   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Only iterate roots while world is stopped");
1083   {
1084     // Include concurrent roots if current cycle can not process those roots concurrently
1085     ShenandoahRootEvacuator rp(workers()->active_workers(),
1086                                ShenandoahPhaseTimings::init_evac,
1087                                !ShenandoahConcurrentRoots::should_do_concurrent_roots(),
1088                                !ShenandoahConcurrentRoots::should_do_concurrent_class_unloading());
1089     ShenandoahEvacuateUpdateRootsTask roots_task(&rp);
1090     workers()->run_task(&roots_task);
1091   }
1092 
1093 #if COMPILER2_OR_JVMCI
1094   DerivedPointerTable::update_pointers();
1095 #endif
1096 }
1097 
1098 // Returns size in bytes
1099 size_t ShenandoahHeap::unsafe_max_tlab_alloc(Thread *thread) const {
1100   if (ShenandoahElasticTLAB) {
1101     // With Elastic TLABs, return the max allowed size, and let the allocation path
1102     // figure out the safe size for current allocation.
1103     return ShenandoahHeapRegion::max_tlab_size_bytes();
1104   } else {
1105     return MIN2(_free_set->unsafe_peek_free(), ShenandoahHeapRegion::max_tlab_size_bytes());
1106   }
1107 }
1108 
1109 size_t ShenandoahHeap::max_tlab_size() const {
1110   // Returns size in words
1111   return ShenandoahHeapRegion::max_tlab_size_words();
1112 }
1113 
1114 class ShenandoahRetireAndResetGCLABClosure : public ThreadClosure {
1115 public:
1116   void do_thread(Thread* thread) {
1117     PLAB* gclab = ShenandoahThreadLocalData::gclab(thread);
1118     gclab->retire();
1119     if (ShenandoahThreadLocalData::gclab_size(thread) > 0) {
1120       ShenandoahThreadLocalData::set_gclab_size(thread, 0);
1121     }
1122   }
1123 };
1124 
1125 void ShenandoahHeap::retire_and_reset_gclabs() {
1126   ShenandoahRetireAndResetGCLABClosure cl;
1127   for (JavaThreadIteratorWithHandle jtiwh; JavaThread *t = jtiwh.next(); ) {
1128     cl.do_thread(t);
1129   }
1130   workers()->threads_do(&cl);
1131 }
1132 
1133 void ShenandoahHeap::collect(GCCause::Cause cause) {
1134   control_thread()->request_gc(cause);
1135 }
1136 
1137 void ShenandoahHeap::do_full_collection(bool clear_all_soft_refs) {
1138   //assert(false, "Shouldn't need to do full collections");
1139 }
1140 
1141 HeapWord* ShenandoahHeap::block_start(const void* addr) const {
1142   ShenandoahHeapRegion* r = heap_region_containing(addr);
1143   if (r != NULL) {
1144     return r->block_start(addr);
1145   }
1146   return NULL;
1147 }
1148 
1149 bool ShenandoahHeap::block_is_obj(const HeapWord* addr) const {
1150   ShenandoahHeapRegion* r = heap_region_containing(addr);
1151   return r->block_is_obj(addr);
1152 }
1153 
1154 bool ShenandoahHeap::print_location(outputStream* st, void* addr) const {
1155   return BlockLocationPrinter<ShenandoahHeap>::print_location(st, addr);
1156 }
1157 
1158 jlong ShenandoahHeap::millis_since_last_gc() {
1159   double v = heuristics()->time_since_last_gc() * 1000;
1160   assert(0 <= v && v <= max_jlong, "value should fit: %f", v);
1161   return (jlong)v;
1162 }
1163 
1164 void ShenandoahHeap::prepare_for_verify() {
1165   if (SafepointSynchronize::is_at_safepoint() || ! UseTLAB) {
1166     make_parsable(false);
1167   }
1168 }
1169 
1170 void ShenandoahHeap::print_gc_threads_on(outputStream* st) const {
1171   workers()->print_worker_threads_on(st);
1172   if (ShenandoahStringDedup::is_enabled()) {
1173     ShenandoahStringDedup::print_worker_threads_on(st);
1174   }
1175 }
1176 
1177 void ShenandoahHeap::gc_threads_do(ThreadClosure* tcl) const {
1178   workers()->threads_do(tcl);
1179   if (_safepoint_workers != NULL) {
1180     _safepoint_workers->threads_do(tcl);
1181   }
1182   if (ShenandoahStringDedup::is_enabled()) {
1183     ShenandoahStringDedup::threads_do(tcl);
1184   }
1185 }
1186 
1187 void ShenandoahHeap::print_tracing_info() const {
1188   LogTarget(Info, gc, stats) lt;
1189   if (lt.is_enabled()) {
1190     ResourceMark rm;
1191     LogStream ls(lt);
1192 
1193     phase_timings()->print_global_on(&ls);
1194 
1195     ls.cr();
1196     ls.cr();
1197 
1198     shenandoah_policy()->print_gc_stats(&ls);
1199 
1200     ls.cr();
1201     ls.cr();
1202 
1203     if (ShenandoahPacing) {
1204       pacer()->print_on(&ls);
1205     }
1206 
1207     ls.cr();
1208     ls.cr();
1209   }
1210 }
1211 
1212 void ShenandoahHeap::verify(VerifyOption vo) {
1213   if (ShenandoahSafepoint::is_at_shenandoah_safepoint()) {
1214     if (ShenandoahVerify) {
1215       verifier()->verify_generic(vo);
1216     } else {
1217       // TODO: Consider allocating verification bitmaps on demand,
1218       // and turn this on unconditionally.
1219     }
1220   }
1221 }
1222 size_t ShenandoahHeap::tlab_capacity(Thread *thr) const {
1223   return _free_set->capacity();
1224 }
1225 
1226 class ObjectIterateScanRootClosure : public BasicOopIterateClosure {
1227 private:
1228   MarkBitMap* _bitmap;
1229   Stack<oop,mtGC>* _oop_stack;
1230   ShenandoahHeap* const _heap;
1231   ShenandoahMarkingContext* const _marking_context;
1232 
1233   template <class T>
1234   void do_oop_work(T* p) {
1235     T o = RawAccess<>::oop_load(p);
1236     if (!CompressedOops::is_null(o)) {
1237       oop obj = CompressedOops::decode_not_null(o);
1238       if (_heap->is_concurrent_weak_root_in_progress() && !_marking_context->is_marked(obj)) {
1239         // There may be dead oops in weak roots in concurrent root phase, do not touch them.
1240         return;
1241       }
1242       obj = ShenandoahBarrierSet::resolve_forwarded_not_null(obj);
1243 
1244       assert(oopDesc::is_oop(obj), "must be a valid oop");
1245       if (!_bitmap->is_marked(obj)) {
1246         _bitmap->mark(obj);
1247         _oop_stack->push(obj);
1248       }
1249     }
1250   }
1251 public:
1252   ObjectIterateScanRootClosure(MarkBitMap* bitmap, Stack<oop,mtGC>* oop_stack) :
1253     _bitmap(bitmap), _oop_stack(oop_stack), _heap(ShenandoahHeap::heap()),
1254     _marking_context(_heap->marking_context()) {}
1255   void do_oop(oop* p)       { do_oop_work(p); }
1256   void do_oop(narrowOop* p) { do_oop_work(p); }
1257 };
1258 
1259 /*
1260  * This is public API, used in preparation of object_iterate().
1261  * Since we don't do linear scan of heap in object_iterate() (see comment below), we don't
1262  * need to make the heap parsable. For Shenandoah-internal linear heap scans that we can
1263  * control, we call SH::make_tlabs_parsable().
1264  */
1265 void ShenandoahHeap::ensure_parsability(bool retire_tlabs) {
1266   // No-op.
1267 }
1268 
1269 /*
1270  * Iterates objects in the heap. This is public API, used for, e.g., heap dumping.
1271  *
1272  * We cannot safely iterate objects by doing a linear scan at random points in time. Linear
1273  * scanning needs to deal with dead objects, which may have dead Klass* pointers (e.g.
1274  * calling oopDesc::size() would crash) or dangling reference fields (crashes) etc. Linear
1275  * scanning therefore depends on having a valid marking bitmap to support it. However, we only
1276  * have a valid marking bitmap after successful marking. In particular, we *don't* have a valid
1277  * marking bitmap during marking, after aborted marking or during/after cleanup (when we just
1278  * wiped the bitmap in preparation for next marking).
1279  *
1280  * For all those reasons, we implement object iteration as a single marking traversal, reporting
1281  * objects as we mark+traverse through the heap, starting from GC roots. JVMTI IterateThroughHeap
1282  * is allowed to report dead objects, but is not required to do so.
1283  */
1284 void ShenandoahHeap::object_iterate(ObjectClosure* cl) {
1285   assert(SafepointSynchronize::is_at_safepoint(), "safe iteration is only available during safepoints");
1286   if (!_aux_bitmap_region_special && !os::commit_memory((char*)_aux_bitmap_region.start(), _aux_bitmap_region.byte_size(), false)) {
1287     log_warning(gc)("Could not commit native memory for auxiliary marking bitmap for heap iteration");
1288     return;
1289   }
1290 
1291   // Reset bitmap
1292   _aux_bit_map.clear();
1293 
1294   Stack<oop,mtGC> oop_stack;
1295 
1296   // First, we process GC roots according to current GC cycle. This populates the work stack with initial objects.
1297   ShenandoahHeapIterationRootScanner rp;
1298   ObjectIterateScanRootClosure oops(&_aux_bit_map, &oop_stack);
1299 
1300   rp.roots_do(&oops);
1301 
1302   // Work through the oop stack to traverse heap.
1303   while (! oop_stack.is_empty()) {
1304     oop obj = oop_stack.pop();
1305     assert(oopDesc::is_oop(obj), "must be a valid oop");
1306     cl->do_object(obj);
1307     obj->oop_iterate(&oops);
1308   }
1309 
1310   assert(oop_stack.is_empty(), "should be empty");
1311 
1312   if (!_aux_bitmap_region_special && !os::uncommit_memory((char*)_aux_bitmap_region.start(), _aux_bitmap_region.byte_size())) {
1313     log_warning(gc)("Could not uncommit native memory for auxiliary marking bitmap for heap iteration");
1314   }
1315 }
1316 
1317 void ShenandoahHeap::run_task(AbstractGangTask* task) {
1318   workers()->run_task(task, workers()->active_workers());
1319 }
1320 
1321 // Keep alive an object that was loaded with AS_NO_KEEPALIVE.
1322 void ShenandoahHeap::keep_alive(oop obj) {
1323   if (is_concurrent_mark_in_progress()) {
1324     ShenandoahBarrierSet::barrier_set()->enqueue(obj);
1325   }
1326 }
1327 
1328 void ShenandoahHeap::heap_region_iterate(ShenandoahHeapRegionClosure* blk) const {
1329   for (size_t i = 0; i < num_regions(); i++) {
1330     ShenandoahHeapRegion* current = get_region(i);
1331     blk->heap_region_do(current);
1332   }
1333 }
1334 
1335 class ShenandoahParallelHeapRegionTask : public AbstractGangTask {
1336 private:
1337   ShenandoahHeap* const _heap;
1338   ShenandoahHeapRegionClosure* const _blk;
1339 
1340   shenandoah_padding(0);
1341   volatile size_t _index;
1342   shenandoah_padding(1);
1343 
1344 public:
1345   ShenandoahParallelHeapRegionTask(ShenandoahHeapRegionClosure* blk) :
1346           AbstractGangTask("Parallel Region Task"),
1347           _heap(ShenandoahHeap::heap()), _blk(blk), _index(0) {}
1348 
1349   void work(uint worker_id) {
1350     size_t stride = ShenandoahParallelRegionStride;
1351 
1352     size_t max = _heap->num_regions();
1353     while (_index < max) {
1354       size_t cur = Atomic::fetch_and_add(&_index, stride);
1355       size_t start = cur;
1356       size_t end = MIN2(cur + stride, max);
1357       if (start >= max) break;
1358 
1359       for (size_t i = cur; i < end; i++) {
1360         ShenandoahHeapRegion* current = _heap->get_region(i);
1361         _blk->heap_region_do(current);
1362       }
1363     }
1364   }
1365 };
1366 
1367 void ShenandoahHeap::parallel_heap_region_iterate(ShenandoahHeapRegionClosure* blk) const {
1368   assert(blk->is_thread_safe(), "Only thread-safe closures here");
1369   if (num_regions() > ShenandoahParallelRegionStride) {
1370     ShenandoahParallelHeapRegionTask task(blk);
1371     workers()->run_task(&task);
1372   } else {
1373     heap_region_iterate(blk);
1374   }
1375 }
1376 
1377 class ShenandoahInitMarkUpdateRegionStateClosure : public ShenandoahHeapRegionClosure {
1378 private:
1379   ShenandoahMarkingContext* const _ctx;
1380 public:
1381   ShenandoahInitMarkUpdateRegionStateClosure() : _ctx(ShenandoahHeap::heap()->marking_context()) {}
1382 
1383   void heap_region_do(ShenandoahHeapRegion* r) {
1384     assert(!r->has_live(), "Region " SIZE_FORMAT " should have no live data", r->index());
1385     if (r->is_active()) {
1386       // Check if region needs updating its TAMS. We have updated it already during concurrent
1387       // reset, so it is very likely we don't need to do another write here.
1388       if (_ctx->top_at_mark_start(r) != r->top()) {
1389         _ctx->capture_top_at_mark_start(r);
1390       }
1391     } else {
1392       assert(_ctx->top_at_mark_start(r) == r->top(),
1393              "Region " SIZE_FORMAT " should already have correct TAMS", r->index());
1394     }
1395   }
1396 
1397   bool is_thread_safe() { return true; }
1398 };
1399 
1400 void ShenandoahHeap::op_init_mark() {
1401   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Should be at safepoint");
1402   assert(Thread::current()->is_VM_thread(), "can only do this in VMThread");
1403 
1404   assert(marking_context()->is_bitmap_clear(), "need clear marking bitmap");
1405   assert(!marking_context()->is_complete(), "should not be complete");
1406   assert(!has_forwarded_objects(), "No forwarded objects on this path");
1407 
1408   if (ShenandoahVerify) {
1409     verifier()->verify_before_concmark();
1410   }
1411 
1412   if (VerifyBeforeGC) {
1413     Universe::verify();
1414   }
1415 
1416   set_concurrent_mark_in_progress(true);
1417   // We need to reset all TLABs because we'd lose marks on all objects allocated in them.
1418   {
1419     ShenandoahGCSubPhase phase(ShenandoahPhaseTimings::make_parsable);
1420     make_parsable(true);
1421   }
1422 
1423   {
1424     ShenandoahGCSubPhase phase(ShenandoahPhaseTimings::init_update_region_states);
1425     ShenandoahInitMarkUpdateRegionStateClosure cl;
1426     parallel_heap_region_iterate(&cl);
1427   }
1428 
1429   // Make above changes visible to worker threads
1430   OrderAccess::fence();
1431 
1432   concurrent_mark()->mark_roots(ShenandoahPhaseTimings::scan_roots);
1433 
1434   if (UseTLAB) {
1435     ShenandoahGCSubPhase phase(ShenandoahPhaseTimings::resize_tlabs);
1436     resize_tlabs();
1437   }
1438 
1439   if (ShenandoahPacing) {
1440     pacer()->setup_for_mark();
1441   }
1442 
1443   // Arm nmethods for concurrent marking. When a nmethod is about to be executed,
1444   // we need to make sure that all its metadata are marked. alternative is to remark
1445   // thread roots at final mark pause, but it can be potential latency killer.
1446   if (ShenandoahConcurrentRoots::should_do_concurrent_class_unloading()) {
1447     ShenandoahCodeRoots::arm_nmethods();
1448   }
1449 }
1450 
1451 void ShenandoahHeap::op_mark() {
1452   concurrent_mark()->mark_from_roots();
1453 }
1454 
1455 class ShenandoahFinalMarkUpdateRegionStateClosure : public ShenandoahHeapRegionClosure {
1456 private:
1457   ShenandoahMarkingContext* const _ctx;
1458   ShenandoahHeapLock* const _lock;
1459 
1460 public:
1461   ShenandoahFinalMarkUpdateRegionStateClosure() :
1462     _ctx(ShenandoahHeap::heap()->complete_marking_context()), _lock(ShenandoahHeap::heap()->lock()) {}
1463 
1464   void heap_region_do(ShenandoahHeapRegion* r) {
1465     if (r->is_active()) {
1466       // All allocations past TAMS are implicitly live, adjust the region data.
1467       // Bitmaps/TAMS are swapped at this point, so we need to poll complete bitmap.
1468       HeapWord *tams = _ctx->top_at_mark_start(r);
1469       HeapWord *top = r->top();
1470       if (top > tams) {
1471         r->increase_live_data_alloc_words(pointer_delta(top, tams));
1472       }
1473 
1474       // We are about to select the collection set, make sure it knows about
1475       // current pinning status. Also, this allows trashing more regions that
1476       // now have their pinning status dropped.
1477       if (r->is_pinned()) {
1478         if (r->pin_count() == 0) {
1479           ShenandoahHeapLocker locker(_lock);
1480           r->make_unpinned();
1481         }
1482       } else {
1483         if (r->pin_count() > 0) {
1484           ShenandoahHeapLocker locker(_lock);
1485           r->make_pinned();
1486         }
1487       }
1488 
1489       // Remember limit for updating refs. It's guaranteed that we get no
1490       // from-space-refs written from here on.
1491       r->set_update_watermark(r->top());
1492     } else {
1493       assert(!r->has_live(), "Region " SIZE_FORMAT " should have no live data", r->index());
1494       assert(_ctx->top_at_mark_start(r) == r->top(),
1495              "Region " SIZE_FORMAT " should have correct TAMS", r->index());
1496     }
1497   }
1498 
1499   bool is_thread_safe() { return true; }
1500 };
1501 
1502 void ShenandoahHeap::op_final_mark() {
1503   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Should be at safepoint");
1504   assert(!has_forwarded_objects(), "No forwarded objects on this path");
1505 
1506   // It is critical that we
1507   // evacuate roots right after finishing marking, so that we don't
1508   // get unmarked objects in the roots.
1509 
1510   if (!cancelled_gc()) {
1511     concurrent_mark()->finish_mark_from_roots(/* full_gc = */ false);
1512 
1513     // Marking is completed, deactivate SATB barrier
1514     set_concurrent_mark_in_progress(false);
1515     mark_complete_marking_context();
1516 
1517     parallel_cleaning(false /* full gc*/);
1518 
1519     if (ShenandoahVerify) {
1520       verifier()->verify_roots_no_forwarded();
1521     }
1522 
1523     {
1524       ShenandoahGCSubPhase phase(ShenandoahPhaseTimings::final_update_region_states);
1525       ShenandoahFinalMarkUpdateRegionStateClosure cl;
1526       parallel_heap_region_iterate(&cl);
1527 
1528       assert_pinned_region_status();
1529     }
1530 
1531     // Retire the TLABs, which will force threads to reacquire their TLABs after the pause.
1532     // This is needed for two reasons. Strong one: new allocations would be with new freeset,
1533     // which would be outside the collection set, so no cset writes would happen there.
1534     // Weaker one: new allocations would happen past update watermark, and so less work would
1535     // be needed for reference updates (would update the large filler instead).
1536     {
1537       ShenandoahGCSubPhase phase(ShenandoahPhaseTimings::retire_tlabs);
1538       make_parsable(true);
1539     }
1540 
1541     {
1542       ShenandoahGCSubPhase phase(ShenandoahPhaseTimings::choose_cset);
1543       ShenandoahHeapLocker locker(lock());
1544       _collection_set->clear();
1545       heuristics()->choose_collection_set(_collection_set);
1546     }
1547 
1548     {
1549       ShenandoahGCSubPhase phase(ShenandoahPhaseTimings::final_rebuild_freeset);
1550       ShenandoahHeapLocker locker(lock());
1551       _free_set->rebuild();
1552     }
1553 
1554     if (!is_degenerated_gc_in_progress()) {
1555       prepare_concurrent_roots();
1556       prepare_concurrent_unloading();
1557     }
1558 
1559     // If collection set has candidates, start evacuation.
1560     // Otherwise, bypass the rest of the cycle.
1561     if (!collection_set()->is_empty()) {
1562       ShenandoahGCSubPhase init_evac(ShenandoahPhaseTimings::init_evac);
1563 
1564       if (ShenandoahVerify) {
1565         verifier()->verify_before_evacuation();
1566       }
1567 
1568       set_evacuation_in_progress(true);
1569       // From here on, we need to update references.
1570       set_has_forwarded_objects(true);
1571 
1572       if (!is_degenerated_gc_in_progress()) {
1573         if (ShenandoahConcurrentRoots::should_do_concurrent_class_unloading()) {
1574           ShenandoahCodeRoots::arm_nmethods();
1575         }
1576         evacuate_and_update_roots();
1577       }
1578 
1579       if (ShenandoahPacing) {
1580         pacer()->setup_for_evac();
1581       }
1582 
1583       if (ShenandoahVerify) {
1584         ShenandoahRootVerifier::RootTypes types = ShenandoahRootVerifier::None;
1585         if (ShenandoahConcurrentRoots::should_do_concurrent_roots()) {
1586           types = ShenandoahRootVerifier::combine(ShenandoahRootVerifier::JNIHandleRoots, ShenandoahRootVerifier::WeakRoots);
1587           types = ShenandoahRootVerifier::combine(types, ShenandoahRootVerifier::CLDGRoots);
1588           types = ShenandoahRootVerifier::combine(types, ShenandoahRootVerifier::StringDedupRoots);
1589         }
1590 
1591         if (ShenandoahConcurrentRoots::should_do_concurrent_class_unloading()) {
1592           types = ShenandoahRootVerifier::combine(types, ShenandoahRootVerifier::CodeRoots);
1593         }
1594         verifier()->verify_roots_no_forwarded_except(types);
1595         verifier()->verify_during_evacuation();
1596       }
1597     } else {
1598       if (ShenandoahVerify) {
1599         verifier()->verify_after_concmark();
1600       }
1601 
1602       if (VerifyAfterGC) {
1603         Universe::verify();
1604       }
1605     }
1606 
1607   } else {
1608     // If this cycle was updating references, we need to keep the has_forwarded_objects
1609     // flag on, for subsequent phases to deal with it.
1610     concurrent_mark()->cancel();
1611     set_concurrent_mark_in_progress(false);
1612 
1613     if (process_references()) {
1614       // Abandon reference processing right away: pre-cleaning must have failed.
1615       ReferenceProcessor *rp = ref_processor();
1616       rp->disable_discovery();
1617       rp->abandon_partial_discovery();
1618       rp->verify_no_references_recorded();
1619     }
1620   }
1621 }
1622 
1623 void ShenandoahHeap::op_conc_evac() {
1624   ShenandoahEvacuationTask task(this, _collection_set, true);
1625   workers()->run_task(&task);
1626 }
1627 
1628 void ShenandoahHeap::op_stw_evac() {
1629   ShenandoahEvacuationTask task(this, _collection_set, false);
1630   workers()->run_task(&task);
1631 }
1632 
1633 void ShenandoahHeap::op_updaterefs() {
1634   update_heap_references(true);
1635 }
1636 
1637 void ShenandoahHeap::op_cleanup_early() {
1638   free_set()->recycle_trash();
1639 }
1640 
1641 void ShenandoahHeap::op_cleanup_complete() {
1642   free_set()->recycle_trash();
1643 }
1644 
1645 class ShenandoahConcurrentRootsEvacUpdateTask : public AbstractGangTask {
1646 private:
1647   ShenandoahVMRoots<true /*concurrent*/>        _vm_roots;
1648   ShenandoahClassLoaderDataRoots<true /*concurrent*/, false /*single threaded*/> _cld_roots;
1649   ShenandoahConcurrentStringDedupRoots          _dedup_roots;
1650 
1651 public:
1652   ShenandoahConcurrentRootsEvacUpdateTask() :
1653     AbstractGangTask("Shenandoah Evacuate/Update Concurrent Strong Roots Task") {
1654   }
1655 
1656   void work(uint worker_id) {
1657     ShenandoahEvacOOMScope oom;
1658     {
1659       // vm_roots and weak_roots are OopStorage backed roots, concurrent iteration
1660       // may race against OopStorage::release() calls.
1661       ShenandoahEvacUpdateOopStorageRootsClosure cl;
1662       _vm_roots.oops_do<ShenandoahEvacUpdateOopStorageRootsClosure>(&cl, worker_id);
1663     }
1664 
1665     {
1666       ShenandoahEvacuateUpdateRootsClosure<> cl;
1667       CLDToOopClosure clds(&cl, ClassLoaderData::_claim_strong);
1668       _cld_roots.cld_do(&clds, worker_id);
1669     }
1670 
1671     {
1672       ShenandoahForwardedIsAliveClosure is_alive;
1673       ShenandoahEvacuateUpdateRootsClosure<MO_RELEASE> keep_alive;
1674       _dedup_roots.oops_do(&is_alive, &keep_alive, worker_id);
1675     }
1676   }
1677 };
1678 
1679 class ShenandoahEvacUpdateCleanupOopStorageRootsClosure : public BasicOopIterateClosure {
1680 private:
1681   ShenandoahHeap* const _heap;
1682   ShenandoahMarkingContext* const _mark_context;
1683   bool  _evac_in_progress;
1684   Thread* const _thread;
1685   size_t  _dead_counter;
1686 
1687 public:
1688   ShenandoahEvacUpdateCleanupOopStorageRootsClosure();
1689   void do_oop(oop* p);
1690   void do_oop(narrowOop* p);
1691 
1692   size_t dead_counter() const;
1693   void reset_dead_counter();
1694 };
1695 
1696 ShenandoahEvacUpdateCleanupOopStorageRootsClosure::ShenandoahEvacUpdateCleanupOopStorageRootsClosure() :
1697   _heap(ShenandoahHeap::heap()),
1698   _mark_context(ShenandoahHeap::heap()->marking_context()),
1699   _evac_in_progress(ShenandoahHeap::heap()->is_evacuation_in_progress()),
1700   _thread(Thread::current()),
1701   _dead_counter(0) {
1702 }
1703 
1704 void ShenandoahEvacUpdateCleanupOopStorageRootsClosure::do_oop(oop* p) {
1705   const oop obj = RawAccess<>::oop_load(p);
1706   if (!CompressedOops::is_null(obj)) {
1707     if (!_mark_context->is_marked(obj)) {
1708       shenandoah_assert_correct(p, obj);
1709       oop old = Atomic::cmpxchg(p, obj, oop(NULL));
1710       if (obj == old) {
1711         _dead_counter ++;
1712       }
1713     } else if (_evac_in_progress && _heap->in_collection_set(obj)) {
1714       oop resolved = ShenandoahBarrierSet::resolve_forwarded_not_null(obj);
1715       if (resolved == obj) {
1716         resolved = _heap->evacuate_object(obj, _thread);
1717       }
1718       Atomic::cmpxchg(p, obj, resolved);
1719       assert(_heap->cancelled_gc() ||
1720              _mark_context->is_marked(resolved) && !_heap->in_collection_set(resolved),
1721              "Sanity");
1722     }
1723   }
1724 }
1725 
1726 void ShenandoahEvacUpdateCleanupOopStorageRootsClosure::do_oop(narrowOop* p) {
1727   ShouldNotReachHere();
1728 }
1729 
1730 size_t ShenandoahEvacUpdateCleanupOopStorageRootsClosure::dead_counter() const {
1731   return _dead_counter;
1732 }
1733 
1734 void ShenandoahEvacUpdateCleanupOopStorageRootsClosure::reset_dead_counter() {
1735   _dead_counter = 0;
1736 }
1737 
1738 class ShenandoahIsCLDAliveClosure : public CLDClosure {
1739 public:
1740   void do_cld(ClassLoaderData* cld) {
1741     cld->is_alive();
1742   }
1743 };
1744 
1745 class ShenandoahIsNMethodAliveClosure: public NMethodClosure {
1746 public:
1747   void do_nmethod(nmethod* n) {
1748     n->is_unloading();
1749   }
1750 };
1751 
1752 // This task not only evacuates/updates marked weak roots, but also "NULL"
1753 // dead weak roots.
1754 class ShenandoahConcurrentWeakRootsEvacUpdateTask : public AbstractGangTask {
1755 private:
1756   ShenandoahWeakRoot<true /*concurrent*/>  _jni_roots;
1757   ShenandoahWeakRoot<true /*concurrent*/>  _string_table_roots;
1758   ShenandoahWeakRoot<true /*concurrent*/>  _resolved_method_table_roots;
1759   ShenandoahWeakRoot<true /*concurrent*/>  _vm_roots;
1760 
1761   // Roots related to concurrent class unloading
1762   ShenandoahClassLoaderDataRoots<true /* concurrent */, false /* single thread*/>
1763                                            _cld_roots;
1764   ShenandoahConcurrentNMethodIterator      _nmethod_itr;
1765   bool                                     _concurrent_class_unloading;
1766 
1767 public:
1768   ShenandoahConcurrentWeakRootsEvacUpdateTask() :
1769     AbstractGangTask("Shenandoah Concurrent Weak Root Task"),
1770     _jni_roots(OopStorageSet::jni_weak(), ShenandoahPhaseTimings::JNIWeakRoots),
1771     _string_table_roots(OopStorageSet::string_table_weak(), ShenandoahPhaseTimings::StringTableRoots),
1772     _resolved_method_table_roots(OopStorageSet::resolved_method_table_weak(), ShenandoahPhaseTimings::ResolvedMethodTableRoots),
1773     _vm_roots(OopStorageSet::vm_weak(), ShenandoahPhaseTimings::VMWeakRoots),
1774     _nmethod_itr(ShenandoahCodeRoots::table()),
1775     _concurrent_class_unloading(ShenandoahConcurrentRoots::should_do_concurrent_class_unloading()) {
1776     StringTable::reset_dead_counter();
1777     ResolvedMethodTable::reset_dead_counter();
1778     if (_concurrent_class_unloading) {
1779       MutexLocker mu(CodeCache_lock, Mutex::_no_safepoint_check_flag);
1780       _nmethod_itr.nmethods_do_begin();
1781     }
1782   }
1783 
1784   ~ShenandoahConcurrentWeakRootsEvacUpdateTask() {
1785     StringTable::finish_dead_counter();
1786     ResolvedMethodTable::finish_dead_counter();
1787     if (_concurrent_class_unloading) {
1788       MutexLocker mu(CodeCache_lock, Mutex::_no_safepoint_check_flag);
1789       _nmethod_itr.nmethods_do_end();
1790     }
1791   }
1792 
1793   void work(uint worker_id) {
1794     {
1795       ShenandoahEvacOOMScope oom;
1796       // jni_roots and weak_roots are OopStorage backed roots, concurrent iteration
1797       // may race against OopStorage::release() calls.
1798       ShenandoahEvacUpdateCleanupOopStorageRootsClosure cl;
1799       _jni_roots.oops_do(&cl, worker_id);
1800       _vm_roots.oops_do(&cl, worker_id);
1801 
1802       cl.reset_dead_counter();
1803       _string_table_roots.oops_do(&cl, worker_id);
1804       StringTable::inc_dead_counter(cl.dead_counter());
1805 
1806       cl.reset_dead_counter();
1807       _resolved_method_table_roots.oops_do(&cl, worker_id);
1808       ResolvedMethodTable::inc_dead_counter(cl.dead_counter());
1809     }
1810 
1811     // If we are going to perform concurrent class unloading later on, we need to
1812     // cleanup the weak oops in CLD and determinate nmethod's unloading state, so that we
1813     // can cleanup immediate garbage sooner.
1814     if (_concurrent_class_unloading) {
1815       // Applies ShenandoahIsCLDAlive closure to CLDs, native barrier will either NULL the
1816       // CLD's holder or evacuate it.
1817       ShenandoahIsCLDAliveClosure is_cld_alive;
1818       _cld_roots.cld_do(&is_cld_alive, worker_id);
1819 
1820       // Applies ShenandoahIsNMethodAliveClosure to registered nmethods.
1821       // The closure calls nmethod->is_unloading(). The is_unloading
1822       // state is cached, therefore, during concurrent class unloading phase,
1823       // we will not touch the metadata of unloading nmethods
1824       ShenandoahIsNMethodAliveClosure is_nmethod_alive;
1825       _nmethod_itr.nmethods_do(&is_nmethod_alive);
1826     }
1827   }
1828 };
1829 
1830 void ShenandoahHeap::op_weak_roots() {
1831   if (is_concurrent_weak_root_in_progress()) {
1832     // Concurrent weak root processing
1833     ShenandoahConcurrentWeakRootsEvacUpdateTask task;
1834     workers()->run_task(&task);
1835     if (!ShenandoahConcurrentRoots::should_do_concurrent_class_unloading()) {
1836       set_concurrent_weak_root_in_progress(false);
1837     }
1838   }
1839 }
1840 
1841 void ShenandoahHeap::op_class_unloading() {
1842   assert (is_concurrent_weak_root_in_progress() &&
1843           ShenandoahConcurrentRoots::should_do_concurrent_class_unloading(),
1844           "Checked by caller");
1845   _unloader.unload();
1846   set_concurrent_weak_root_in_progress(false);
1847 }
1848 
1849 void ShenandoahHeap::op_strong_roots() {
1850   assert(is_concurrent_strong_root_in_progress(), "Checked by caller");
1851   ShenandoahConcurrentRootsEvacUpdateTask task;
1852   workers()->run_task(&task);
1853   set_concurrent_strong_root_in_progress(false);
1854 }
1855 
1856 class ShenandoahResetUpdateRegionStateClosure : public ShenandoahHeapRegionClosure {
1857 private:
1858   ShenandoahMarkingContext* const _ctx;
1859 public:
1860   ShenandoahResetUpdateRegionStateClosure() : _ctx(ShenandoahHeap::heap()->marking_context()) {}
1861 
1862   void heap_region_do(ShenandoahHeapRegion* r) {
1863     if (r->is_active()) {
1864       // Reset live data and set TAMS optimistically. We would recheck these under the pause
1865       // anyway to capture any updates that happened since now.
1866       r->clear_live_data();
1867       _ctx->capture_top_at_mark_start(r);
1868     }
1869   }
1870 
1871   bool is_thread_safe() { return true; }
1872 };
1873 
1874 void ShenandoahHeap::op_reset() {
1875   if (ShenandoahPacing) {
1876     pacer()->setup_for_reset();
1877   }
1878   reset_mark_bitmap();
1879 
1880   ShenandoahResetUpdateRegionStateClosure cl;
1881   parallel_heap_region_iterate(&cl);
1882 }
1883 
1884 void ShenandoahHeap::op_preclean() {
1885   if (ShenandoahPacing) {
1886     pacer()->setup_for_preclean();
1887   }
1888   concurrent_mark()->preclean_weak_refs();
1889 }
1890 
1891 void ShenandoahHeap::op_full(GCCause::Cause cause) {
1892   ShenandoahMetricsSnapshot metrics;
1893   metrics.snap_before();
1894 
1895   full_gc()->do_it(cause);
1896   if (UseTLAB) {
1897     ShenandoahGCSubPhase phase(ShenandoahPhaseTimings::full_gc_resize_tlabs);
1898     resize_all_tlabs();
1899   }
1900 
1901   metrics.snap_after();
1902 
1903   if (metrics.is_good_progress()) {
1904     _progress_last_gc.set();
1905   } else {
1906     // Nothing to do. Tell the allocation path that we have failed to make
1907     // progress, and it can finally fail.
1908     _progress_last_gc.unset();
1909   }
1910 }
1911 
1912 void ShenandoahHeap::op_degenerated(ShenandoahDegenPoint point) {
1913   // Degenerated GC is STW, but it can also fail. Current mechanics communicates
1914   // GC failure via cancelled_concgc() flag. So, if we detect the failure after
1915   // some phase, we have to upgrade the Degenerate GC to Full GC.
1916 
1917   clear_cancelled_gc();
1918 
1919   ShenandoahMetricsSnapshot metrics;
1920   metrics.snap_before();
1921 
1922   switch (point) {
1923     // The cases below form the Duff's-like device: it describes the actual GC cycle,
1924     // but enters it at different points, depending on which concurrent phase had
1925     // degenerated.
1926 
1927     case _degenerated_outside_cycle:
1928       // We have degenerated from outside the cycle, which means something is bad with
1929       // the heap, most probably heavy humongous fragmentation, or we are very low on free
1930       // space. It makes little sense to wait for Full GC to reclaim as much as it can, when
1931       // we can do the most aggressive degen cycle, which includes processing references and
1932       // class unloading, unless those features are explicitly disabled.
1933       //
1934       // Note that we can only do this for "outside-cycle" degens, otherwise we would risk
1935       // changing the cycle parameters mid-cycle during concurrent -> degenerated handover.
1936       set_process_references(heuristics()->can_process_references());
1937       set_unload_classes(heuristics()->can_unload_classes());
1938 
1939       op_reset();
1940 
1941       op_init_mark();
1942       if (cancelled_gc()) {
1943         op_degenerated_fail();
1944         return;
1945       }
1946 
1947     case _degenerated_mark:
1948       op_final_mark();
1949       if (cancelled_gc()) {
1950         op_degenerated_fail();
1951         return;
1952       }
1953 
1954       if (!has_forwarded_objects() && ShenandoahConcurrentRoots::can_do_concurrent_class_unloading()) {
1955         // Disarm nmethods that armed for concurrent mark. On normal cycle, it would
1956         // be disarmed while conc-roots phase is running.
1957         // TODO: Call op_conc_roots() here instead
1958         ShenandoahCodeRoots::disarm_nmethods();
1959       }
1960 
1961       op_cleanup_early();
1962 
1963     case _degenerated_evac:
1964       // If heuristics thinks we should do the cycle, this flag would be set,
1965       // and we can do evacuation. Otherwise, it would be the shortcut cycle.
1966       if (is_evacuation_in_progress()) {
1967 
1968         // Degeneration under oom-evac protocol might have left some objects in
1969         // collection set un-evacuated. Restart evacuation from the beginning to
1970         // capture all objects. For all the objects that are already evacuated,
1971         // it would be a simple check, which is supposed to be fast. This is also
1972         // safe to do even without degeneration, as CSet iterator is at beginning
1973         // in preparation for evacuation anyway.
1974         //
1975         // Before doing that, we need to make sure we never had any cset-pinned
1976         // regions. This may happen if allocation failure happened when evacuating
1977         // the about-to-be-pinned object, oom-evac protocol left the object in
1978         // the collection set, and then the pin reached the cset region. If we continue
1979         // the cycle here, we would trash the cset and alive objects in it. To avoid
1980         // it, we fail degeneration right away and slide into Full GC to recover.
1981 
1982         {
1983           sync_pinned_region_status();
1984           collection_set()->clear_current_index();
1985 
1986           ShenandoahHeapRegion* r;
1987           while ((r = collection_set()->next()) != NULL) {
1988             if (r->is_pinned()) {
1989               cancel_gc(GCCause::_shenandoah_upgrade_to_full_gc);
1990               op_degenerated_fail();
1991               return;
1992             }
1993           }
1994 
1995           collection_set()->clear_current_index();
1996         }
1997 
1998         op_stw_evac();
1999         if (cancelled_gc()) {
2000           op_degenerated_fail();
2001           return;
2002         }
2003       }
2004 
2005       // If heuristics thinks we should do the cycle, this flag would be set,
2006       // and we need to do update-refs. Otherwise, it would be the shortcut cycle.
2007       if (has_forwarded_objects()) {
2008         op_init_updaterefs();
2009         if (cancelled_gc()) {
2010           op_degenerated_fail();
2011           return;
2012         }
2013       }
2014 
2015     case _degenerated_updaterefs:
2016       if (has_forwarded_objects()) {
2017         op_final_updaterefs();
2018         if (cancelled_gc()) {
2019           op_degenerated_fail();
2020           return;
2021         }
2022       }
2023 
2024       op_cleanup_complete();
2025       break;
2026 
2027     default:
2028       ShouldNotReachHere();
2029   }
2030 
2031   if (ShenandoahVerify) {
2032     verifier()->verify_after_degenerated();
2033   }
2034 
2035   if (VerifyAfterGC) {
2036     Universe::verify();
2037   }
2038 
2039   metrics.snap_after();
2040 
2041   // Check for futility and fail. There is no reason to do several back-to-back Degenerated cycles,
2042   // because that probably means the heap is overloaded and/or fragmented.
2043   if (!metrics.is_good_progress()) {
2044     _progress_last_gc.unset();
2045     cancel_gc(GCCause::_shenandoah_upgrade_to_full_gc);
2046     op_degenerated_futile();
2047   } else {
2048     _progress_last_gc.set();
2049   }
2050 }
2051 
2052 void ShenandoahHeap::op_degenerated_fail() {
2053   log_info(gc)("Cannot finish degeneration, upgrading to Full GC");
2054   shenandoah_policy()->record_degenerated_upgrade_to_full();
2055   op_full(GCCause::_shenandoah_upgrade_to_full_gc);
2056 }
2057 
2058 void ShenandoahHeap::op_degenerated_futile() {
2059   shenandoah_policy()->record_degenerated_upgrade_to_full();
2060   op_full(GCCause::_shenandoah_upgrade_to_full_gc);
2061 }
2062 
2063 void ShenandoahHeap::force_satb_flush_all_threads() {
2064   if (!is_concurrent_mark_in_progress()) {
2065     // No need to flush SATBs
2066     return;
2067   }
2068 
2069   for (JavaThreadIteratorWithHandle jtiwh; JavaThread *t = jtiwh.next(); ) {
2070     ShenandoahThreadLocalData::set_force_satb_flush(t, true);
2071   }
2072   // The threads are not "acquiring" their thread-local data, but it does not
2073   // hurt to "release" the updates here anyway.
2074   OrderAccess::fence();
2075 }
2076 
2077 void ShenandoahHeap::set_gc_state_all_threads(char state) {
2078   for (JavaThreadIteratorWithHandle jtiwh; JavaThread *t = jtiwh.next(); ) {
2079     ShenandoahThreadLocalData::set_gc_state(t, state);
2080   }
2081 }
2082 
2083 void ShenandoahHeap::set_gc_state_mask(uint mask, bool value) {
2084   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Should really be Shenandoah safepoint");
2085   _gc_state.set_cond(mask, value);
2086   set_gc_state_all_threads(_gc_state.raw_value());
2087 }
2088 
2089 void ShenandoahHeap::set_concurrent_mark_in_progress(bool in_progress) {
2090   if (has_forwarded_objects()) {
2091     set_gc_state_mask(MARKING | UPDATEREFS, in_progress);
2092   } else {
2093     set_gc_state_mask(MARKING, in_progress);
2094   }
2095   ShenandoahBarrierSet::satb_mark_queue_set().set_active_all_threads(in_progress, !in_progress);
2096 }
2097 
2098 void ShenandoahHeap::set_evacuation_in_progress(bool in_progress) {
2099   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Only call this at safepoint");
2100   set_gc_state_mask(EVACUATION, in_progress);
2101 }
2102 
2103 void ShenandoahHeap::set_concurrent_strong_root_in_progress(bool in_progress) {
2104   assert(ShenandoahConcurrentRoots::can_do_concurrent_roots(), "Why set the flag?");
2105   if (in_progress) {
2106     _concurrent_strong_root_in_progress.set();
2107   } else {
2108     _concurrent_strong_root_in_progress.unset();
2109   }
2110 }
2111 
2112 void ShenandoahHeap::set_concurrent_weak_root_in_progress(bool in_progress) {
2113   assert(ShenandoahConcurrentRoots::can_do_concurrent_roots(), "Why set the flag?");
2114   if (in_progress) {
2115     _concurrent_weak_root_in_progress.set();
2116   } else {
2117     _concurrent_weak_root_in_progress.unset();
2118   }
2119 }
2120 
2121 void ShenandoahHeap::ref_processing_init() {
2122   assert(_max_workers > 0, "Sanity");
2123 
2124   bool mt_processing = ParallelRefProcEnabled && (ParallelGCThreads > 1);
2125   bool mt_discovery = _max_workers > 1;
2126 
2127   _ref_processor =
2128     new ReferenceProcessor(&_subject_to_discovery,  // is_subject_to_discovery
2129                            mt_processing,           // MT processing
2130                            _max_workers,            // Degree of MT processing
2131                            mt_discovery,            // MT discovery
2132                            _max_workers,            // Degree of MT discovery
2133                            false,                   // Reference discovery is not atomic
2134                            NULL,                    // No closure, should be installed before use
2135                            true);                   // Scale worker threads
2136 
2137   log_info(gc, init)("Reference processing: %s discovery, %s processing",
2138           mt_discovery ? "parallel" : "serial",
2139           mt_processing ? "parallel" : "serial");
2140 
2141   shenandoah_assert_rp_isalive_not_installed();
2142 }
2143 
2144 GCTracer* ShenandoahHeap::tracer() {
2145   return shenandoah_policy()->tracer();
2146 }
2147 
2148 size_t ShenandoahHeap::tlab_used(Thread* thread) const {
2149   return _free_set->used();
2150 }
2151 
2152 bool ShenandoahHeap::try_cancel_gc() {
2153   while (true) {
2154     jbyte prev = _cancelled_gc.cmpxchg(CANCELLED, CANCELLABLE);
2155     if (prev == CANCELLABLE) return true;
2156     else if (prev == CANCELLED) return false;
2157     assert(ShenandoahSuspendibleWorkers, "should not get here when not using suspendible workers");
2158     assert(prev == NOT_CANCELLED, "must be NOT_CANCELLED");
2159     if (Thread::current()->is_Java_thread()) {
2160       // We need to provide a safepoint here, otherwise we might
2161       // spin forever if a SP is pending.
2162       ThreadBlockInVM sp(JavaThread::current());
2163       SpinPause();
2164     }
2165   }
2166 }
2167 
2168 void ShenandoahHeap::cancel_gc(GCCause::Cause cause) {
2169   if (try_cancel_gc()) {
2170     FormatBuffer<> msg("Cancelling GC: %s", GCCause::to_string(cause));
2171     log_info(gc)("%s", msg.buffer());
2172     Events::log(Thread::current(), "%s", msg.buffer());
2173   }
2174 }
2175 
2176 uint ShenandoahHeap::max_workers() {
2177   return _max_workers;
2178 }
2179 
2180 void ShenandoahHeap::stop() {
2181   // The shutdown sequence should be able to terminate when GC is running.
2182 
2183   // Step 0. Notify policy to disable event recording.
2184   _shenandoah_policy->record_shutdown();
2185 
2186   // Step 1. Notify control thread that we are in shutdown.
2187   // Note that we cannot do that with stop(), because stop() is blocking and waits for the actual shutdown.
2188   // Doing stop() here would wait for the normal GC cycle to complete, never falling through to cancel below.
2189   control_thread()->prepare_for_graceful_shutdown();
2190 
2191   // Step 2. Notify GC workers that we are cancelling GC.
2192   cancel_gc(GCCause::_shenandoah_stop_vm);
2193 
2194   // Step 3. Wait until GC worker exits normally.
2195   control_thread()->stop();
2196 
2197   // Step 4. Stop String Dedup thread if it is active
2198   if (ShenandoahStringDedup::is_enabled()) {
2199     ShenandoahStringDedup::stop();
2200   }
2201 }
2202 
2203 void ShenandoahHeap::stw_unload_classes(bool full_gc) {
2204   if (!unload_classes()) return;
2205 
2206   // Unload classes and purge SystemDictionary.
2207   {
2208     ShenandoahGCSubPhase phase(full_gc ?
2209                                ShenandoahPhaseTimings::full_gc_purge_class_unload :
2210                                ShenandoahPhaseTimings::purge_class_unload);
2211     bool purged_class = SystemDictionary::do_unloading(gc_timer());
2212 
2213     ShenandoahIsAliveSelector is_alive;
2214     uint num_workers = _workers->active_workers();
2215     ShenandoahClassUnloadingTask unlink_task(is_alive.is_alive_closure(), num_workers, purged_class);
2216     _workers->run_task(&unlink_task);
2217   }
2218 
2219   {
2220     ShenandoahGCSubPhase phase(full_gc ?
2221                                ShenandoahPhaseTimings::full_gc_purge_cldg :
2222                                ShenandoahPhaseTimings::purge_cldg);
2223     ClassLoaderDataGraph::purge();
2224   }
2225   // Resize and verify metaspace
2226   MetaspaceGC::compute_new_size();
2227   MetaspaceUtils::verify_metrics();
2228 }
2229 
2230 // Weak roots are either pre-evacuated (final mark) or updated (final updaterefs),
2231 // so they should not have forwarded oops.
2232 // However, we do need to "null" dead oops in the roots, if can not be done
2233 // in concurrent cycles.
2234 void ShenandoahHeap::stw_process_weak_roots(bool full_gc) {
2235   ShenandoahGCSubPhase root_phase(full_gc ?
2236                                   ShenandoahPhaseTimings::full_gc_purge :
2237                                   ShenandoahPhaseTimings::purge);
2238   uint num_workers = _workers->active_workers();
2239   ShenandoahPhaseTimings::Phase timing_phase = full_gc ?
2240                                                ShenandoahPhaseTimings::full_gc_purge_weak_par :
2241                                                ShenandoahPhaseTimings::purge_weak_par;
2242   ShenandoahGCSubPhase phase(timing_phase);
2243   ShenandoahGCWorkerPhase worker_phase(timing_phase);
2244 
2245   // Cleanup weak roots
2246   if (has_forwarded_objects()) {
2247     ShenandoahForwardedIsAliveClosure is_alive;
2248     ShenandoahUpdateRefsClosure keep_alive;
2249     ShenandoahParallelWeakRootsCleaningTask<ShenandoahForwardedIsAliveClosure, ShenandoahUpdateRefsClosure>
2250       cleaning_task(&is_alive, &keep_alive, num_workers, !ShenandoahConcurrentRoots::should_do_concurrent_class_unloading());
2251     _workers->run_task(&cleaning_task);
2252   } else {
2253     ShenandoahIsAliveClosure is_alive;
2254 #ifdef ASSERT
2255     ShenandoahAssertNotForwardedClosure verify_cl;
2256     ShenandoahParallelWeakRootsCleaningTask<ShenandoahIsAliveClosure, ShenandoahAssertNotForwardedClosure>
2257       cleaning_task(&is_alive, &verify_cl, num_workers, !ShenandoahConcurrentRoots::should_do_concurrent_class_unloading());
2258 #else
2259     ShenandoahParallelWeakRootsCleaningTask<ShenandoahIsAliveClosure, DoNothingClosure>
2260       cleaning_task(&is_alive, &do_nothing_cl, num_workers, !ShenandoahConcurrentRoots::should_do_concurrent_class_unloading());
2261 #endif
2262     _workers->run_task(&cleaning_task);
2263   }
2264 }
2265 
2266 void ShenandoahHeap::parallel_cleaning(bool full_gc) {
2267   assert(SafepointSynchronize::is_at_safepoint(), "Must be at a safepoint");
2268   stw_process_weak_roots(full_gc);
2269   if (!ShenandoahConcurrentRoots::should_do_concurrent_class_unloading()) {
2270     stw_unload_classes(full_gc);
2271   }
2272 }
2273 
2274 void ShenandoahHeap::set_has_forwarded_objects(bool cond) {
2275   set_gc_state_mask(HAS_FORWARDED, cond);
2276 }
2277 
2278 void ShenandoahHeap::set_process_references(bool pr) {
2279   _process_references.set_cond(pr);
2280 }
2281 
2282 void ShenandoahHeap::set_unload_classes(bool uc) {
2283   _unload_classes.set_cond(uc);
2284 }
2285 
2286 bool ShenandoahHeap::process_references() const {
2287   return _process_references.is_set();
2288 }
2289 
2290 bool ShenandoahHeap::unload_classes() const {
2291   return _unload_classes.is_set();
2292 }
2293 
2294 address ShenandoahHeap::in_cset_fast_test_addr() {
2295   ShenandoahHeap* heap = ShenandoahHeap::heap();
2296   assert(heap->collection_set() != NULL, "Sanity");
2297   return (address) heap->collection_set()->biased_map_address();
2298 }
2299 
2300 address ShenandoahHeap::cancelled_gc_addr() {
2301   return (address) ShenandoahHeap::heap()->_cancelled_gc.addr_of();
2302 }
2303 
2304 address ShenandoahHeap::gc_state_addr() {
2305   return (address) ShenandoahHeap::heap()->_gc_state.addr_of();
2306 }
2307 
2308 size_t ShenandoahHeap::bytes_allocated_since_gc_start() {
2309   return Atomic::load_acquire(&_bytes_allocated_since_gc_start);
2310 }
2311 
2312 void ShenandoahHeap::reset_bytes_allocated_since_gc_start() {
2313   Atomic::release_store_fence(&_bytes_allocated_since_gc_start, (size_t)0);
2314 }
2315 
2316 void ShenandoahHeap::set_degenerated_gc_in_progress(bool in_progress) {
2317   _degenerated_gc_in_progress.set_cond(in_progress);
2318 }
2319 
2320 void ShenandoahHeap::set_full_gc_in_progress(bool in_progress) {
2321   _full_gc_in_progress.set_cond(in_progress);
2322 }
2323 
2324 void ShenandoahHeap::set_full_gc_move_in_progress(bool in_progress) {
2325   assert (is_full_gc_in_progress(), "should be");
2326   _full_gc_move_in_progress.set_cond(in_progress);
2327 }
2328 
2329 void ShenandoahHeap::set_update_refs_in_progress(bool in_progress) {
2330   set_gc_state_mask(UPDATEREFS, in_progress);
2331 }
2332 
2333 void ShenandoahHeap::register_nmethod(nmethod* nm) {
2334   ShenandoahCodeRoots::register_nmethod(nm);
2335 }
2336 
2337 void ShenandoahHeap::unregister_nmethod(nmethod* nm) {
2338   ShenandoahCodeRoots::unregister_nmethod(nm);
2339 }
2340 
2341 void ShenandoahHeap::flush_nmethod(nmethod* nm) {
2342   ShenandoahCodeRoots::flush_nmethod(nm);
2343 }
2344 
2345 oop ShenandoahHeap::pin_object(JavaThread* thr, oop o) {
2346   heap_region_containing(o)->record_pin();
2347   return o;
2348 }
2349 
2350 void ShenandoahHeap::unpin_object(JavaThread* thr, oop o) {
2351   heap_region_containing(o)->record_unpin();
2352 }
2353 
2354 void ShenandoahHeap::sync_pinned_region_status() {
2355   ShenandoahHeapLocker locker(lock());
2356 
2357   for (size_t i = 0; i < num_regions(); i++) {
2358     ShenandoahHeapRegion *r = get_region(i);
2359     if (r->is_active()) {
2360       if (r->is_pinned()) {
2361         if (r->pin_count() == 0) {
2362           r->make_unpinned();
2363         }
2364       } else {
2365         if (r->pin_count() > 0) {
2366           r->make_pinned();
2367         }
2368       }
2369     }
2370   }
2371 
2372   assert_pinned_region_status();
2373 }
2374 
2375 #ifdef ASSERT
2376 void ShenandoahHeap::assert_pinned_region_status() {
2377   for (size_t i = 0; i < num_regions(); i++) {
2378     ShenandoahHeapRegion* r = get_region(i);
2379     assert((r->is_pinned() && r->pin_count() > 0) || (!r->is_pinned() && r->pin_count() == 0),
2380            "Region " SIZE_FORMAT " pinning status is inconsistent", i);
2381   }
2382 }
2383 #endif
2384 
2385 ConcurrentGCTimer* ShenandoahHeap::gc_timer() const {
2386   return _gc_timer;
2387 }
2388 
2389 void ShenandoahHeap::prepare_concurrent_roots() {
2390   assert(SafepointSynchronize::is_at_safepoint(), "Must be at a safepoint");
2391   if (ShenandoahConcurrentRoots::should_do_concurrent_roots()) {
2392     set_concurrent_strong_root_in_progress(!collection_set()->is_empty());
2393     set_concurrent_weak_root_in_progress(true);
2394   }
2395 }
2396 
2397 void ShenandoahHeap::prepare_concurrent_unloading() {
2398   assert(SafepointSynchronize::is_at_safepoint(), "Must be at a safepoint");
2399   if (ShenandoahConcurrentRoots::should_do_concurrent_class_unloading()) {
2400     _unloader.prepare();
2401   }
2402 }
2403 
2404 void ShenandoahHeap::finish_concurrent_unloading() {
2405   assert(SafepointSynchronize::is_at_safepoint(), "Must be at a safepoint");
2406   if (ShenandoahConcurrentRoots::should_do_concurrent_class_unloading()) {
2407     _unloader.finish();
2408   }
2409 }
2410 
2411 #ifdef ASSERT
2412 void ShenandoahHeap::assert_gc_workers(uint nworkers) {
2413   assert(nworkers > 0 && nworkers <= max_workers(), "Sanity");
2414 
2415   if (ShenandoahSafepoint::is_at_shenandoah_safepoint()) {
2416     if (UseDynamicNumberOfGCThreads) {
2417       assert(nworkers <= ParallelGCThreads, "Cannot use more than it has");
2418     } else {
2419       // Use ParallelGCThreads inside safepoints
2420       assert(nworkers == ParallelGCThreads, "Use ParallelGCThreads within safepoints");
2421     }
2422   } else {
2423     if (UseDynamicNumberOfGCThreads) {
2424       assert(nworkers <= ConcGCThreads, "Cannot use more than it has");
2425     } else {
2426       // Use ConcGCThreads outside safepoints
2427       assert(nworkers == ConcGCThreads, "Use ConcGCThreads outside safepoints");
2428     }
2429   }
2430 }
2431 #endif
2432 
2433 ShenandoahVerifier* ShenandoahHeap::verifier() {
2434   guarantee(ShenandoahVerify, "Should be enabled");
2435   assert (_verifier != NULL, "sanity");
2436   return _verifier;
2437 }
2438 
2439 template<class T>
2440 class ShenandoahUpdateHeapRefsTask : public AbstractGangTask {
2441 private:
2442   T cl;
2443   ShenandoahHeap* _heap;
2444   ShenandoahRegionIterator* _regions;
2445   bool _concurrent;
2446 public:
2447   ShenandoahUpdateHeapRefsTask(ShenandoahRegionIterator* regions, bool concurrent) :
2448     AbstractGangTask("Concurrent Update References Task"),
2449     cl(T()),
2450     _heap(ShenandoahHeap::heap()),
2451     _regions(regions),
2452     _concurrent(concurrent) {
2453   }
2454 
2455   void work(uint worker_id) {
2456     if (_concurrent) {
2457       ShenandoahConcurrentWorkerSession worker_session(worker_id);
2458       ShenandoahSuspendibleThreadSetJoiner stsj(ShenandoahSuspendibleWorkers);
2459       do_work();
2460     } else {
2461       ShenandoahParallelWorkerSession worker_session(worker_id);
2462       do_work();
2463     }
2464   }
2465 
2466 private:
2467   void do_work() {
2468     ShenandoahHeapRegion* r = _regions->next();
2469     ShenandoahMarkingContext* const ctx = _heap->complete_marking_context();
2470     while (r != NULL) {
2471       HeapWord* update_watermark = r->get_update_watermark();
2472       assert (update_watermark >= r->bottom(), "sanity");
2473       if (r->is_active() && !r->is_cset()) {
2474         _heap->marked_object_oop_iterate(r, &cl, update_watermark);
2475       }
2476       if (ShenandoahPacing) {
2477         _heap->pacer()->report_updaterefs(pointer_delta(update_watermark, r->bottom()));
2478       }
2479       if (_heap->check_cancelled_gc_and_yield(_concurrent)) {
2480         return;
2481       }
2482       r = _regions->next();
2483     }
2484   }
2485 };
2486 
2487 void ShenandoahHeap::update_heap_references(bool concurrent) {
2488   ShenandoahUpdateHeapRefsTask<ShenandoahUpdateHeapRefsClosure> task(&_update_refs_iterator, concurrent);
2489   workers()->run_task(&task);
2490 }
2491 
2492 void ShenandoahHeap::op_init_updaterefs() {
2493   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "must be at safepoint");
2494 
2495   set_evacuation_in_progress(false);
2496 
2497   {
2498     ShenandoahGCSubPhase phase(ShenandoahPhaseTimings::init_update_refs_retire_gclabs);
2499     retire_and_reset_gclabs();
2500   }
2501 
2502   if (ShenandoahVerify) {
2503     if (!is_degenerated_gc_in_progress()) {
2504       verifier()->verify_roots_in_to_space_except(ShenandoahRootVerifier::ThreadRoots);
2505     }
2506     verifier()->verify_before_updaterefs();
2507   }
2508 
2509   set_update_refs_in_progress(true);
2510 
2511   _update_refs_iterator.reset();
2512 
2513   if (ShenandoahPacing) {
2514     pacer()->setup_for_updaterefs();
2515   }
2516 }
2517 
2518 class ShenandoahFinalUpdateRefsUpdateRegionStateClosure : public ShenandoahHeapRegionClosure {
2519 private:
2520   ShenandoahHeapLock* const _lock;
2521 
2522 public:
2523   ShenandoahFinalUpdateRefsUpdateRegionStateClosure() : _lock(ShenandoahHeap::heap()->lock()) {}
2524 
2525   void heap_region_do(ShenandoahHeapRegion* r) {
2526     // Drop unnecessary "pinned" state from regions that does not have CP marks
2527     // anymore, as this would allow trashing them.
2528 
2529     if (r->is_active()) {
2530       if (r->is_pinned()) {
2531         if (r->pin_count() == 0) {
2532           ShenandoahHeapLocker locker(_lock);
2533           r->make_unpinned();
2534         }
2535       } else {
2536         if (r->pin_count() > 0) {
2537           ShenandoahHeapLocker locker(_lock);
2538           r->make_pinned();
2539         }
2540       }
2541     }
2542   }
2543 
2544   bool is_thread_safe() { return true; }
2545 };
2546 
2547 void ShenandoahHeap::op_final_updaterefs() {
2548   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "must be at safepoint");
2549 
2550   finish_concurrent_unloading();
2551 
2552   // Check if there is left-over work, and finish it
2553   if (_update_refs_iterator.has_next()) {
2554     ShenandoahGCSubPhase phase(ShenandoahPhaseTimings::final_update_refs_finish_work);
2555 
2556     // Finish updating references where we left off.
2557     clear_cancelled_gc();
2558     update_heap_references(false);
2559   }
2560 
2561   // Clear cancelled GC, if set. On cancellation path, the block before would handle
2562   // everything. On degenerated paths, cancelled gc would not be set anyway.
2563   if (cancelled_gc()) {
2564     clear_cancelled_gc();
2565   }
2566   assert(!cancelled_gc(), "Should have been done right before");
2567 
2568   if (ShenandoahVerify && !is_degenerated_gc_in_progress()) {
2569     verifier()->verify_roots_in_to_space_except(ShenandoahRootVerifier::ThreadRoots);
2570   }
2571 
2572   if (is_degenerated_gc_in_progress()) {
2573     concurrent_mark()->update_roots(ShenandoahPhaseTimings::degen_gc_update_roots);
2574   } else {
2575     concurrent_mark()->update_thread_roots(ShenandoahPhaseTimings::final_update_refs_roots);
2576   }
2577 
2578   // Has to be done before cset is clear
2579   if (ShenandoahVerify) {
2580     verifier()->verify_roots_in_to_space();
2581   }
2582 
2583   {
2584     ShenandoahGCSubPhase phase(ShenandoahPhaseTimings::final_update_refs_update_region_states);
2585     ShenandoahFinalUpdateRefsUpdateRegionStateClosure cl;
2586     parallel_heap_region_iterate(&cl);
2587 
2588     assert_pinned_region_status();
2589   }
2590 
2591   {
2592     ShenandoahGCSubPhase phase(ShenandoahPhaseTimings::final_update_refs_trash_cset);
2593     trash_cset_regions();
2594   }
2595 
2596   set_has_forwarded_objects(false);
2597   set_update_refs_in_progress(false);
2598 
2599   if (ShenandoahVerify) {
2600     verifier()->verify_after_updaterefs();
2601   }
2602 
2603   if (VerifyAfterGC) {
2604     Universe::verify();
2605   }
2606 
2607   {
2608     ShenandoahGCSubPhase phase(ShenandoahPhaseTimings::final_update_refs_rebuild_freeset);
2609     ShenandoahHeapLocker locker(lock());
2610     _free_set->rebuild();
2611   }
2612 }
2613 
2614 void ShenandoahHeap::print_extended_on(outputStream *st) const {
2615   print_on(st);
2616   print_heap_regions_on(st);
2617 }
2618 
2619 bool ShenandoahHeap::is_bitmap_slice_committed(ShenandoahHeapRegion* r, bool skip_self) {
2620   size_t slice = r->index() / _bitmap_regions_per_slice;
2621 
2622   size_t regions_from = _bitmap_regions_per_slice * slice;
2623   size_t regions_to   = MIN2(num_regions(), _bitmap_regions_per_slice * (slice + 1));
2624   for (size_t g = regions_from; g < regions_to; g++) {
2625     assert (g / _bitmap_regions_per_slice == slice, "same slice");
2626     if (skip_self && g == r->index()) continue;
2627     if (get_region(g)->is_committed()) {
2628       return true;
2629     }
2630   }
2631   return false;
2632 }
2633 
2634 bool ShenandoahHeap::commit_bitmap_slice(ShenandoahHeapRegion* r) {
2635   shenandoah_assert_heaplocked();
2636 
2637   // Bitmaps in special regions do not need commits
2638   if (_bitmap_region_special) {
2639     return true;
2640   }
2641 
2642   if (is_bitmap_slice_committed(r, true)) {
2643     // Some other region from the group is already committed, meaning the bitmap
2644     // slice is already committed, we exit right away.
2645     return true;
2646   }
2647 
2648   // Commit the bitmap slice:
2649   size_t slice = r->index() / _bitmap_regions_per_slice;
2650   size_t off = _bitmap_bytes_per_slice * slice;
2651   size_t len = _bitmap_bytes_per_slice;
2652   if (!os::commit_memory((char*)_bitmap_region.start() + off, len, false)) {
2653     return false;
2654   }
2655   return true;
2656 }
2657 
2658 bool ShenandoahHeap::uncommit_bitmap_slice(ShenandoahHeapRegion *r) {
2659   shenandoah_assert_heaplocked();
2660 
2661   // Bitmaps in special regions do not need uncommits
2662   if (_bitmap_region_special) {
2663     return true;
2664   }
2665 
2666   if (is_bitmap_slice_committed(r, true)) {
2667     // Some other region from the group is still committed, meaning the bitmap
2668     // slice is should stay committed, exit right away.
2669     return true;
2670   }
2671 
2672   // Uncommit the bitmap slice:
2673   size_t slice = r->index() / _bitmap_regions_per_slice;
2674   size_t off = _bitmap_bytes_per_slice * slice;
2675   size_t len = _bitmap_bytes_per_slice;
2676   if (!os::uncommit_memory((char*)_bitmap_region.start() + off, len)) {
2677     return false;
2678   }
2679   return true;
2680 }
2681 
2682 void ShenandoahHeap::safepoint_synchronize_begin() {
2683   if (ShenandoahSuspendibleWorkers || UseStringDeduplication) {
2684     SuspendibleThreadSet::synchronize();
2685   }
2686 }
2687 
2688 void ShenandoahHeap::safepoint_synchronize_end() {
2689   if (ShenandoahSuspendibleWorkers || UseStringDeduplication) {
2690     SuspendibleThreadSet::desynchronize();
2691   }
2692 }
2693 
2694 void ShenandoahHeap::vmop_entry_init_mark() {
2695   TraceCollectorStats tcs(monitoring_support()->stw_collection_counters());
2696   ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_mark_gross);
2697 
2698   try_inject_alloc_failure();
2699   VM_ShenandoahInitMark op;
2700   VMThread::execute(&op); // jump to entry_init_mark() under safepoint
2701 }
2702 
2703 void ShenandoahHeap::vmop_entry_final_mark() {
2704   TraceCollectorStats tcs(monitoring_support()->stw_collection_counters());
2705   ShenandoahGCPhase phase(ShenandoahPhaseTimings::final_mark_gross);
2706 
2707   try_inject_alloc_failure();
2708   VM_ShenandoahFinalMarkStartEvac op;
2709   VMThread::execute(&op); // jump to entry_final_mark under safepoint
2710 }
2711 
2712 void ShenandoahHeap::vmop_entry_init_updaterefs() {
2713   TraceCollectorStats tcs(monitoring_support()->stw_collection_counters());
2714   ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_update_refs_gross);
2715 
2716   try_inject_alloc_failure();
2717   VM_ShenandoahInitUpdateRefs op;
2718   VMThread::execute(&op);
2719 }
2720 
2721 void ShenandoahHeap::vmop_entry_final_updaterefs() {
2722   TraceCollectorStats tcs(monitoring_support()->stw_collection_counters());
2723   ShenandoahGCPhase phase(ShenandoahPhaseTimings::final_update_refs_gross);
2724 
2725   try_inject_alloc_failure();
2726   VM_ShenandoahFinalUpdateRefs op;
2727   VMThread::execute(&op);
2728 }
2729 
2730 void ShenandoahHeap::vmop_entry_full(GCCause::Cause cause) {
2731   TraceCollectorStats tcs(monitoring_support()->full_stw_collection_counters());
2732   ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_gross);
2733 
2734   try_inject_alloc_failure();
2735   VM_ShenandoahFullGC op(cause);
2736   VMThread::execute(&op);
2737 }
2738 
2739 void ShenandoahHeap::vmop_degenerated(ShenandoahDegenPoint point) {
2740   TraceCollectorStats tcs(monitoring_support()->full_stw_collection_counters());
2741   ShenandoahGCPhase phase(ShenandoahPhaseTimings::degen_gc_gross);
2742 
2743   VM_ShenandoahDegeneratedGC degenerated_gc((int)point);
2744   VMThread::execute(&degenerated_gc);
2745 }
2746 
2747 void ShenandoahHeap::entry_init_mark() {
2748   const char* msg = init_mark_event_message();
2749   ShenandoahPausePhase gc_phase(msg);
2750   EventMark em("%s", msg);
2751 
2752   ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_mark);
2753 
2754   ShenandoahWorkerScope scope(workers(),
2755                               ShenandoahWorkerPolicy::calc_workers_for_init_marking(),
2756                               "init marking");
2757 
2758   op_init_mark();
2759 }
2760 
2761 void ShenandoahHeap::entry_final_mark() {
2762   const char* msg = final_mark_event_message();
2763   ShenandoahPausePhase gc_phase(msg);
2764   EventMark em("%s", msg);
2765 
2766   ShenandoahGCPhase phase(ShenandoahPhaseTimings::final_mark);
2767 
2768   ShenandoahWorkerScope scope(workers(),
2769                               ShenandoahWorkerPolicy::calc_workers_for_final_marking(),
2770                               "final marking");
2771 
2772   op_final_mark();
2773 }
2774 
2775 void ShenandoahHeap::entry_init_updaterefs() {
2776   static const char* msg = "Pause Init Update Refs";
2777   ShenandoahPausePhase gc_phase(msg);
2778   EventMark em("%s", msg);
2779 
2780   ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_update_refs);
2781 
2782   // No workers used in this phase, no setup required
2783 
2784   op_init_updaterefs();
2785 }
2786 
2787 void ShenandoahHeap::entry_final_updaterefs() {
2788   static const char* msg = "Pause Final Update Refs";
2789   ShenandoahPausePhase gc_phase(msg);
2790   EventMark em("%s", msg);
2791 
2792   ShenandoahGCPhase phase(ShenandoahPhaseTimings::final_update_refs);
2793 
2794   ShenandoahWorkerScope scope(workers(),
2795                               ShenandoahWorkerPolicy::calc_workers_for_final_update_ref(),
2796                               "final reference update");
2797 
2798   op_final_updaterefs();
2799 }
2800 
2801 void ShenandoahHeap::entry_full(GCCause::Cause cause) {
2802   static const char* msg = "Pause Full";
2803   ShenandoahPausePhase gc_phase(msg, true /* log_heap_usage */);
2804   EventMark em("%s", msg);
2805 
2806   ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc);
2807 
2808   ShenandoahWorkerScope scope(workers(),
2809                               ShenandoahWorkerPolicy::calc_workers_for_fullgc(),
2810                               "full gc");
2811 
2812   op_full(cause);
2813 }
2814 
2815 void ShenandoahHeap::entry_degenerated(int point) {
2816   ShenandoahDegenPoint dpoint = (ShenandoahDegenPoint)point;
2817   const char* msg = degen_event_message(dpoint);
2818   ShenandoahPausePhase gc_phase(msg, true /* log_heap_usage */);
2819   EventMark em("%s", msg);
2820 
2821   ShenandoahGCPhase phase(ShenandoahPhaseTimings::degen_gc);
2822 
2823   ShenandoahWorkerScope scope(workers(),
2824                               ShenandoahWorkerPolicy::calc_workers_for_stw_degenerated(),
2825                               "stw degenerated gc");
2826 
2827   set_degenerated_gc_in_progress(true);
2828   op_degenerated(dpoint);
2829   set_degenerated_gc_in_progress(false);
2830 }
2831 
2832 void ShenandoahHeap::entry_mark() {
2833   TraceCollectorStats tcs(monitoring_support()->concurrent_collection_counters());
2834 
2835   const char* msg = conc_mark_event_message();
2836   ShenandoahConcurrentPhase gc_phase(msg);
2837   EventMark em("%s", msg);
2838 
2839   ShenandoahGCPhase conc_mark_phase(ShenandoahPhaseTimings::conc_mark);
2840 
2841   ShenandoahWorkerScope scope(workers(),
2842                               ShenandoahWorkerPolicy::calc_workers_for_conc_marking(),
2843                               "concurrent marking");
2844 
2845   try_inject_alloc_failure();
2846   op_mark();
2847 }
2848 
2849 void ShenandoahHeap::entry_evac() {
2850   TraceCollectorStats tcs(monitoring_support()->concurrent_collection_counters());
2851 
2852   static const char* msg = "Concurrent evacuation";
2853   ShenandoahConcurrentPhase gc_phase(msg);
2854   EventMark em("%s", msg);
2855 
2856   ShenandoahGCPhase conc_evac_phase(ShenandoahPhaseTimings::conc_evac);
2857 
2858   ShenandoahWorkerScope scope(workers(),
2859                               ShenandoahWorkerPolicy::calc_workers_for_conc_evac(),
2860                               "concurrent evacuation");
2861 
2862   try_inject_alloc_failure();
2863   op_conc_evac();
2864 }
2865 
2866 void ShenandoahHeap::entry_updaterefs() {
2867   static const char* msg = "Concurrent update references";
2868   ShenandoahConcurrentPhase gc_phase(msg);
2869   EventMark em("%s", msg);
2870 
2871   ShenandoahGCPhase phase(ShenandoahPhaseTimings::conc_update_refs);
2872 
2873   ShenandoahWorkerScope scope(workers(),
2874                               ShenandoahWorkerPolicy::calc_workers_for_conc_update_ref(),
2875                               "concurrent reference update");
2876 
2877   try_inject_alloc_failure();
2878   op_updaterefs();
2879 }
2880 
2881 void ShenandoahHeap::entry_weak_roots() {
2882   static const char* msg = "Concurrent weak roots";
2883   ShenandoahConcurrentPhase gc_phase(msg);
2884   EventMark em("%s", msg);
2885 
2886   ShenandoahGCPhase phase(ShenandoahPhaseTimings::conc_weak_roots);
2887 
2888   ShenandoahWorkerScope scope(workers(),
2889                               ShenandoahWorkerPolicy::calc_workers_for_conc_root_processing(),
2890                               "concurrent weak root");
2891 
2892   try_inject_alloc_failure();
2893   op_weak_roots();
2894 }
2895 
2896 void ShenandoahHeap::entry_class_unloading() {
2897   static const char* msg = "Concurrent class unloading";
2898   ShenandoahConcurrentPhase gc_phase(msg);
2899   EventMark em("%s", msg);
2900 
2901   ShenandoahGCPhase phase(ShenandoahPhaseTimings::conc_class_unloading);
2902 
2903   ShenandoahWorkerScope scope(workers(),
2904                               ShenandoahWorkerPolicy::calc_workers_for_conc_root_processing(),
2905                               "concurrent class unloading");
2906 
2907   try_inject_alloc_failure();
2908   op_class_unloading();
2909 }
2910 
2911 void ShenandoahHeap::entry_strong_roots() {
2912   static const char* msg = "Concurrent strong roots";
2913   ShenandoahConcurrentPhase gc_phase(msg);
2914   EventMark em("%s", msg);
2915 
2916   ShenandoahGCPhase phase(ShenandoahPhaseTimings::conc_strong_roots);
2917 
2918   ShenandoahWorkerScope scope(workers(),
2919                               ShenandoahWorkerPolicy::calc_workers_for_conc_root_processing(),
2920                               "concurrent strong root");
2921 
2922   try_inject_alloc_failure();
2923   op_strong_roots();
2924 }
2925 
2926 void ShenandoahHeap::entry_cleanup_early() {
2927   static const char* msg = "Concurrent cleanup";
2928   ShenandoahConcurrentPhase gc_phase(msg,  true /* log_heap_usage */);
2929   EventMark em("%s", msg);
2930 
2931   ShenandoahGCSubPhase phase(ShenandoahPhaseTimings::conc_cleanup_early);
2932 
2933   // This phase does not use workers, no need for setup
2934 
2935   try_inject_alloc_failure();
2936   op_cleanup_early();
2937 }
2938 
2939 void ShenandoahHeap::entry_cleanup_complete() {
2940   static const char* msg = "Concurrent cleanup";
2941   ShenandoahConcurrentPhase gc_phase(msg,  true /* log_heap_usage */);
2942   EventMark em("%s", msg);
2943 
2944   ShenandoahGCSubPhase phase(ShenandoahPhaseTimings::conc_cleanup_complete);
2945 
2946   // This phase does not use workers, no need for setup
2947 
2948   try_inject_alloc_failure();
2949   op_cleanup_complete();
2950 }
2951 
2952 void ShenandoahHeap::entry_reset() {
2953   static const char* msg = "Concurrent reset";
2954   ShenandoahConcurrentPhase gc_phase(msg);
2955   EventMark em("%s", msg);
2956 
2957   ShenandoahGCSubPhase phase(ShenandoahPhaseTimings::conc_reset);
2958 
2959   ShenandoahWorkerScope scope(workers(),
2960                               ShenandoahWorkerPolicy::calc_workers_for_conc_reset(),
2961                               "concurrent reset");
2962 
2963   try_inject_alloc_failure();
2964   op_reset();
2965 }
2966 
2967 void ShenandoahHeap::entry_preclean() {
2968   if (ShenandoahPreclean && process_references()) {
2969     static const char* msg = "Concurrent precleaning";
2970     ShenandoahConcurrentPhase gc_phase(msg);
2971     EventMark em("%s", msg);
2972 
2973     ShenandoahGCSubPhase conc_preclean(ShenandoahPhaseTimings::conc_preclean);
2974 
2975     ShenandoahWorkerScope scope(workers(),
2976                                 ShenandoahWorkerPolicy::calc_workers_for_conc_preclean(),
2977                                 "concurrent preclean",
2978                                 /* check_workers = */ false);
2979 
2980     try_inject_alloc_failure();
2981     op_preclean();
2982   }
2983 }
2984 
2985 void ShenandoahHeap::entry_uncommit(double shrink_before) {
2986   static const char *msg = "Concurrent uncommit";
2987   ShenandoahConcurrentPhase gc_phase(msg, true /* log_heap_usage */);
2988   EventMark em("%s", msg);
2989 
2990   ShenandoahGCSubPhase phase(ShenandoahPhaseTimings::conc_uncommit);
2991 
2992   op_uncommit(shrink_before);
2993 }
2994 
2995 void ShenandoahHeap::try_inject_alloc_failure() {
2996   if (ShenandoahAllocFailureALot && !cancelled_gc() && ((os::random() % 1000) > 950)) {
2997     _inject_alloc_failure.set();
2998     os::naked_short_sleep(1);
2999     if (cancelled_gc()) {
3000       log_info(gc)("Allocation failure was successfully injected");
3001     }
3002   }
3003 }
3004 
3005 bool ShenandoahHeap::should_inject_alloc_failure() {
3006   return _inject_alloc_failure.is_set() && _inject_alloc_failure.try_unset();
3007 }
3008 
3009 void ShenandoahHeap::initialize_serviceability() {
3010   _memory_pool = new ShenandoahMemoryPool(this);
3011   _cycle_memory_manager.add_pool(_memory_pool);
3012   _stw_memory_manager.add_pool(_memory_pool);
3013 }
3014 
3015 GrowableArray<GCMemoryManager*> ShenandoahHeap::memory_managers() {
3016   GrowableArray<GCMemoryManager*> memory_managers(2);
3017   memory_managers.append(&_cycle_memory_manager);
3018   memory_managers.append(&_stw_memory_manager);
3019   return memory_managers;
3020 }
3021 
3022 GrowableArray<MemoryPool*> ShenandoahHeap::memory_pools() {
3023   GrowableArray<MemoryPool*> memory_pools(1);
3024   memory_pools.append(_memory_pool);
3025   return memory_pools;
3026 }
3027 
3028 MemoryUsage ShenandoahHeap::memory_usage() {
3029   return _memory_pool->get_memory_usage();
3030 }
3031 
3032 void ShenandoahHeap::enter_evacuation() {
3033   _oom_evac_handler.enter_evacuation();
3034 }
3035 
3036 void ShenandoahHeap::leave_evacuation() {
3037   _oom_evac_handler.leave_evacuation();
3038 }
3039 
3040 ShenandoahRegionIterator::ShenandoahRegionIterator() :
3041   _heap(ShenandoahHeap::heap()),
3042   _index(0) {}
3043 
3044 ShenandoahRegionIterator::ShenandoahRegionIterator(ShenandoahHeap* heap) :
3045   _heap(heap),
3046   _index(0) {}
3047 
3048 void ShenandoahRegionIterator::reset() {
3049   _index = 0;
3050 }
3051 
3052 bool ShenandoahRegionIterator::has_next() const {
3053   return _index < _heap->num_regions();
3054 }
3055 
3056 char ShenandoahHeap::gc_state() const {
3057   return _gc_state.raw_value();
3058 }
3059 
3060 void ShenandoahHeap::deduplicate_string(oop str) {
3061   assert(java_lang_String::is_instance(str), "invariant");
3062 
3063   if (ShenandoahStringDedup::is_enabled()) {
3064     ShenandoahStringDedup::deduplicate(str);
3065   }
3066 }
3067 
3068 const char* ShenandoahHeap::init_mark_event_message() const {
3069   assert(!has_forwarded_objects(), "Should not have forwarded objects here");
3070 
3071   bool proc_refs = process_references();
3072   bool unload_cls = unload_classes();
3073 
3074   if (proc_refs && unload_cls) {
3075     return "Pause Init Mark (process weakrefs) (unload classes)";
3076   } else if (proc_refs) {
3077     return "Pause Init Mark (process weakrefs)";
3078   } else if (unload_cls) {
3079     return "Pause Init Mark (unload classes)";
3080   } else {
3081     return "Pause Init Mark";
3082   }
3083 }
3084 
3085 const char* ShenandoahHeap::final_mark_event_message() const {
3086   assert(!has_forwarded_objects(), "Should not have forwarded objects here");
3087 
3088   bool proc_refs = process_references();
3089   bool unload_cls = unload_classes();
3090 
3091   if (proc_refs && unload_cls) {
3092     return "Pause Final Mark (process weakrefs) (unload classes)";
3093   } else if (proc_refs) {
3094     return "Pause Final Mark (process weakrefs)";
3095   } else if (unload_cls) {
3096     return "Pause Final Mark (unload classes)";
3097   } else {
3098     return "Pause Final Mark";
3099   }
3100 }
3101 
3102 const char* ShenandoahHeap::conc_mark_event_message() const {
3103   assert(!has_forwarded_objects(), "Should not have forwarded objects here");
3104 
3105   bool proc_refs = process_references();
3106   bool unload_cls = unload_classes();
3107 
3108   if (proc_refs && unload_cls) {
3109     return "Concurrent marking (process weakrefs) (unload classes)";
3110   } else if (proc_refs) {
3111     return "Concurrent marking (process weakrefs)";
3112   } else if (unload_cls) {
3113     return "Concurrent marking (unload classes)";
3114   } else {
3115     return "Concurrent marking";
3116   }
3117 }
3118 
3119 const char* ShenandoahHeap::degen_event_message(ShenandoahDegenPoint point) const {
3120   switch (point) {
3121     case _degenerated_unset:
3122       return "Pause Degenerated GC (<UNSET>)";
3123     case _degenerated_outside_cycle:
3124       return "Pause Degenerated GC (Outside of Cycle)";
3125     case _degenerated_mark:
3126       return "Pause Degenerated GC (Mark)";
3127     case _degenerated_evac:
3128       return "Pause Degenerated GC (Evacuation)";
3129     case _degenerated_updaterefs:
3130       return "Pause Degenerated GC (Update Refs)";
3131     default:
3132       ShouldNotReachHere();
3133       return "ERROR";
3134   }
3135 }
3136 
3137 ShenandoahLiveData* ShenandoahHeap::get_liveness_cache(uint worker_id) {
3138 #ifdef ASSERT
3139   assert(_liveness_cache != NULL, "sanity");
3140   assert(worker_id < _max_workers, "sanity");
3141   for (uint i = 0; i < num_regions(); i++) {
3142     assert(_liveness_cache[worker_id][i] == 0, "liveness cache should be empty");
3143   }
3144 #endif
3145   return _liveness_cache[worker_id];
3146 }
3147 
3148 void ShenandoahHeap::flush_liveness_cache(uint worker_id) {
3149   assert(worker_id < _max_workers, "sanity");
3150   assert(_liveness_cache != NULL, "sanity");
3151   ShenandoahLiveData* ld = _liveness_cache[worker_id];
3152   for (uint i = 0; i < num_regions(); i++) {
3153     ShenandoahLiveData live = ld[i];
3154     if (live > 0) {
3155       ShenandoahHeapRegion* r = get_region(i);
3156       r->increase_live_data_gc_words(live);
3157       ld[i] = 0;
3158     }
3159   }
3160 }