1 /*
   2  * Copyright (c) 2013, 2020, Red Hat, Inc. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "memory/allocation.hpp"
  27 #include "memory/universe.hpp"
  28 
  29 #include "gc/shared/gcArguments.hpp"
  30 #include "gc/shared/gcTimer.hpp"
  31 #include "gc/shared/gcTraceTime.inline.hpp"
  32 #include "gc/shared/locationPrinter.inline.hpp"
  33 #include "gc/shared/memAllocator.hpp"
  34 #include "gc/shared/oopStorageSet.hpp"
  35 #include "gc/shared/plab.hpp"
  36 
  37 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
  38 #include "gc/shenandoah/shenandoahClosures.inline.hpp"
  39 #include "gc/shenandoah/shenandoahCollectionSet.hpp"
  40 #include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
  41 #include "gc/shenandoah/shenandoahConcurrentMark.inline.hpp"
  42 #include "gc/shenandoah/shenandoahConcurrentRoots.hpp"
  43 #include "gc/shenandoah/shenandoahControlThread.hpp"
  44 #include "gc/shenandoah/shenandoahFreeSet.hpp"
  45 #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
  46 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
  47 #include "gc/shenandoah/shenandoahHeapRegion.inline.hpp"
  48 #include "gc/shenandoah/shenandoahHeapRegionSet.hpp"
  49 #include "gc/shenandoah/shenandoahMarkCompact.hpp"
  50 #include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"
  51 #include "gc/shenandoah/shenandoahMemoryPool.hpp"
  52 #include "gc/shenandoah/shenandoahMetrics.hpp"
  53 #include "gc/shenandoah/shenandoahMonitoringSupport.hpp"
  54 #include "gc/shenandoah/shenandoahOopClosures.inline.hpp"
  55 #include "gc/shenandoah/shenandoahPacer.inline.hpp"
  56 #include "gc/shenandoah/shenandoahPadding.hpp"
  57 #include "gc/shenandoah/shenandoahParallelCleaning.inline.hpp"
  58 #include "gc/shenandoah/shenandoahRootProcessor.inline.hpp"
  59 #include "gc/shenandoah/shenandoahStringDedup.hpp"
  60 #include "gc/shenandoah/shenandoahTaskqueue.hpp"
  61 #include "gc/shenandoah/shenandoahUtils.hpp"
  62 #include "gc/shenandoah/shenandoahVerifier.hpp"
  63 #include "gc/shenandoah/shenandoahCodeRoots.hpp"
  64 #include "gc/shenandoah/shenandoahVMOperations.hpp"
  65 #include "gc/shenandoah/shenandoahWorkGroup.hpp"
  66 #include "gc/shenandoah/shenandoahWorkerPolicy.hpp"
  67 #include "gc/shenandoah/mode/shenandoahIUMode.hpp"
  68 #include "gc/shenandoah/mode/shenandoahPassiveMode.hpp"
  69 #include "gc/shenandoah/mode/shenandoahSATBMode.hpp"
  70 #if INCLUDE_JFR
  71 #include "gc/shenandoah/shenandoahJfrSupport.hpp"
  72 #endif
  73 
  74 #include "memory/metaspace.hpp"
  75 #include "oops/compressedOops.inline.hpp"
  76 #include "runtime/atomic.hpp"
  77 #include "runtime/globals.hpp"
  78 #include "runtime/interfaceSupport.inline.hpp"
  79 #include "runtime/orderAccess.hpp"
  80 #include "runtime/safepointMechanism.hpp"
  81 #include "runtime/vmThread.hpp"
  82 #include "services/mallocTracker.hpp"
  83 #include "utilities/powerOfTwo.hpp"
  84 
  85 ShenandoahHeap* ShenandoahHeap::_heap = NULL;
  86 
  87 #ifdef ASSERT
  88 template <class T>
  89 void ShenandoahAssertToSpaceClosure::do_oop_work(T* p) {
  90   T o = RawAccess<>::oop_load(p);
  91   if (! CompressedOops::is_null(o)) {
  92     oop obj = CompressedOops::decode_not_null(o);
  93     shenandoah_assert_not_forwarded(p, obj);
  94   }
  95 }
  96 
  97 void ShenandoahAssertToSpaceClosure::do_oop(narrowOop* p) { do_oop_work(p); }
  98 void ShenandoahAssertToSpaceClosure::do_oop(oop* p)       { do_oop_work(p); }
  99 #endif
 100 
 101 class ShenandoahPretouchHeapTask : public AbstractGangTask {
 102 private:
 103   ShenandoahRegionIterator _regions;
 104   const size_t _page_size;
 105 public:
 106   ShenandoahPretouchHeapTask(size_t page_size) :
 107     AbstractGangTask("Shenandoah Pretouch Heap"),
 108     _page_size(page_size) {}
 109 
 110   virtual void work(uint worker_id) {
 111     ShenandoahHeapRegion* r = _regions.next();
 112     while (r != NULL) {
 113       os::pretouch_memory(r->bottom(), r->end(), _page_size);
 114       r = _regions.next();
 115     }
 116   }
 117 };
 118 
 119 class ShenandoahPretouchBitmapTask : public AbstractGangTask {
 120 private:
 121   ShenandoahRegionIterator _regions;
 122   char* _bitmap_base;
 123   const size_t _bitmap_size;
 124   const size_t _page_size;
 125 public:
 126   ShenandoahPretouchBitmapTask(char* bitmap_base, size_t bitmap_size, size_t page_size) :
 127     AbstractGangTask("Shenandoah Pretouch Bitmap"),
 128     _bitmap_base(bitmap_base),
 129     _bitmap_size(bitmap_size),
 130     _page_size(page_size) {}
 131 
 132   virtual void work(uint worker_id) {
 133     ShenandoahHeapRegion* r = _regions.next();
 134     while (r != NULL) {
 135       size_t start = r->index()       * ShenandoahHeapRegion::region_size_bytes() / MarkBitMap::heap_map_factor();
 136       size_t end   = (r->index() + 1) * ShenandoahHeapRegion::region_size_bytes() / MarkBitMap::heap_map_factor();
 137       assert (end <= _bitmap_size, "end is sane: " SIZE_FORMAT " < " SIZE_FORMAT, end, _bitmap_size);
 138 
 139       os::pretouch_memory(_bitmap_base + start, _bitmap_base + end, _page_size);
 140 
 141       r = _regions.next();
 142     }
 143   }
 144 };
 145 
 146 jint ShenandoahHeap::initialize() {
 147   //
 148   // Figure out heap sizing
 149   //
 150 
 151   size_t init_byte_size = InitialHeapSize;
 152   size_t min_byte_size  = MinHeapSize;
 153   size_t max_byte_size  = MaxHeapSize;
 154   size_t heap_alignment = HeapAlignment;
 155 
 156   size_t reg_size_bytes = ShenandoahHeapRegion::region_size_bytes();
 157 
 158   if (ShenandoahAlwaysPreTouch) {
 159     // Enabled pre-touch means the entire heap is committed right away.
 160     init_byte_size = max_byte_size;
 161   }
 162 
 163   Universe::check_alignment(max_byte_size,  reg_size_bytes, "Shenandoah heap");
 164   Universe::check_alignment(init_byte_size, reg_size_bytes, "Shenandoah heap");
 165 
 166   _num_regions = ShenandoahHeapRegion::region_count();
 167 
 168   // Now we know the number of regions, initialize the heuristics.
 169   initialize_heuristics();
 170 
 171   size_t num_committed_regions = init_byte_size / reg_size_bytes;
 172   num_committed_regions = MIN2(num_committed_regions, _num_regions);
 173   assert(num_committed_regions <= _num_regions, "sanity");
 174   _initial_size = num_committed_regions * reg_size_bytes;
 175 
 176   size_t num_min_regions = min_byte_size / reg_size_bytes;
 177   num_min_regions = MIN2(num_min_regions, _num_regions);
 178   assert(num_min_regions <= _num_regions, "sanity");
 179   _minimum_size = num_min_regions * reg_size_bytes;
 180 
 181   _committed = _initial_size;
 182 
 183   size_t heap_page_size   = UseLargePages ? (size_t)os::large_page_size() : (size_t)os::vm_page_size();
 184   size_t bitmap_page_size = UseLargePages ? (size_t)os::large_page_size() : (size_t)os::vm_page_size();
 185   size_t region_page_size = UseLargePages ? (size_t)os::large_page_size() : (size_t)os::vm_page_size();
 186 
 187   //
 188   // Reserve and commit memory for heap
 189   //
 190 
 191   ReservedHeapSpace heap_rs = Universe::reserve_heap(max_byte_size, heap_alignment);
 192   initialize_reserved_region(heap_rs);
 193   _heap_region = MemRegion((HeapWord*)heap_rs.base(), heap_rs.size() / HeapWordSize);
 194   _heap_region_special = heap_rs.special();
 195 
 196   assert((((size_t) base()) & ShenandoahHeapRegion::region_size_bytes_mask()) == 0,
 197          "Misaligned heap: " PTR_FORMAT, p2i(base()));
 198 
 199 #if SHENANDOAH_OPTIMIZED_OBJTASK
 200   // The optimized ObjArrayChunkedTask takes some bits away from the full object bits.
 201   // Fail if we ever attempt to address more than we can.
 202   if ((uintptr_t)heap_rs.end() >= ObjArrayChunkedTask::max_addressable()) {
 203     FormatBuffer<512> buf("Shenandoah reserved [" PTR_FORMAT ", " PTR_FORMAT") for the heap, \n"
 204                           "but max object address is " PTR_FORMAT ". Try to reduce heap size, or try other \n"
 205                           "VM options that allocate heap at lower addresses (HeapBaseMinAddress, AllocateHeapAt, etc).",
 206                 p2i(heap_rs.base()), p2i(heap_rs.end()), ObjArrayChunkedTask::max_addressable());
 207     vm_exit_during_initialization("Fatal Error", buf);
 208   }
 209 #endif
 210 
 211   ReservedSpace sh_rs = heap_rs.first_part(max_byte_size);
 212   if (!_heap_region_special) {
 213     os::commit_memory_or_exit(sh_rs.base(), _initial_size, heap_alignment, false,
 214                               "Cannot commit heap memory");
 215   }
 216 
 217   //
 218   // Reserve and commit memory for bitmap(s)
 219   //
 220 
 221   _bitmap_size = MarkBitMap::compute_size(heap_rs.size());
 222   _bitmap_size = align_up(_bitmap_size, bitmap_page_size);
 223 
 224   size_t bitmap_bytes_per_region = reg_size_bytes / MarkBitMap::heap_map_factor();
 225 
 226   guarantee(bitmap_bytes_per_region != 0,
 227             "Bitmap bytes per region should not be zero");
 228   guarantee(is_power_of_2(bitmap_bytes_per_region),
 229             "Bitmap bytes per region should be power of two: " SIZE_FORMAT, bitmap_bytes_per_region);
 230 
 231   if (bitmap_page_size > bitmap_bytes_per_region) {
 232     _bitmap_regions_per_slice = bitmap_page_size / bitmap_bytes_per_region;
 233     _bitmap_bytes_per_slice = bitmap_page_size;
 234   } else {
 235     _bitmap_regions_per_slice = 1;
 236     _bitmap_bytes_per_slice = bitmap_bytes_per_region;
 237   }
 238 
 239   guarantee(_bitmap_regions_per_slice >= 1,
 240             "Should have at least one region per slice: " SIZE_FORMAT,
 241             _bitmap_regions_per_slice);
 242 
 243   guarantee(((_bitmap_bytes_per_slice) % bitmap_page_size) == 0,
 244             "Bitmap slices should be page-granular: bps = " SIZE_FORMAT ", page size = " SIZE_FORMAT,
 245             _bitmap_bytes_per_slice, bitmap_page_size);
 246 
 247   ReservedSpace bitmap(_bitmap_size, bitmap_page_size);
 248   MemTracker::record_virtual_memory_type(bitmap.base(), mtGC);
 249   _bitmap_region = MemRegion((HeapWord*) bitmap.base(), bitmap.size() / HeapWordSize);
 250   _bitmap_region_special = bitmap.special();
 251 
 252   size_t bitmap_init_commit = _bitmap_bytes_per_slice *
 253                               align_up(num_committed_regions, _bitmap_regions_per_slice) / _bitmap_regions_per_slice;
 254   bitmap_init_commit = MIN2(_bitmap_size, bitmap_init_commit);
 255   if (!_bitmap_region_special) {
 256     os::commit_memory_or_exit((char *) _bitmap_region.start(), bitmap_init_commit, bitmap_page_size, false,
 257                               "Cannot commit bitmap memory");
 258   }
 259 
 260   _marking_context = new ShenandoahMarkingContext(_heap_region, _bitmap_region, _num_regions);
 261 
 262   if (ShenandoahVerify) {
 263     ReservedSpace verify_bitmap(_bitmap_size, bitmap_page_size);
 264     if (!verify_bitmap.special()) {
 265       os::commit_memory_or_exit(verify_bitmap.base(), verify_bitmap.size(), bitmap_page_size, false,
 266                                 "Cannot commit verification bitmap memory");
 267     }
 268     MemTracker::record_virtual_memory_type(verify_bitmap.base(), mtGC);
 269     MemRegion verify_bitmap_region = MemRegion((HeapWord *) verify_bitmap.base(), verify_bitmap.size() / HeapWordSize);
 270     _verification_bit_map.initialize(_heap_region, verify_bitmap_region);
 271     _verifier = new ShenandoahVerifier(this, &_verification_bit_map);
 272   }
 273 
 274   // Reserve aux bitmap for use in object_iterate(). We don't commit it here.
 275   ReservedSpace aux_bitmap(_bitmap_size, bitmap_page_size);
 276   MemTracker::record_virtual_memory_type(aux_bitmap.base(), mtGC);
 277   _aux_bitmap_region = MemRegion((HeapWord*) aux_bitmap.base(), aux_bitmap.size() / HeapWordSize);
 278   _aux_bitmap_region_special = aux_bitmap.special();
 279   _aux_bit_map.initialize(_heap_region, _aux_bitmap_region);
 280 
 281   //
 282   // Create regions and region sets
 283   //
 284   size_t region_align = align_up(sizeof(ShenandoahHeapRegion), SHENANDOAH_CACHE_LINE_SIZE);
 285   size_t region_storage_size = align_up(region_align * _num_regions, region_page_size);
 286   region_storage_size = align_up(region_storage_size, os::vm_allocation_granularity());
 287 
 288   ReservedSpace region_storage(region_storage_size, region_page_size);
 289   MemTracker::record_virtual_memory_type(region_storage.base(), mtGC);
 290   if (!region_storage.special()) {
 291     os::commit_memory_or_exit(region_storage.base(), region_storage_size, region_page_size, false,
 292                               "Cannot commit region memory");
 293   }
 294 
 295   _regions = NEW_C_HEAP_ARRAY(ShenandoahHeapRegion*, _num_regions, mtGC);
 296   _free_set = new ShenandoahFreeSet(this, _num_regions);
 297   _collection_set = new ShenandoahCollectionSet(this, sh_rs.base(), sh_rs.size());
 298 
 299   {
 300     ShenandoahHeapLocker locker(lock());
 301 
 302     for (size_t i = 0; i < _num_regions; i++) {
 303       HeapWord* start = (HeapWord*)sh_rs.base() + ShenandoahHeapRegion::region_size_words() * i;
 304       bool is_committed = i < num_committed_regions;
 305       void* loc = region_storage.base() + i * region_align;
 306 
 307       ShenandoahHeapRegion* r = new (loc) ShenandoahHeapRegion(start, i, is_committed);
 308       assert(is_aligned(r, SHENANDOAH_CACHE_LINE_SIZE), "Sanity");
 309 
 310       _marking_context->initialize_top_at_mark_start(r);
 311       _regions[i] = r;
 312       assert(!collection_set()->is_in(i), "New region should not be in collection set");
 313     }
 314 
 315     // Initialize to complete
 316     _marking_context->mark_complete();
 317 
 318     _free_set->rebuild();
 319   }
 320 
 321   if (ShenandoahAlwaysPreTouch) {
 322     assert(!AlwaysPreTouch, "Should have been overridden");
 323 
 324     // For NUMA, it is important to pre-touch the storage under bitmaps with worker threads,
 325     // before initialize() below zeroes it with initializing thread. For any given region,
 326     // we touch the region and the corresponding bitmaps from the same thread.
 327     ShenandoahPushWorkerScope scope(workers(), _max_workers, false);
 328 
 329     size_t pretouch_heap_page_size = heap_page_size;
 330     size_t pretouch_bitmap_page_size = bitmap_page_size;
 331 
 332 #ifdef LINUX
 333     // UseTransparentHugePages would madvise that backing memory can be coalesced into huge
 334     // pages. But, the kernel needs to know that every small page is used, in order to coalesce
 335     // them into huge one. Therefore, we need to pretouch with smaller pages.
 336     if (UseTransparentHugePages) {
 337       pretouch_heap_page_size = (size_t)os::vm_page_size();
 338       pretouch_bitmap_page_size = (size_t)os::vm_page_size();
 339     }
 340 #endif
 341 
 342     // OS memory managers may want to coalesce back-to-back pages. Make their jobs
 343     // simpler by pre-touching continuous spaces (heap and bitmap) separately.
 344 
 345     log_info(gc, init)("Pretouch bitmap: " SIZE_FORMAT " regions, " SIZE_FORMAT " bytes page",
 346                        _num_regions, pretouch_bitmap_page_size);
 347     ShenandoahPretouchBitmapTask bcl(bitmap.base(), _bitmap_size, pretouch_bitmap_page_size);
 348     _workers->run_task(&bcl);
 349 
 350     log_info(gc, init)("Pretouch heap: " SIZE_FORMAT " regions, " SIZE_FORMAT " bytes page",
 351                        _num_regions, pretouch_heap_page_size);
 352     ShenandoahPretouchHeapTask hcl(pretouch_heap_page_size);
 353     _workers->run_task(&hcl);
 354   }
 355 
 356   //
 357   // Initialize the rest of GC subsystems
 358   //
 359 
 360   _liveness_cache = NEW_C_HEAP_ARRAY(ShenandoahLiveData*, _max_workers, mtGC);
 361   for (uint worker = 0; worker < _max_workers; worker++) {
 362     _liveness_cache[worker] = NEW_C_HEAP_ARRAY(ShenandoahLiveData, _num_regions, mtGC);
 363     Copy::fill_to_bytes(_liveness_cache[worker], _num_regions * sizeof(ShenandoahLiveData));
 364   }
 365 
 366   // There should probably be Shenandoah-specific options for these,
 367   // just as there are G1-specific options.
 368   {
 369     ShenandoahSATBMarkQueueSet& satbqs = ShenandoahBarrierSet::satb_mark_queue_set();
 370     satbqs.set_process_completed_buffers_threshold(20); // G1SATBProcessCompletedThreshold
 371     satbqs.set_buffer_enqueue_threshold_percentage(60); // G1SATBBufferEnqueueingThresholdPercent
 372   }
 373 
 374   _monitoring_support = new ShenandoahMonitoringSupport(this);
 375   _phase_timings = new ShenandoahPhaseTimings(max_workers());
 376   ShenandoahStringDedup::initialize();
 377   ShenandoahCodeRoots::initialize();
 378 
 379   if (ShenandoahPacing) {
 380     _pacer = new ShenandoahPacer(this);
 381     _pacer->setup_for_idle();
 382   } else {
 383     _pacer = NULL;
 384   }
 385 
 386   _control_thread = new ShenandoahControlThread();
 387 
 388   log_info(gc, init)("Initialize Shenandoah heap: " SIZE_FORMAT "%s initial, " SIZE_FORMAT "%s min, " SIZE_FORMAT "%s max",
 389                      byte_size_in_proper_unit(_initial_size),  proper_unit_for_byte_size(_initial_size),
 390                      byte_size_in_proper_unit(_minimum_size),  proper_unit_for_byte_size(_minimum_size),
 391                      byte_size_in_proper_unit(max_capacity()), proper_unit_for_byte_size(max_capacity())
 392   );
 393 
 394   log_info(gc, init)("Safepointing mechanism: thread-local poll");
 395 
 396   return JNI_OK;
 397 }
 398 
 399 void ShenandoahHeap::initialize_heuristics() {
 400   if (ShenandoahGCMode != NULL) {
 401     if (strcmp(ShenandoahGCMode, "satb") == 0) {
 402       _gc_mode = new ShenandoahSATBMode();
 403     } else if (strcmp(ShenandoahGCMode, "iu") == 0) {
 404       _gc_mode = new ShenandoahIUMode();
 405     } else if (strcmp(ShenandoahGCMode, "passive") == 0) {
 406       _gc_mode = new ShenandoahPassiveMode();
 407     } else {
 408       vm_exit_during_initialization("Unknown -XX:ShenandoahGCMode option");
 409     }
 410   } else {
 411     ShouldNotReachHere();
 412   }
 413   _gc_mode->initialize_flags();
 414   if (_gc_mode->is_diagnostic() && !UnlockDiagnosticVMOptions) {
 415     vm_exit_during_initialization(
 416             err_msg("GC mode \"%s\" is diagnostic, and must be enabled via -XX:+UnlockDiagnosticVMOptions.",
 417                     _gc_mode->name()));
 418   }
 419   if (_gc_mode->is_experimental() && !UnlockExperimentalVMOptions) {
 420     vm_exit_during_initialization(
 421             err_msg("GC mode \"%s\" is experimental, and must be enabled via -XX:+UnlockExperimentalVMOptions.",
 422                     _gc_mode->name()));
 423   }
 424   log_info(gc, init)("Shenandoah GC mode: %s",
 425                      _gc_mode->name());
 426 
 427   _heuristics = _gc_mode->initialize_heuristics();
 428 
 429   if (_heuristics->is_diagnostic() && !UnlockDiagnosticVMOptions) {
 430     vm_exit_during_initialization(
 431             err_msg("Heuristics \"%s\" is diagnostic, and must be enabled via -XX:+UnlockDiagnosticVMOptions.",
 432                     _heuristics->name()));
 433   }
 434   if (_heuristics->is_experimental() && !UnlockExperimentalVMOptions) {
 435     vm_exit_during_initialization(
 436             err_msg("Heuristics \"%s\" is experimental, and must be enabled via -XX:+UnlockExperimentalVMOptions.",
 437                     _heuristics->name()));
 438   }
 439   log_info(gc, init)("Shenandoah heuristics: %s",
 440                      _heuristics->name());
 441 }
 442 
 443 #ifdef _MSC_VER
 444 #pragma warning( push )
 445 #pragma warning( disable:4355 ) // 'this' : used in base member initializer list
 446 #endif
 447 
 448 ShenandoahHeap::ShenandoahHeap(ShenandoahCollectorPolicy* policy) :
 449   CollectedHeap(),
 450   _initial_size(0),
 451   _used(0),
 452   _committed(0),
 453   _bytes_allocated_since_gc_start(0),
 454   _max_workers(MAX2(ConcGCThreads, ParallelGCThreads)),
 455   _workers(NULL),
 456   _safepoint_workers(NULL),
 457   _heap_region_special(false),
 458   _num_regions(0),
 459   _regions(NULL),
 460   _update_refs_iterator(this),
 461   _control_thread(NULL),
 462   _shenandoah_policy(policy),
 463   _heuristics(NULL),
 464   _free_set(NULL),
 465   _scm(new ShenandoahConcurrentMark()),
 466   _full_gc(new ShenandoahMarkCompact()),
 467   _pacer(NULL),
 468   _verifier(NULL),
 469   _phase_timings(NULL),
 470   _monitoring_support(NULL),
 471   _memory_pool(NULL),
 472   _stw_memory_manager("Shenandoah Pauses", "end of GC pause"),
 473   _cycle_memory_manager("Shenandoah Cycles", "end of GC cycle"),
 474   _gc_timer(new (ResourceObj::C_HEAP, mtGC) ConcurrentGCTimer()),
 475   _soft_ref_policy(),
 476   _log_min_obj_alignment_in_bytes(LogMinObjAlignmentInBytes),
 477   _ref_processor(NULL),
 478   _marking_context(NULL),
 479   _bitmap_size(0),
 480   _bitmap_regions_per_slice(0),
 481   _bitmap_bytes_per_slice(0),
 482   _bitmap_region_special(false),
 483   _aux_bitmap_region_special(false),
 484   _liveness_cache(NULL),
 485   _collection_set(NULL)
 486 {
 487   _heap = this;
 488 
 489   log_info(gc, init)("GC threads: " UINT32_FORMAT " parallel, " UINT32_FORMAT " concurrent", ParallelGCThreads, ConcGCThreads);
 490 
 491   BarrierSet::set_barrier_set(new ShenandoahBarrierSet(this));
 492 
 493   _max_workers = MAX2(_max_workers, 1U);
 494   _workers = new ShenandoahWorkGang("Shenandoah GC Threads", _max_workers,
 495                             /* are_GC_task_threads */ true,
 496                             /* are_ConcurrentGC_threads */ true);
 497   if (_workers == NULL) {
 498     vm_exit_during_initialization("Failed necessary allocation.");
 499   } else {
 500     _workers->initialize_workers();
 501   }
 502 
 503   if (ParallelGCThreads > 1) {
 504     _safepoint_workers = new ShenandoahWorkGang("Safepoint Cleanup Thread",
 505                                                 ParallelGCThreads,
 506                       /* are_GC_task_threads */ false,
 507                  /* are_ConcurrentGC_threads */ false);
 508     _safepoint_workers->initialize_workers();
 509   }
 510 }
 511 
 512 #ifdef _MSC_VER
 513 #pragma warning( pop )
 514 #endif
 515 
 516 class ShenandoahResetBitmapTask : public AbstractGangTask {
 517 private:
 518   ShenandoahRegionIterator _regions;
 519 
 520 public:
 521   ShenandoahResetBitmapTask() :
 522     AbstractGangTask("Parallel Reset Bitmap Task") {}
 523 
 524   void work(uint worker_id) {
 525     ShenandoahHeapRegion* region = _regions.next();
 526     ShenandoahHeap* heap = ShenandoahHeap::heap();
 527     ShenandoahMarkingContext* const ctx = heap->marking_context();
 528     while (region != NULL) {
 529       if (heap->is_bitmap_slice_committed(region)) {
 530         ctx->clear_bitmap(region);
 531       }
 532       region = _regions.next();
 533     }
 534   }
 535 };
 536 
 537 void ShenandoahHeap::reset_mark_bitmap() {
 538   assert_gc_workers(_workers->active_workers());
 539   mark_incomplete_marking_context();
 540 
 541   ShenandoahResetBitmapTask task;
 542   _workers->run_task(&task);
 543 }
 544 
 545 void ShenandoahHeap::print_on(outputStream* st) const {
 546   st->print_cr("Shenandoah Heap");
 547   st->print_cr(" " SIZE_FORMAT "%s total, " SIZE_FORMAT "%s committed, " SIZE_FORMAT "%s used",
 548                byte_size_in_proper_unit(max_capacity()), proper_unit_for_byte_size(max_capacity()),
 549                byte_size_in_proper_unit(committed()),    proper_unit_for_byte_size(committed()),
 550                byte_size_in_proper_unit(used()),         proper_unit_for_byte_size(used()));
 551   st->print_cr(" " SIZE_FORMAT " x " SIZE_FORMAT"%s regions",
 552                num_regions(),
 553                byte_size_in_proper_unit(ShenandoahHeapRegion::region_size_bytes()),
 554                proper_unit_for_byte_size(ShenandoahHeapRegion::region_size_bytes()));
 555 
 556   st->print("Status: ");
 557   if (has_forwarded_objects())                 st->print("has forwarded objects, ");
 558   if (is_concurrent_mark_in_progress())        st->print("marking, ");
 559   if (is_evacuation_in_progress())             st->print("evacuating, ");
 560   if (is_update_refs_in_progress())            st->print("updating refs, ");
 561   if (is_degenerated_gc_in_progress())         st->print("degenerated gc, ");
 562   if (is_full_gc_in_progress())                st->print("full gc, ");
 563   if (is_full_gc_move_in_progress())           st->print("full gc move, ");
 564   if (is_concurrent_weak_root_in_progress())   st->print("concurrent weak roots, ");
 565   if (is_concurrent_strong_root_in_progress() &&
 566       !is_concurrent_weak_root_in_progress())  st->print("concurrent strong roots, ");
 567 
 568   if (cancelled_gc()) {
 569     st->print("cancelled");
 570   } else {
 571     st->print("not cancelled");
 572   }
 573   st->cr();
 574 
 575   st->print_cr("Reserved region:");
 576   st->print_cr(" - [" PTR_FORMAT ", " PTR_FORMAT ") ",
 577                p2i(reserved_region().start()),
 578                p2i(reserved_region().end()));
 579 
 580   ShenandoahCollectionSet* cset = collection_set();
 581   st->print_cr("Collection set:");
 582   if (cset != NULL) {
 583     st->print_cr(" - map (vanilla): " PTR_FORMAT, p2i(cset->map_address()));
 584     st->print_cr(" - map (biased):  " PTR_FORMAT, p2i(cset->biased_map_address()));
 585   } else {
 586     st->print_cr(" (NULL)");
 587   }
 588 
 589   st->cr();
 590   MetaspaceUtils::print_on(st);
 591 
 592   if (Verbose) {
 593     print_heap_regions_on(st);
 594   }
 595 }
 596 
 597 class ShenandoahInitWorkerGCLABClosure : public ThreadClosure {
 598 public:
 599   void do_thread(Thread* thread) {
 600     assert(thread != NULL, "Sanity");
 601     assert(thread->is_Worker_thread(), "Only worker thread expected");
 602     ShenandoahThreadLocalData::initialize_gclab(thread);
 603   }
 604 };
 605 
 606 void ShenandoahHeap::post_initialize() {
 607   CollectedHeap::post_initialize();
 608   MutexLocker ml(Threads_lock);
 609 
 610   ShenandoahInitWorkerGCLABClosure init_gclabs;
 611   _workers->threads_do(&init_gclabs);
 612 
 613   // gclab can not be initialized early during VM startup, as it can not determinate its max_size.
 614   // Now, we will let WorkGang to initialize gclab when new worker is created.
 615   _workers->set_initialize_gclab();
 616 
 617   _scm->initialize(_max_workers);
 618   _full_gc->initialize(_gc_timer);
 619 
 620   ref_processing_init();
 621 
 622   _heuristics->initialize();
 623 
 624   JFR_ONLY(ShenandoahJFRSupport::register_jfr_type_serializers());
 625 }
 626 
 627 size_t ShenandoahHeap::used() const {
 628   return Atomic::load_acquire(&_used);
 629 }
 630 
 631 size_t ShenandoahHeap::committed() const {
 632   OrderAccess::acquire();
 633   return _committed;
 634 }
 635 
 636 void ShenandoahHeap::increase_committed(size_t bytes) {
 637   shenandoah_assert_heaplocked_or_safepoint();
 638   _committed += bytes;
 639 }
 640 
 641 void ShenandoahHeap::decrease_committed(size_t bytes) {
 642   shenandoah_assert_heaplocked_or_safepoint();
 643   _committed -= bytes;
 644 }
 645 
 646 void ShenandoahHeap::increase_used(size_t bytes) {
 647   Atomic::add(&_used, bytes);
 648 }
 649 
 650 void ShenandoahHeap::set_used(size_t bytes) {
 651   Atomic::release_store_fence(&_used, bytes);
 652 }
 653 
 654 void ShenandoahHeap::decrease_used(size_t bytes) {
 655   assert(used() >= bytes, "never decrease heap size by more than we've left");
 656   Atomic::sub(&_used, bytes);
 657 }
 658 
 659 void ShenandoahHeap::increase_allocated(size_t bytes) {
 660   Atomic::add(&_bytes_allocated_since_gc_start, bytes);
 661 }
 662 
 663 void ShenandoahHeap::notify_mutator_alloc_words(size_t words, bool waste) {
 664   size_t bytes = words * HeapWordSize;
 665   if (!waste) {
 666     increase_used(bytes);
 667   }
 668   increase_allocated(bytes);
 669   if (ShenandoahPacing) {
 670     control_thread()->pacing_notify_alloc(words);
 671     if (waste) {
 672       pacer()->claim_for_alloc(words, true);
 673     }
 674   }
 675 }
 676 
 677 size_t ShenandoahHeap::capacity() const {
 678   return committed();
 679 }
 680 
 681 size_t ShenandoahHeap::max_capacity() const {
 682   return _num_regions * ShenandoahHeapRegion::region_size_bytes();
 683 }
 684 
 685 size_t ShenandoahHeap::min_capacity() const {
 686   return _minimum_size;
 687 }
 688 
 689 size_t ShenandoahHeap::initial_capacity() const {
 690   return _initial_size;
 691 }
 692 
 693 bool ShenandoahHeap::is_in(const void* p) const {
 694   HeapWord* heap_base = (HeapWord*) base();
 695   HeapWord* last_region_end = heap_base + ShenandoahHeapRegion::region_size_words() * num_regions();
 696   return p >= heap_base && p < last_region_end;
 697 }
 698 
 699 void ShenandoahHeap::op_uncommit(double shrink_before) {
 700   assert (ShenandoahUncommit, "should be enabled");
 701 
 702   // Application allocates from the beginning of the heap, and GC allocates at
 703   // the end of it. It is more efficient to uncommit from the end, so that applications
 704   // could enjoy the near committed regions. GC allocations are much less frequent,
 705   // and therefore can accept the committing costs.
 706 
 707   size_t count = 0;
 708   for (size_t i = num_regions(); i > 0; i--) { // care about size_t underflow
 709     ShenandoahHeapRegion* r = get_region(i - 1);
 710     if (r->is_empty_committed() && (r->empty_time() < shrink_before)) {
 711       ShenandoahHeapLocker locker(lock());
 712       if (r->is_empty_committed()) {
 713         // Do not uncommit below minimal capacity
 714         if (committed() < min_capacity() + ShenandoahHeapRegion::region_size_bytes()) {
 715           break;
 716         }
 717 
 718         r->make_uncommitted();
 719         count++;
 720       }
 721     }
 722     SpinPause(); // allow allocators to take the lock
 723   }
 724 
 725   if (count > 0) {
 726     control_thread()->notify_heap_changed();
 727   }
 728 }
 729 
 730 HeapWord* ShenandoahHeap::allocate_from_gclab_slow(Thread* thread, size_t size) {
 731   // New object should fit the GCLAB size
 732   size_t min_size = MAX2(size, PLAB::min_size());
 733 
 734   // Figure out size of new GCLAB, looking back at heuristics. Expand aggressively.
 735   size_t new_size = ShenandoahThreadLocalData::gclab_size(thread) * 2;
 736   new_size = MIN2(new_size, PLAB::max_size());
 737   new_size = MAX2(new_size, PLAB::min_size());
 738 
 739   // Record new heuristic value even if we take any shortcut. This captures
 740   // the case when moderately-sized objects always take a shortcut. At some point,
 741   // heuristics should catch up with them.
 742   ShenandoahThreadLocalData::set_gclab_size(thread, new_size);
 743 
 744   if (new_size < size) {
 745     // New size still does not fit the object. Fall back to shared allocation.
 746     // This avoids retiring perfectly good GCLABs, when we encounter a large object.
 747     return NULL;
 748   }
 749 
 750   // Retire current GCLAB, and allocate a new one.
 751   PLAB* gclab = ShenandoahThreadLocalData::gclab(thread);
 752   gclab->retire();
 753 
 754   size_t actual_size = 0;
 755   HeapWord* gclab_buf = allocate_new_gclab(min_size, new_size, &actual_size);
 756   if (gclab_buf == NULL) {
 757     return NULL;
 758   }
 759 
 760   assert (size <= actual_size, "allocation should fit");
 761 
 762   if (ZeroTLAB) {
 763     // ..and clear it.
 764     Copy::zero_to_words(gclab_buf, actual_size);
 765   } else {
 766     // ...and zap just allocated object.
 767 #ifdef ASSERT
 768     // Skip mangling the space corresponding to the object header to
 769     // ensure that the returned space is not considered parsable by
 770     // any concurrent GC thread.
 771     size_t hdr_size = oopDesc::header_size();
 772     Copy::fill_to_words(gclab_buf + hdr_size, actual_size - hdr_size, badHeapWordVal);
 773 #endif // ASSERT
 774   }
 775   gclab->set_buf(gclab_buf, actual_size);
 776   return gclab->allocate(size);
 777 }
 778 
 779 HeapWord* ShenandoahHeap::allocate_new_tlab(size_t min_size,
 780                                             size_t requested_size,
 781                                             size_t* actual_size) {
 782   ShenandoahAllocRequest req = ShenandoahAllocRequest::for_tlab(min_size, requested_size);
 783   HeapWord* res = allocate_memory(req);
 784   if (res != NULL) {
 785     *actual_size = req.actual_size();
 786   } else {
 787     *actual_size = 0;
 788   }
 789   return res;
 790 }
 791 
 792 HeapWord* ShenandoahHeap::allocate_new_gclab(size_t min_size,
 793                                              size_t word_size,
 794                                              size_t* actual_size) {
 795   ShenandoahAllocRequest req = ShenandoahAllocRequest::for_gclab(min_size, word_size);
 796   HeapWord* res = allocate_memory(req);
 797   if (res != NULL) {
 798     *actual_size = req.actual_size();
 799   } else {
 800     *actual_size = 0;
 801   }
 802   return res;
 803 }
 804 
 805 HeapWord* ShenandoahHeap::allocate_memory(ShenandoahAllocRequest& req) {
 806   intptr_t pacer_epoch = 0;
 807   bool in_new_region = false;
 808   HeapWord* result = NULL;
 809 
 810   if (req.is_mutator_alloc()) {
 811     if (ShenandoahPacing) {
 812       pacer()->pace_for_alloc(req.size());
 813       pacer_epoch = pacer()->epoch();
 814     }
 815 
 816     if (!ShenandoahAllocFailureALot || !should_inject_alloc_failure()) {
 817       result = allocate_memory_under_lock(req, in_new_region);
 818     }
 819 
 820     // Allocation failed, block until control thread reacted, then retry allocation.
 821     //
 822     // It might happen that one of the threads requesting allocation would unblock
 823     // way later after GC happened, only to fail the second allocation, because
 824     // other threads have already depleted the free storage. In this case, a better
 825     // strategy is to try again, as long as GC makes progress.
 826     //
 827     // Then, we need to make sure the allocation was retried after at least one
 828     // Full GC, which means we want to try more than ShenandoahFullGCThreshold times.
 829 
 830     size_t tries = 0;
 831 
 832     while (result == NULL && _progress_last_gc.is_set()) {
 833       tries++;
 834       control_thread()->handle_alloc_failure(req);
 835       result = allocate_memory_under_lock(req, in_new_region);
 836     }
 837 
 838     while (result == NULL && tries <= ShenandoahFullGCThreshold) {
 839       tries++;
 840       control_thread()->handle_alloc_failure(req);
 841       result = allocate_memory_under_lock(req, in_new_region);
 842     }
 843 
 844   } else {
 845     assert(req.is_gc_alloc(), "Can only accept GC allocs here");
 846     result = allocate_memory_under_lock(req, in_new_region);
 847     // Do not call handle_alloc_failure() here, because we cannot block.
 848     // The allocation failure would be handled by the LRB slowpath with handle_alloc_failure_evac().
 849   }
 850 
 851   if (in_new_region) {
 852     control_thread()->notify_heap_changed();
 853   }
 854 
 855   if (result != NULL) {
 856     size_t requested = req.size();
 857     size_t actual = req.actual_size();
 858 
 859     assert (req.is_lab_alloc() || (requested == actual),
 860             "Only LAB allocations are elastic: %s, requested = " SIZE_FORMAT ", actual = " SIZE_FORMAT,
 861             ShenandoahAllocRequest::alloc_type_to_string(req.type()), requested, actual);
 862 
 863     if (req.is_mutator_alloc()) {
 864       notify_mutator_alloc_words(actual, false);
 865 
 866       // If we requested more than we were granted, give the rest back to pacer.
 867       // This only matters if we are in the same pacing epoch: do not try to unpace
 868       // over the budget for the other phase.
 869       if (ShenandoahPacing && (pacer_epoch > 0) && (requested > actual)) {
 870         pacer()->unpace_for_alloc(pacer_epoch, requested - actual);
 871       }
 872     } else {
 873       increase_used(actual*HeapWordSize);
 874     }
 875   }
 876 
 877   return result;
 878 }
 879 
 880 HeapWord* ShenandoahHeap::allocate_memory_under_lock(ShenandoahAllocRequest& req, bool& in_new_region) {
 881   ShenandoahHeapLocker locker(lock());
 882   return _free_set->allocate(req, in_new_region);
 883 }
 884 
 885 HeapWord* ShenandoahHeap::mem_allocate(size_t size,
 886                                         bool*  gc_overhead_limit_was_exceeded) {
 887   ShenandoahAllocRequest req = ShenandoahAllocRequest::for_shared(size);
 888   return allocate_memory(req);
 889 }
 890 
 891 MetaWord* ShenandoahHeap::satisfy_failed_metadata_allocation(ClassLoaderData* loader_data,
 892                                                              size_t size,
 893                                                              Metaspace::MetadataType mdtype) {
 894   MetaWord* result;
 895 
 896   // Inform metaspace OOM to GC heuristics if class unloading is possible.
 897   if (heuristics()->can_unload_classes()) {
 898     ShenandoahHeuristics* h = heuristics();
 899     h->record_metaspace_oom();
 900   }
 901 
 902   // Expand and retry allocation
 903   result = loader_data->metaspace_non_null()->expand_and_allocate(size, mdtype);
 904   if (result != NULL) {
 905     return result;
 906   }
 907 
 908   // Start full GC
 909   collect(GCCause::_metadata_GC_clear_soft_refs);
 910 
 911   // Retry allocation
 912   result = loader_data->metaspace_non_null()->allocate(size, mdtype);
 913   if (result != NULL) {
 914     return result;
 915   }
 916 
 917   // Expand and retry allocation
 918   result = loader_data->metaspace_non_null()->expand_and_allocate(size, mdtype);
 919   if (result != NULL) {
 920     return result;
 921   }
 922 
 923   // Out of memory
 924   return NULL;
 925 }
 926 
 927 class ShenandoahConcurrentEvacuateRegionObjectClosure : public ObjectClosure {
 928 private:
 929   ShenandoahHeap* const _heap;
 930   Thread* const _thread;
 931 public:
 932   ShenandoahConcurrentEvacuateRegionObjectClosure(ShenandoahHeap* heap) :
 933     _heap(heap), _thread(Thread::current()) {}
 934 
 935   void do_object(oop p) {
 936     shenandoah_assert_marked(NULL, p);
 937     if (!p->is_forwarded()) {
 938       _heap->evacuate_object(p, _thread);
 939     }
 940   }
 941 };
 942 
 943 class ShenandoahEvacuationTask : public AbstractGangTask {
 944 private:
 945   ShenandoahHeap* const _sh;
 946   ShenandoahCollectionSet* const _cs;
 947   bool _concurrent;
 948 public:
 949   ShenandoahEvacuationTask(ShenandoahHeap* sh,
 950                            ShenandoahCollectionSet* cs,
 951                            bool concurrent) :
 952     AbstractGangTask("Parallel Evacuation Task"),
 953     _sh(sh),
 954     _cs(cs),
 955     _concurrent(concurrent)
 956   {}
 957 
 958   void work(uint worker_id) {
 959     if (_concurrent) {
 960       ShenandoahConcurrentWorkerSession worker_session(worker_id);
 961       ShenandoahSuspendibleThreadSetJoiner stsj(ShenandoahSuspendibleWorkers);
 962       ShenandoahEvacOOMScope oom_evac_scope;
 963       do_work();
 964     } else {
 965       ShenandoahParallelWorkerSession worker_session(worker_id);
 966       ShenandoahEvacOOMScope oom_evac_scope;
 967       do_work();
 968     }
 969   }
 970 
 971 private:
 972   void do_work() {
 973     ShenandoahConcurrentEvacuateRegionObjectClosure cl(_sh);
 974     ShenandoahHeapRegion* r;
 975     while ((r =_cs->claim_next()) != NULL) {
 976       assert(r->has_live(), "Region " SIZE_FORMAT " should have been reclaimed early", r->index());
 977       _sh->marked_object_iterate(r, &cl);
 978 
 979       if (ShenandoahPacing) {
 980         _sh->pacer()->report_evac(r->used() >> LogHeapWordSize);
 981       }
 982 
 983       if (_sh->check_cancelled_gc_and_yield(_concurrent)) {
 984         break;
 985       }
 986     }
 987   }
 988 };
 989 
 990 void ShenandoahHeap::trash_cset_regions() {
 991   ShenandoahHeapLocker locker(lock());
 992 
 993   ShenandoahCollectionSet* set = collection_set();
 994   ShenandoahHeapRegion* r;
 995   set->clear_current_index();
 996   while ((r = set->next()) != NULL) {
 997     r->make_trash();
 998   }
 999   collection_set()->clear();
1000 }
1001 
1002 void ShenandoahHeap::print_heap_regions_on(outputStream* st) const {
1003   st->print_cr("Heap Regions:");
1004   st->print_cr("EU=empty-uncommitted, EC=empty-committed, R=regular, H=humongous start, HC=humongous continuation, CS=collection set, T=trash, P=pinned");
1005   st->print_cr("BTE=bottom/top/end, U=used, T=TLAB allocs, G=GCLAB allocs, S=shared allocs, L=live data");
1006   st->print_cr("R=root, CP=critical pins, TAMS=top-at-mark-start, UWM=update watermark");
1007   st->print_cr("SN=alloc sequence number");
1008 
1009   for (size_t i = 0; i < num_regions(); i++) {
1010     get_region(i)->print_on(st);
1011   }
1012 }
1013 
1014 void ShenandoahHeap::trash_humongous_region_at(ShenandoahHeapRegion* start) {
1015   assert(start->is_humongous_start(), "reclaim regions starting with the first one");
1016 
1017   oop humongous_obj = oop(start->bottom());
1018   size_t size = humongous_obj->size();
1019   size_t required_regions = ShenandoahHeapRegion::required_regions(size * HeapWordSize);
1020   size_t index = start->index() + required_regions - 1;
1021 
1022   assert(!start->has_live(), "liveness must be zero");
1023 
1024   for(size_t i = 0; i < required_regions; i++) {
1025     // Reclaim from tail. Otherwise, assertion fails when printing region to trace log,
1026     // as it expects that every region belongs to a humongous region starting with a humongous start region.
1027     ShenandoahHeapRegion* region = get_region(index --);
1028 
1029     assert(region->is_humongous(), "expect correct humongous start or continuation");
1030     assert(!region->is_cset(), "Humongous region should not be in collection set");
1031 
1032     region->make_trash_immediate();
1033   }
1034 }
1035 
1036 class ShenandoahRetireGCLABClosure : public ThreadClosure {
1037 public:
1038   void do_thread(Thread* thread) {
1039     PLAB* gclab = ShenandoahThreadLocalData::gclab(thread);
1040     assert(gclab != NULL, "GCLAB should be initialized for %s", thread->name());
1041     gclab->retire();
1042   }
1043 };
1044 
1045 void ShenandoahHeap::make_parsable(bool retire_tlabs) {
1046   if (UseTLAB) {
1047     CollectedHeap::ensure_parsability(retire_tlabs);
1048   }
1049   ShenandoahRetireGCLABClosure cl;
1050   for (JavaThreadIteratorWithHandle jtiwh; JavaThread *t = jtiwh.next(); ) {
1051     cl.do_thread(t);
1052   }
1053   workers()->threads_do(&cl);
1054 }
1055 
1056 void ShenandoahHeap::resize_tlabs() {
1057   CollectedHeap::resize_all_tlabs();
1058 }
1059 
1060 class ShenandoahEvacuateUpdateRootsTask : public AbstractGangTask {
1061 private:
1062   ShenandoahRootEvacuator* _rp;
1063 
1064 public:
1065   ShenandoahEvacuateUpdateRootsTask(ShenandoahRootEvacuator* rp) :
1066     AbstractGangTask("Shenandoah evacuate and update roots"),
1067     _rp(rp) {}
1068 
1069   void work(uint worker_id) {
1070     ShenandoahParallelWorkerSession worker_session(worker_id);
1071     ShenandoahEvacOOMScope oom_evac_scope;
1072     ShenandoahEvacuateUpdateRootsClosure<> cl;
1073     MarkingCodeBlobClosure blobsCl(&cl, CodeBlobToOopClosure::FixRelocations);
1074     _rp->roots_do(worker_id, &cl);
1075   }
1076 };
1077 
1078 void ShenandoahHeap::evacuate_and_update_roots() {
1079 #if COMPILER2_OR_JVMCI
1080   DerivedPointerTable::clear();
1081 #endif
1082   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Only iterate roots while world is stopped");
1083   {
1084     // Include concurrent roots if current cycle can not process those roots concurrently
1085     ShenandoahRootEvacuator rp(workers()->active_workers(),
1086                                ShenandoahPhaseTimings::init_evac,
1087                                !ShenandoahConcurrentRoots::should_do_concurrent_roots(),
1088                                !ShenandoahConcurrentRoots::should_do_concurrent_class_unloading());
1089     ShenandoahEvacuateUpdateRootsTask roots_task(&rp);
1090     workers()->run_task(&roots_task);
1091   }
1092 
1093 #if COMPILER2_OR_JVMCI
1094   DerivedPointerTable::update_pointers();
1095 #endif
1096 }
1097 
1098 // Returns size in bytes
1099 size_t ShenandoahHeap::unsafe_max_tlab_alloc(Thread *thread) const {
1100   if (ShenandoahElasticTLAB) {
1101     // With Elastic TLABs, return the max allowed size, and let the allocation path
1102     // figure out the safe size for current allocation.
1103     return ShenandoahHeapRegion::max_tlab_size_bytes();
1104   } else {
1105     return MIN2(_free_set->unsafe_peek_free(), ShenandoahHeapRegion::max_tlab_size_bytes());
1106   }
1107 }
1108 
1109 size_t ShenandoahHeap::max_tlab_size() const {
1110   // Returns size in words
1111   return ShenandoahHeapRegion::max_tlab_size_words();
1112 }
1113 
1114 class ShenandoahRetireAndResetGCLABClosure : public ThreadClosure {
1115 public:
1116   void do_thread(Thread* thread) {
1117     PLAB* gclab = ShenandoahThreadLocalData::gclab(thread);
1118     gclab->retire();
1119     if (ShenandoahThreadLocalData::gclab_size(thread) > 0) {
1120       ShenandoahThreadLocalData::set_gclab_size(thread, 0);
1121     }
1122   }
1123 };
1124 
1125 void ShenandoahHeap::retire_and_reset_gclabs() {
1126   ShenandoahRetireAndResetGCLABClosure cl;
1127   for (JavaThreadIteratorWithHandle jtiwh; JavaThread *t = jtiwh.next(); ) {
1128     cl.do_thread(t);
1129   }
1130   workers()->threads_do(&cl);
1131 }
1132 
1133 void ShenandoahHeap::collect(GCCause::Cause cause) {
1134   control_thread()->request_gc(cause);
1135 }
1136 
1137 void ShenandoahHeap::do_full_collection(bool clear_all_soft_refs) {
1138   //assert(false, "Shouldn't need to do full collections");
1139 }
1140 
1141 HeapWord* ShenandoahHeap::block_start(const void* addr) const {
1142   ShenandoahHeapRegion* r = heap_region_containing(addr);
1143   if (r != NULL) {
1144     return r->block_start(addr);
1145   }
1146   return NULL;
1147 }
1148 
1149 bool ShenandoahHeap::block_is_obj(const HeapWord* addr) const {
1150   ShenandoahHeapRegion* r = heap_region_containing(addr);
1151   return r->block_is_obj(addr);
1152 }
1153 
1154 bool ShenandoahHeap::print_location(outputStream* st, void* addr) const {
1155   return BlockLocationPrinter<ShenandoahHeap>::print_location(st, addr);
1156 }
1157 
1158 jlong ShenandoahHeap::millis_since_last_gc() {
1159   double v = heuristics()->time_since_last_gc() * 1000;
1160   assert(0 <= v && v <= max_jlong, "value should fit: %f", v);
1161   return (jlong)v;
1162 }
1163 
1164 void ShenandoahHeap::prepare_for_verify() {
1165   if (SafepointSynchronize::is_at_safepoint() || ! UseTLAB) {
1166     make_parsable(false);
1167   }
1168 }
1169 
1170 void ShenandoahHeap::print_gc_threads_on(outputStream* st) const {
1171   workers()->print_worker_threads_on(st);
1172   if (ShenandoahStringDedup::is_enabled()) {
1173     ShenandoahStringDedup::print_worker_threads_on(st);
1174   }
1175 }
1176 
1177 void ShenandoahHeap::gc_threads_do(ThreadClosure* tcl) const {
1178   workers()->threads_do(tcl);
1179   if (_safepoint_workers != NULL) {
1180     _safepoint_workers->threads_do(tcl);
1181   }
1182   if (ShenandoahStringDedup::is_enabled()) {
1183     ShenandoahStringDedup::threads_do(tcl);
1184   }
1185 }
1186 
1187 void ShenandoahHeap::print_tracing_info() const {
1188   LogTarget(Info, gc, stats) lt;
1189   if (lt.is_enabled()) {
1190     ResourceMark rm;
1191     LogStream ls(lt);
1192 
1193     phase_timings()->print_global_on(&ls);
1194 
1195     ls.cr();
1196     ls.cr();
1197 
1198     shenandoah_policy()->print_gc_stats(&ls);
1199 
1200     ls.cr();
1201     ls.cr();
1202 
1203     if (ShenandoahPacing) {
1204       pacer()->print_on(&ls);
1205     }
1206 
1207     ls.cr();
1208     ls.cr();
1209   }
1210 }
1211 
1212 void ShenandoahHeap::verify(VerifyOption vo) {
1213   if (ShenandoahSafepoint::is_at_shenandoah_safepoint()) {
1214     if (ShenandoahVerify) {
1215       verifier()->verify_generic(vo);
1216     } else {
1217       // TODO: Consider allocating verification bitmaps on demand,
1218       // and turn this on unconditionally.
1219     }
1220   }
1221 }
1222 size_t ShenandoahHeap::tlab_capacity(Thread *thr) const {
1223   return _free_set->capacity();
1224 }
1225 
1226 class ObjectIterateScanRootClosure : public BasicOopIterateClosure {
1227 private:
1228   MarkBitMap* _bitmap;
1229   Stack<oop,mtGC>* _oop_stack;
1230   ShenandoahHeap* const _heap;
1231   ShenandoahMarkingContext* const _marking_context;
1232 
1233   template <class T>
1234   void do_oop_work(T* p) {
1235     T o = RawAccess<>::oop_load(p);
1236     if (!CompressedOops::is_null(o)) {
1237       oop obj = CompressedOops::decode_not_null(o);
1238       if (_heap->is_concurrent_weak_root_in_progress() && !_marking_context->is_marked(obj)) {
1239         // There may be dead oops in weak roots in concurrent root phase, do not touch them.
1240         return;
1241       }
1242       obj = ShenandoahBarrierSet::resolve_forwarded_not_null(obj);
1243 
1244       assert(oopDesc::is_oop(obj), "must be a valid oop");
1245       if (!_bitmap->is_marked(obj)) {
1246         _bitmap->mark(obj);
1247         _oop_stack->push(obj);
1248       }
1249     }
1250   }
1251 public:
1252   ObjectIterateScanRootClosure(MarkBitMap* bitmap, Stack<oop,mtGC>* oop_stack) :
1253     _bitmap(bitmap), _oop_stack(oop_stack), _heap(ShenandoahHeap::heap()),
1254     _marking_context(_heap->marking_context()) {}
1255   void do_oop(oop* p)       { do_oop_work(p); }
1256   void do_oop(narrowOop* p) { do_oop_work(p); }
1257 };
1258 
1259 /*
1260  * This is public API, used in preparation of object_iterate().
1261  * Since we don't do linear scan of heap in object_iterate() (see comment below), we don't
1262  * need to make the heap parsable. For Shenandoah-internal linear heap scans that we can
1263  * control, we call SH::make_tlabs_parsable().
1264  */
1265 void ShenandoahHeap::ensure_parsability(bool retire_tlabs) {
1266   // No-op.
1267 }
1268 
1269 /*
1270  * Iterates objects in the heap. This is public API, used for, e.g., heap dumping.
1271  *
1272  * We cannot safely iterate objects by doing a linear scan at random points in time. Linear
1273  * scanning needs to deal with dead objects, which may have dead Klass* pointers (e.g.
1274  * calling oopDesc::size() would crash) or dangling reference fields (crashes) etc. Linear
1275  * scanning therefore depends on having a valid marking bitmap to support it. However, we only
1276  * have a valid marking bitmap after successful marking. In particular, we *don't* have a valid
1277  * marking bitmap during marking, after aborted marking or during/after cleanup (when we just
1278  * wiped the bitmap in preparation for next marking).
1279  *
1280  * For all those reasons, we implement object iteration as a single marking traversal, reporting
1281  * objects as we mark+traverse through the heap, starting from GC roots. JVMTI IterateThroughHeap
1282  * is allowed to report dead objects, but is not required to do so.
1283  */
1284 void ShenandoahHeap::object_iterate(ObjectClosure* cl) {
1285   assert(SafepointSynchronize::is_at_safepoint(), "safe iteration is only available during safepoints");
1286   if (!_aux_bitmap_region_special && !os::commit_memory((char*)_aux_bitmap_region.start(), _aux_bitmap_region.byte_size(), false)) {
1287     log_warning(gc)("Could not commit native memory for auxiliary marking bitmap for heap iteration");
1288     return;
1289   }
1290 
1291   // Reset bitmap
1292   _aux_bit_map.clear();
1293 
1294   Stack<oop,mtGC> oop_stack;
1295 
1296   // First, we process GC roots according to current GC cycle. This populates the work stack with initial objects.
1297   ShenandoahHeapIterationRootScanner rp;
1298   ObjectIterateScanRootClosure oops(&_aux_bit_map, &oop_stack);
1299 
1300   rp.roots_do(&oops);
1301 
1302   // Work through the oop stack to traverse heap.
1303   while (! oop_stack.is_empty()) {
1304     oop obj = oop_stack.pop();
1305     assert(oopDesc::is_oop(obj), "must be a valid oop");
1306     cl->do_object(obj);
1307     obj->oop_iterate(&oops);
1308   }
1309 
1310   assert(oop_stack.is_empty(), "should be empty");
1311 
1312   if (!_aux_bitmap_region_special && !os::uncommit_memory((char*)_aux_bitmap_region.start(), _aux_bitmap_region.byte_size())) {
1313     log_warning(gc)("Could not uncommit native memory for auxiliary marking bitmap for heap iteration");
1314   }
1315 }
1316 
1317 // Keep alive an object that was loaded with AS_NO_KEEPALIVE.
1318 void ShenandoahHeap::keep_alive(oop obj) {
1319   if (is_concurrent_mark_in_progress()) {
1320     ShenandoahBarrierSet::barrier_set()->enqueue(obj);
1321   }
1322 }
1323 
1324 void ShenandoahHeap::heap_region_iterate(ShenandoahHeapRegionClosure* blk) const {
1325   for (size_t i = 0; i < num_regions(); i++) {
1326     ShenandoahHeapRegion* current = get_region(i);
1327     blk->heap_region_do(current);
1328   }
1329 }
1330 
1331 class ShenandoahParallelHeapRegionTask : public AbstractGangTask {
1332 private:
1333   ShenandoahHeap* const _heap;
1334   ShenandoahHeapRegionClosure* const _blk;
1335 
1336   shenandoah_padding(0);
1337   volatile size_t _index;
1338   shenandoah_padding(1);
1339 
1340 public:
1341   ShenandoahParallelHeapRegionTask(ShenandoahHeapRegionClosure* blk) :
1342           AbstractGangTask("Parallel Region Task"),
1343           _heap(ShenandoahHeap::heap()), _blk(blk), _index(0) {}
1344 
1345   void work(uint worker_id) {
1346     ShenandoahParallelWorkerSession worker_session(worker_id);
1347     size_t stride = ShenandoahParallelRegionStride;
1348 
1349     size_t max = _heap->num_regions();
1350     while (_index < max) {
1351       size_t cur = Atomic::fetch_and_add(&_index, stride);
1352       size_t start = cur;
1353       size_t end = MIN2(cur + stride, max);
1354       if (start >= max) break;
1355 
1356       for (size_t i = cur; i < end; i++) {
1357         ShenandoahHeapRegion* current = _heap->get_region(i);
1358         _blk->heap_region_do(current);
1359       }
1360     }
1361   }
1362 };
1363 
1364 void ShenandoahHeap::parallel_heap_region_iterate(ShenandoahHeapRegionClosure* blk) const {
1365   assert(blk->is_thread_safe(), "Only thread-safe closures here");
1366   if (num_regions() > ShenandoahParallelRegionStride) {
1367     ShenandoahParallelHeapRegionTask task(blk);
1368     workers()->run_task(&task);
1369   } else {
1370     heap_region_iterate(blk);
1371   }
1372 }
1373 
1374 class ShenandoahInitMarkUpdateRegionStateClosure : public ShenandoahHeapRegionClosure {
1375 private:
1376   ShenandoahMarkingContext* const _ctx;
1377 public:
1378   ShenandoahInitMarkUpdateRegionStateClosure() : _ctx(ShenandoahHeap::heap()->marking_context()) {}
1379 
1380   void heap_region_do(ShenandoahHeapRegion* r) {
1381     assert(!r->has_live(), "Region " SIZE_FORMAT " should have no live data", r->index());
1382     if (r->is_active()) {
1383       // Check if region needs updating its TAMS. We have updated it already during concurrent
1384       // reset, so it is very likely we don't need to do another write here.
1385       if (_ctx->top_at_mark_start(r) != r->top()) {
1386         _ctx->capture_top_at_mark_start(r);
1387       }
1388     } else {
1389       assert(_ctx->top_at_mark_start(r) == r->top(),
1390              "Region " SIZE_FORMAT " should already have correct TAMS", r->index());
1391     }
1392   }
1393 
1394   bool is_thread_safe() { return true; }
1395 };
1396 
1397 void ShenandoahHeap::op_init_mark() {
1398   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Should be at safepoint");
1399   assert(Thread::current()->is_VM_thread(), "can only do this in VMThread");
1400 
1401   assert(marking_context()->is_bitmap_clear(), "need clear marking bitmap");
1402   assert(!marking_context()->is_complete(), "should not be complete");
1403   assert(!has_forwarded_objects(), "No forwarded objects on this path");
1404 
1405   if (ShenandoahVerify) {
1406     verifier()->verify_before_concmark();
1407   }
1408 
1409   if (VerifyBeforeGC) {
1410     Universe::verify();
1411   }
1412 
1413   set_concurrent_mark_in_progress(true);
1414   // We need to reset all TLABs because we'd lose marks on all objects allocated in them.
1415   {
1416     ShenandoahGCPhase phase(ShenandoahPhaseTimings::make_parsable);
1417     make_parsable(true);
1418   }
1419 
1420   {
1421     ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_update_region_states);
1422     ShenandoahInitMarkUpdateRegionStateClosure cl;
1423     parallel_heap_region_iterate(&cl);
1424   }
1425 
1426   // Make above changes visible to worker threads
1427   OrderAccess::fence();
1428 
1429   concurrent_mark()->mark_roots(ShenandoahPhaseTimings::scan_roots);
1430 
1431   if (UseTLAB) {
1432     ShenandoahGCPhase phase(ShenandoahPhaseTimings::resize_tlabs);
1433     resize_tlabs();
1434   }
1435 
1436   if (ShenandoahPacing) {
1437     pacer()->setup_for_mark();
1438   }
1439 
1440   // Arm nmethods for concurrent marking. When a nmethod is about to be executed,
1441   // we need to make sure that all its metadata are marked. alternative is to remark
1442   // thread roots at final mark pause, but it can be potential latency killer.
1443   if (ShenandoahConcurrentRoots::should_do_concurrent_class_unloading()) {
1444     ShenandoahCodeRoots::arm_nmethods();
1445   }
1446 }
1447 
1448 void ShenandoahHeap::op_mark() {
1449   concurrent_mark()->mark_from_roots();
1450 }
1451 
1452 class ShenandoahFinalMarkUpdateRegionStateClosure : public ShenandoahHeapRegionClosure {
1453 private:
1454   ShenandoahMarkingContext* const _ctx;
1455   ShenandoahHeapLock* const _lock;
1456 
1457 public:
1458   ShenandoahFinalMarkUpdateRegionStateClosure() :
1459     _ctx(ShenandoahHeap::heap()->complete_marking_context()), _lock(ShenandoahHeap::heap()->lock()) {}
1460 
1461   void heap_region_do(ShenandoahHeapRegion* r) {
1462     if (r->is_active()) {
1463       // All allocations past TAMS are implicitly live, adjust the region data.
1464       // Bitmaps/TAMS are swapped at this point, so we need to poll complete bitmap.
1465       HeapWord *tams = _ctx->top_at_mark_start(r);
1466       HeapWord *top = r->top();
1467       if (top > tams) {
1468         r->increase_live_data_alloc_words(pointer_delta(top, tams));
1469       }
1470 
1471       // We are about to select the collection set, make sure it knows about
1472       // current pinning status. Also, this allows trashing more regions that
1473       // now have their pinning status dropped.
1474       if (r->is_pinned()) {
1475         if (r->pin_count() == 0) {
1476           ShenandoahHeapLocker locker(_lock);
1477           r->make_unpinned();
1478         }
1479       } else {
1480         if (r->pin_count() > 0) {
1481           ShenandoahHeapLocker locker(_lock);
1482           r->make_pinned();
1483         }
1484       }
1485 
1486       // Remember limit for updating refs. It's guaranteed that we get no
1487       // from-space-refs written from here on.
1488       r->set_update_watermark_at_safepoint(r->top());
1489     } else {
1490       assert(!r->has_live(), "Region " SIZE_FORMAT " should have no live data", r->index());
1491       assert(_ctx->top_at_mark_start(r) == r->top(),
1492              "Region " SIZE_FORMAT " should have correct TAMS", r->index());
1493     }
1494   }
1495 
1496   bool is_thread_safe() { return true; }
1497 };
1498 
1499 void ShenandoahHeap::op_final_mark() {
1500   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Should be at safepoint");
1501   assert(!has_forwarded_objects(), "No forwarded objects on this path");
1502 
1503   // It is critical that we
1504   // evacuate roots right after finishing marking, so that we don't
1505   // get unmarked objects in the roots.
1506 
1507   if (!cancelled_gc()) {
1508     concurrent_mark()->finish_mark_from_roots(/* full_gc = */ false);
1509 
1510     // Marking is completed, deactivate SATB barrier
1511     set_concurrent_mark_in_progress(false);
1512     mark_complete_marking_context();
1513 
1514     parallel_cleaning(false /* full gc*/);
1515 
1516     if (ShenandoahVerify) {
1517       verifier()->verify_roots_no_forwarded();
1518     }
1519 
1520     {
1521       ShenandoahGCPhase phase(ShenandoahPhaseTimings::final_update_region_states);
1522       ShenandoahFinalMarkUpdateRegionStateClosure cl;
1523       parallel_heap_region_iterate(&cl);
1524 
1525       assert_pinned_region_status();
1526     }
1527 
1528     // Retire the TLABs, which will force threads to reacquire their TLABs after the pause.
1529     // This is needed for two reasons. Strong one: new allocations would be with new freeset,
1530     // which would be outside the collection set, so no cset writes would happen there.
1531     // Weaker one: new allocations would happen past update watermark, and so less work would
1532     // be needed for reference updates (would update the large filler instead).
1533     {
1534       ShenandoahGCPhase phase(ShenandoahPhaseTimings::retire_tlabs);
1535       make_parsable(true);
1536     }
1537 
1538     {
1539       ShenandoahGCPhase phase(ShenandoahPhaseTimings::choose_cset);
1540       ShenandoahHeapLocker locker(lock());
1541       _collection_set->clear();
1542       heuristics()->choose_collection_set(_collection_set);
1543     }
1544 
1545     {
1546       ShenandoahGCPhase phase(ShenandoahPhaseTimings::final_rebuild_freeset);
1547       ShenandoahHeapLocker locker(lock());
1548       _free_set->rebuild();
1549     }
1550 
1551     if (!is_degenerated_gc_in_progress()) {
1552       prepare_concurrent_roots();
1553       prepare_concurrent_unloading();
1554     }
1555 
1556     // If collection set has candidates, start evacuation.
1557     // Otherwise, bypass the rest of the cycle.
1558     if (!collection_set()->is_empty()) {
1559       ShenandoahGCPhase init_evac(ShenandoahPhaseTimings::init_evac);
1560 
1561       if (ShenandoahVerify) {
1562         verifier()->verify_before_evacuation();
1563       }
1564 
1565       set_evacuation_in_progress(true);
1566       // From here on, we need to update references.
1567       set_has_forwarded_objects(true);
1568 
1569       if (!is_degenerated_gc_in_progress()) {
1570         if (ShenandoahConcurrentRoots::should_do_concurrent_class_unloading()) {
1571           ShenandoahCodeRoots::arm_nmethods();
1572         }
1573         evacuate_and_update_roots();
1574       }
1575 
1576       if (ShenandoahPacing) {
1577         pacer()->setup_for_evac();
1578       }
1579 
1580       if (ShenandoahVerify) {
1581         ShenandoahRootVerifier::RootTypes types = ShenandoahRootVerifier::None;
1582         if (ShenandoahConcurrentRoots::should_do_concurrent_roots()) {
1583           types = ShenandoahRootVerifier::combine(ShenandoahRootVerifier::JNIHandleRoots, ShenandoahRootVerifier::WeakRoots);
1584           types = ShenandoahRootVerifier::combine(types, ShenandoahRootVerifier::CLDGRoots);
1585           types = ShenandoahRootVerifier::combine(types, ShenandoahRootVerifier::StringDedupRoots);
1586         }
1587 
1588         if (ShenandoahConcurrentRoots::should_do_concurrent_class_unloading()) {
1589           types = ShenandoahRootVerifier::combine(types, ShenandoahRootVerifier::CodeRoots);
1590         }
1591         verifier()->verify_roots_no_forwarded_except(types);
1592         verifier()->verify_during_evacuation();
1593       }
1594     } else {
1595       if (ShenandoahVerify) {
1596         verifier()->verify_after_concmark();
1597       }
1598 
1599       if (VerifyAfterGC) {
1600         Universe::verify();
1601       }
1602     }
1603 
1604   } else {
1605     // If this cycle was updating references, we need to keep the has_forwarded_objects
1606     // flag on, for subsequent phases to deal with it.
1607     concurrent_mark()->cancel();
1608     set_concurrent_mark_in_progress(false);
1609 
1610     if (process_references()) {
1611       // Abandon reference processing right away: pre-cleaning must have failed.
1612       ReferenceProcessor *rp = ref_processor();
1613       rp->disable_discovery();
1614       rp->abandon_partial_discovery();
1615       rp->verify_no_references_recorded();
1616     }
1617   }
1618 }
1619 
1620 void ShenandoahHeap::op_conc_evac() {
1621   ShenandoahEvacuationTask task(this, _collection_set, true);
1622   workers()->run_task(&task);
1623 }
1624 
1625 void ShenandoahHeap::op_stw_evac() {
1626   ShenandoahEvacuationTask task(this, _collection_set, false);
1627   workers()->run_task(&task);
1628 }
1629 
1630 void ShenandoahHeap::op_updaterefs() {
1631   update_heap_references(true);
1632 }
1633 
1634 void ShenandoahHeap::op_cleanup_early() {
1635   free_set()->recycle_trash();
1636 }
1637 
1638 void ShenandoahHeap::op_cleanup_complete() {
1639   free_set()->recycle_trash();
1640 }
1641 
1642 class ShenandoahConcurrentRootsEvacUpdateTask : public AbstractGangTask {
1643 private:
1644   ShenandoahVMRoots<true /*concurrent*/>        _vm_roots;
1645   ShenandoahClassLoaderDataRoots<true /*concurrent*/, false /*single threaded*/> _cld_roots;
1646   ShenandoahConcurrentStringDedupRoots          _dedup_roots;
1647 
1648 public:
1649   ShenandoahConcurrentRootsEvacUpdateTask(ShenandoahPhaseTimings::Phase phase) :
1650     AbstractGangTask("Shenandoah Evacuate/Update Concurrent Strong Roots Task"),
1651     _vm_roots(phase),
1652     _cld_roots(phase) {}
1653 
1654   void work(uint worker_id) {
1655     ShenandoahConcurrentWorkerSession worker_session(worker_id);
1656     ShenandoahEvacOOMScope oom;
1657     {
1658       // vm_roots and weak_roots are OopStorage backed roots, concurrent iteration
1659       // may race against OopStorage::release() calls.
1660       ShenandoahEvacUpdateOopStorageRootsClosure cl;
1661       _vm_roots.oops_do<ShenandoahEvacUpdateOopStorageRootsClosure>(&cl, worker_id);
1662     }
1663 
1664     {
1665       ShenandoahEvacuateUpdateRootsClosure<> cl;
1666       CLDToOopClosure clds(&cl, ClassLoaderData::_claim_strong);
1667       _cld_roots.cld_do(&clds, worker_id);
1668     }
1669 
1670     {
1671       ShenandoahForwardedIsAliveClosure is_alive;
1672       ShenandoahEvacuateUpdateRootsClosure<MO_RELEASE> keep_alive;
1673       _dedup_roots.oops_do(&is_alive, &keep_alive, worker_id);
1674     }
1675   }
1676 };
1677 
1678 class ShenandoahEvacUpdateCleanupOopStorageRootsClosure : public BasicOopIterateClosure {
1679 private:
1680   ShenandoahHeap* const _heap;
1681   ShenandoahMarkingContext* const _mark_context;
1682   bool  _evac_in_progress;
1683   Thread* const _thread;
1684   size_t  _dead_counter;
1685 
1686 public:
1687   ShenandoahEvacUpdateCleanupOopStorageRootsClosure();
1688   void do_oop(oop* p);
1689   void do_oop(narrowOop* p);
1690 
1691   size_t dead_counter() const;
1692   void reset_dead_counter();
1693 };
1694 
1695 ShenandoahEvacUpdateCleanupOopStorageRootsClosure::ShenandoahEvacUpdateCleanupOopStorageRootsClosure() :
1696   _heap(ShenandoahHeap::heap()),
1697   _mark_context(ShenandoahHeap::heap()->marking_context()),
1698   _evac_in_progress(ShenandoahHeap::heap()->is_evacuation_in_progress()),
1699   _thread(Thread::current()),
1700   _dead_counter(0) {
1701 }
1702 
1703 void ShenandoahEvacUpdateCleanupOopStorageRootsClosure::do_oop(oop* p) {
1704   const oop obj = RawAccess<>::oop_load(p);
1705   if (!CompressedOops::is_null(obj)) {
1706     if (!_mark_context->is_marked(obj)) {
1707       shenandoah_assert_correct(p, obj);
1708       oop old = Atomic::cmpxchg(p, obj, oop(NULL));
1709       if (obj == old) {
1710         _dead_counter ++;
1711       }
1712     } else if (_evac_in_progress && _heap->in_collection_set(obj)) {
1713       oop resolved = ShenandoahBarrierSet::resolve_forwarded_not_null(obj);
1714       if (resolved == obj) {
1715         resolved = _heap->evacuate_object(obj, _thread);
1716       }
1717       Atomic::cmpxchg(p, obj, resolved);
1718       assert(_heap->cancelled_gc() ||
1719              _mark_context->is_marked(resolved) && !_heap->in_collection_set(resolved),
1720              "Sanity");
1721     }
1722   }
1723 }
1724 
1725 void ShenandoahEvacUpdateCleanupOopStorageRootsClosure::do_oop(narrowOop* p) {
1726   ShouldNotReachHere();
1727 }
1728 
1729 size_t ShenandoahEvacUpdateCleanupOopStorageRootsClosure::dead_counter() const {
1730   return _dead_counter;
1731 }
1732 
1733 void ShenandoahEvacUpdateCleanupOopStorageRootsClosure::reset_dead_counter() {
1734   _dead_counter = 0;
1735 }
1736 
1737 class ShenandoahIsCLDAliveClosure : public CLDClosure {
1738 public:
1739   void do_cld(ClassLoaderData* cld) {
1740     cld->is_alive();
1741   }
1742 };
1743 
1744 class ShenandoahIsNMethodAliveClosure: public NMethodClosure {
1745 public:
1746   void do_nmethod(nmethod* n) {
1747     n->is_unloading();
1748   }
1749 };
1750 
1751 // This task not only evacuates/updates marked weak roots, but also "NULL"
1752 // dead weak roots.
1753 class ShenandoahConcurrentWeakRootsEvacUpdateTask : public AbstractGangTask {
1754 private:
1755   ShenandoahWeakRoot<true /*concurrent*/>  _jni_roots;
1756   ShenandoahWeakRoot<true /*concurrent*/>  _string_table_roots;
1757   ShenandoahWeakRoot<true /*concurrent*/>  _resolved_method_table_roots;
1758   ShenandoahWeakRoot<true /*concurrent*/>  _vm_roots;
1759 
1760   // Roots related to concurrent class unloading
1761   ShenandoahClassLoaderDataRoots<true /* concurrent */, false /* single thread*/>
1762                                            _cld_roots;
1763   ShenandoahConcurrentNMethodIterator      _nmethod_itr;
1764   bool                                     _concurrent_class_unloading;
1765 
1766 public:
1767   ShenandoahConcurrentWeakRootsEvacUpdateTask(ShenandoahPhaseTimings::Phase phase) :
1768     AbstractGangTask("Shenandoah Concurrent Weak Root Task"),
1769     _jni_roots(OopStorageSet::jni_weak(), phase, ShenandoahPhaseTimings::JNIWeakRoots),
1770     _string_table_roots(OopStorageSet::string_table_weak(), phase, ShenandoahPhaseTimings::StringTableRoots),
1771     _resolved_method_table_roots(OopStorageSet::resolved_method_table_weak(), phase, ShenandoahPhaseTimings::ResolvedMethodTableRoots),
1772     _vm_roots(OopStorageSet::vm_weak(), phase, ShenandoahPhaseTimings::VMWeakRoots),
1773     _cld_roots(phase),
1774     _nmethod_itr(ShenandoahCodeRoots::table()),
1775     _concurrent_class_unloading(ShenandoahConcurrentRoots::should_do_concurrent_class_unloading()) {
1776     StringTable::reset_dead_counter();
1777     ResolvedMethodTable::reset_dead_counter();
1778     if (_concurrent_class_unloading) {
1779       MutexLocker mu(CodeCache_lock, Mutex::_no_safepoint_check_flag);
1780       _nmethod_itr.nmethods_do_begin();
1781     }
1782   }
1783 
1784   ~ShenandoahConcurrentWeakRootsEvacUpdateTask() {
1785     StringTable::finish_dead_counter();
1786     ResolvedMethodTable::finish_dead_counter();
1787     if (_concurrent_class_unloading) {
1788       MutexLocker mu(CodeCache_lock, Mutex::_no_safepoint_check_flag);
1789       _nmethod_itr.nmethods_do_end();
1790     }
1791   }
1792 
1793   void work(uint worker_id) {
1794     ShenandoahConcurrentWorkerSession worker_session(worker_id);
1795     {
1796       ShenandoahEvacOOMScope oom;
1797       // jni_roots and weak_roots are OopStorage backed roots, concurrent iteration
1798       // may race against OopStorage::release() calls.
1799       ShenandoahEvacUpdateCleanupOopStorageRootsClosure cl;
1800       _jni_roots.oops_do(&cl, worker_id);
1801       _vm_roots.oops_do(&cl, worker_id);
1802 
1803       cl.reset_dead_counter();
1804       _string_table_roots.oops_do(&cl, worker_id);
1805       StringTable::inc_dead_counter(cl.dead_counter());
1806 
1807       cl.reset_dead_counter();
1808       _resolved_method_table_roots.oops_do(&cl, worker_id);
1809       ResolvedMethodTable::inc_dead_counter(cl.dead_counter());
1810     }
1811 
1812     // If we are going to perform concurrent class unloading later on, we need to
1813     // cleanup the weak oops in CLD and determinate nmethod's unloading state, so that we
1814     // can cleanup immediate garbage sooner.
1815     if (_concurrent_class_unloading) {
1816       // Applies ShenandoahIsCLDAlive closure to CLDs, native barrier will either NULL the
1817       // CLD's holder or evacuate it.
1818       ShenandoahIsCLDAliveClosure is_cld_alive;
1819       _cld_roots.cld_do(&is_cld_alive, worker_id);
1820 
1821       // Applies ShenandoahIsNMethodAliveClosure to registered nmethods.
1822       // The closure calls nmethod->is_unloading(). The is_unloading
1823       // state is cached, therefore, during concurrent class unloading phase,
1824       // we will not touch the metadata of unloading nmethods
1825       ShenandoahIsNMethodAliveClosure is_nmethod_alive;
1826       _nmethod_itr.nmethods_do(&is_nmethod_alive);
1827     }
1828   }
1829 };
1830 
1831 void ShenandoahHeap::op_weak_roots() {
1832   if (is_concurrent_weak_root_in_progress()) {
1833     {
1834       // Concurrent weak root processing
1835       ShenandoahGCWorkerPhase worker_phase(ShenandoahPhaseTimings::conc_weak_roots_work);
1836       ShenandoahConcurrentWeakRootsEvacUpdateTask task(ShenandoahPhaseTimings::conc_weak_roots_work);
1837       workers()->run_task(&task);
1838       if (!ShenandoahConcurrentRoots::should_do_concurrent_class_unloading()) {
1839         set_concurrent_weak_root_in_progress(false);
1840       }
1841     }
1842 
1843     // Perform handshake to flush out dead oops
1844     {
1845       ShenandoahTimingsTracker t(ShenandoahPhaseTimings::conc_weak_roots_rendezvous);
1846       ShenandoahRendezvousClosure cl;
1847       Handshake::execute(&cl);
1848     }
1849   }
1850 }
1851 
1852 void ShenandoahHeap::op_class_unloading() {
1853   assert (is_concurrent_weak_root_in_progress() &&
1854           ShenandoahConcurrentRoots::should_do_concurrent_class_unloading(),
1855           "Checked by caller");
1856   _unloader.unload();
1857   set_concurrent_weak_root_in_progress(false);
1858 }
1859 
1860 void ShenandoahHeap::op_strong_roots() {
1861   assert(is_concurrent_strong_root_in_progress(), "Checked by caller");
1862   ShenandoahConcurrentRootsEvacUpdateTask task(ShenandoahPhaseTimings::conc_strong_roots);
1863   workers()->run_task(&task);
1864   set_concurrent_strong_root_in_progress(false);
1865 }
1866 
1867 class ShenandoahResetUpdateRegionStateClosure : public ShenandoahHeapRegionClosure {
1868 private:
1869   ShenandoahMarkingContext* const _ctx;
1870 public:
1871   ShenandoahResetUpdateRegionStateClosure() : _ctx(ShenandoahHeap::heap()->marking_context()) {}
1872 
1873   void heap_region_do(ShenandoahHeapRegion* r) {
1874     if (r->is_active()) {
1875       // Reset live data and set TAMS optimistically. We would recheck these under the pause
1876       // anyway to capture any updates that happened since now.
1877       r->clear_live_data();
1878       _ctx->capture_top_at_mark_start(r);
1879     }
1880   }
1881 
1882   bool is_thread_safe() { return true; }
1883 };
1884 
1885 void ShenandoahHeap::op_reset() {
1886   if (ShenandoahPacing) {
1887     pacer()->setup_for_reset();
1888   }
1889   reset_mark_bitmap();
1890 
1891   ShenandoahResetUpdateRegionStateClosure cl;
1892   parallel_heap_region_iterate(&cl);
1893 }
1894 
1895 void ShenandoahHeap::op_preclean() {
1896   if (ShenandoahPacing) {
1897     pacer()->setup_for_preclean();
1898   }
1899   concurrent_mark()->preclean_weak_refs();
1900 }
1901 
1902 void ShenandoahHeap::op_full(GCCause::Cause cause) {
1903   ShenandoahMetricsSnapshot metrics;
1904   metrics.snap_before();
1905 
1906   full_gc()->do_it(cause);
1907   if (UseTLAB) {
1908     ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_resize_tlabs);
1909     resize_all_tlabs();
1910   }
1911 
1912   metrics.snap_after();
1913 
1914   if (metrics.is_good_progress()) {
1915     _progress_last_gc.set();
1916   } else {
1917     // Nothing to do. Tell the allocation path that we have failed to make
1918     // progress, and it can finally fail.
1919     _progress_last_gc.unset();
1920   }
1921 }
1922 
1923 void ShenandoahHeap::op_degenerated(ShenandoahDegenPoint point) {
1924   // Degenerated GC is STW, but it can also fail. Current mechanics communicates
1925   // GC failure via cancelled_concgc() flag. So, if we detect the failure after
1926   // some phase, we have to upgrade the Degenerate GC to Full GC.
1927 
1928   clear_cancelled_gc();
1929 
1930   ShenandoahMetricsSnapshot metrics;
1931   metrics.snap_before();
1932 
1933   switch (point) {
1934     // The cases below form the Duff's-like device: it describes the actual GC cycle,
1935     // but enters it at different points, depending on which concurrent phase had
1936     // degenerated.
1937 
1938     case _degenerated_outside_cycle:
1939       // We have degenerated from outside the cycle, which means something is bad with
1940       // the heap, most probably heavy humongous fragmentation, or we are very low on free
1941       // space. It makes little sense to wait for Full GC to reclaim as much as it can, when
1942       // we can do the most aggressive degen cycle, which includes processing references and
1943       // class unloading, unless those features are explicitly disabled.
1944       //
1945       // Note that we can only do this for "outside-cycle" degens, otherwise we would risk
1946       // changing the cycle parameters mid-cycle during concurrent -> degenerated handover.
1947       set_process_references(heuristics()->can_process_references());
1948       set_unload_classes(heuristics()->can_unload_classes());
1949 
1950       op_reset();
1951 
1952       op_init_mark();
1953       if (cancelled_gc()) {
1954         op_degenerated_fail();
1955         return;
1956       }
1957 
1958     case _degenerated_mark:
1959       op_final_mark();
1960       if (cancelled_gc()) {
1961         op_degenerated_fail();
1962         return;
1963       }
1964 
1965       if (!has_forwarded_objects() && ShenandoahConcurrentRoots::can_do_concurrent_class_unloading()) {
1966         // Disarm nmethods that armed for concurrent mark. On normal cycle, it would
1967         // be disarmed while conc-roots phase is running.
1968         // TODO: Call op_conc_roots() here instead
1969         ShenandoahCodeRoots::disarm_nmethods();
1970       }
1971 
1972       op_cleanup_early();
1973 
1974     case _degenerated_evac:
1975       // If heuristics thinks we should do the cycle, this flag would be set,
1976       // and we can do evacuation. Otherwise, it would be the shortcut cycle.
1977       if (is_evacuation_in_progress()) {
1978 
1979         // Degeneration under oom-evac protocol might have left some objects in
1980         // collection set un-evacuated. Restart evacuation from the beginning to
1981         // capture all objects. For all the objects that are already evacuated,
1982         // it would be a simple check, which is supposed to be fast. This is also
1983         // safe to do even without degeneration, as CSet iterator is at beginning
1984         // in preparation for evacuation anyway.
1985         //
1986         // Before doing that, we need to make sure we never had any cset-pinned
1987         // regions. This may happen if allocation failure happened when evacuating
1988         // the about-to-be-pinned object, oom-evac protocol left the object in
1989         // the collection set, and then the pin reached the cset region. If we continue
1990         // the cycle here, we would trash the cset and alive objects in it. To avoid
1991         // it, we fail degeneration right away and slide into Full GC to recover.
1992 
1993         {
1994           sync_pinned_region_status();
1995           collection_set()->clear_current_index();
1996 
1997           ShenandoahHeapRegion* r;
1998           while ((r = collection_set()->next()) != NULL) {
1999             if (r->is_pinned()) {
2000               cancel_gc(GCCause::_shenandoah_upgrade_to_full_gc);
2001               op_degenerated_fail();
2002               return;
2003             }
2004           }
2005 
2006           collection_set()->clear_current_index();
2007         }
2008 
2009         op_stw_evac();
2010         if (cancelled_gc()) {
2011           op_degenerated_fail();
2012           return;
2013         }
2014       }
2015 
2016       // If heuristics thinks we should do the cycle, this flag would be set,
2017       // and we need to do update-refs. Otherwise, it would be the shortcut cycle.
2018       if (has_forwarded_objects()) {
2019         op_init_updaterefs();
2020         if (cancelled_gc()) {
2021           op_degenerated_fail();
2022           return;
2023         }
2024       }
2025 
2026     case _degenerated_updaterefs:
2027       if (has_forwarded_objects()) {
2028         op_final_updaterefs();
2029         if (cancelled_gc()) {
2030           op_degenerated_fail();
2031           return;
2032         }
2033       }
2034 
2035       op_cleanup_complete();
2036       break;
2037 
2038     default:
2039       ShouldNotReachHere();
2040   }
2041 
2042   if (ShenandoahVerify) {
2043     verifier()->verify_after_degenerated();
2044   }
2045 
2046   if (VerifyAfterGC) {
2047     Universe::verify();
2048   }
2049 
2050   metrics.snap_after();
2051 
2052   // Check for futility and fail. There is no reason to do several back-to-back Degenerated cycles,
2053   // because that probably means the heap is overloaded and/or fragmented.
2054   if (!metrics.is_good_progress()) {
2055     _progress_last_gc.unset();
2056     cancel_gc(GCCause::_shenandoah_upgrade_to_full_gc);
2057     op_degenerated_futile();
2058   } else {
2059     _progress_last_gc.set();
2060   }
2061 }
2062 
2063 void ShenandoahHeap::op_degenerated_fail() {
2064   log_info(gc)("Cannot finish degeneration, upgrading to Full GC");
2065   shenandoah_policy()->record_degenerated_upgrade_to_full();
2066   op_full(GCCause::_shenandoah_upgrade_to_full_gc);
2067 }
2068 
2069 void ShenandoahHeap::op_degenerated_futile() {
2070   shenandoah_policy()->record_degenerated_upgrade_to_full();
2071   op_full(GCCause::_shenandoah_upgrade_to_full_gc);
2072 }
2073 
2074 void ShenandoahHeap::force_satb_flush_all_threads() {
2075   if (!is_concurrent_mark_in_progress()) {
2076     // No need to flush SATBs
2077     return;
2078   }
2079 
2080   for (JavaThreadIteratorWithHandle jtiwh; JavaThread *t = jtiwh.next(); ) {
2081     ShenandoahThreadLocalData::set_force_satb_flush(t, true);
2082   }
2083   // The threads are not "acquiring" their thread-local data, but it does not
2084   // hurt to "release" the updates here anyway.
2085   OrderAccess::fence();
2086 }
2087 
2088 void ShenandoahHeap::set_gc_state_all_threads(char state) {
2089   for (JavaThreadIteratorWithHandle jtiwh; JavaThread *t = jtiwh.next(); ) {
2090     ShenandoahThreadLocalData::set_gc_state(t, state);
2091   }
2092 }
2093 
2094 void ShenandoahHeap::set_gc_state_mask(uint mask, bool value) {
2095   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Should really be Shenandoah safepoint");
2096   _gc_state.set_cond(mask, value);
2097   set_gc_state_all_threads(_gc_state.raw_value());
2098 }
2099 
2100 void ShenandoahHeap::set_concurrent_mark_in_progress(bool in_progress) {
2101   if (has_forwarded_objects()) {
2102     set_gc_state_mask(MARKING | UPDATEREFS, in_progress);
2103   } else {
2104     set_gc_state_mask(MARKING, in_progress);
2105   }
2106   ShenandoahBarrierSet::satb_mark_queue_set().set_active_all_threads(in_progress, !in_progress);
2107 }
2108 
2109 void ShenandoahHeap::set_evacuation_in_progress(bool in_progress) {
2110   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Only call this at safepoint");
2111   set_gc_state_mask(EVACUATION, in_progress);
2112 }
2113 
2114 void ShenandoahHeap::set_concurrent_strong_root_in_progress(bool in_progress) {
2115   assert(ShenandoahConcurrentRoots::can_do_concurrent_roots(), "Why set the flag?");
2116   if (in_progress) {
2117     _concurrent_strong_root_in_progress.set();
2118   } else {
2119     _concurrent_strong_root_in_progress.unset();
2120   }
2121 }
2122 
2123 void ShenandoahHeap::set_concurrent_weak_root_in_progress(bool in_progress) {
2124   assert(ShenandoahConcurrentRoots::can_do_concurrent_roots(), "Why set the flag?");
2125   if (in_progress) {
2126     _concurrent_weak_root_in_progress.set();
2127   } else {
2128     _concurrent_weak_root_in_progress.unset();
2129   }
2130 }
2131 
2132 void ShenandoahHeap::ref_processing_init() {
2133   assert(_max_workers > 0, "Sanity");
2134 
2135   bool mt_processing = ParallelRefProcEnabled && (ParallelGCThreads > 1);
2136   bool mt_discovery = _max_workers > 1;
2137 
2138   _ref_processor =
2139     new ReferenceProcessor(&_subject_to_discovery,  // is_subject_to_discovery
2140                            mt_processing,           // MT processing
2141                            _max_workers,            // Degree of MT processing
2142                            mt_discovery,            // MT discovery
2143                            _max_workers,            // Degree of MT discovery
2144                            false,                   // Reference discovery is not atomic
2145                            NULL,                    // No closure, should be installed before use
2146                            true);                   // Scale worker threads
2147 
2148   log_info(gc, init)("Reference processing: %s discovery, %s processing",
2149           mt_discovery ? "parallel" : "serial",
2150           mt_processing ? "parallel" : "serial");
2151 
2152   shenandoah_assert_rp_isalive_not_installed();
2153 }
2154 
2155 GCTracer* ShenandoahHeap::tracer() {
2156   return shenandoah_policy()->tracer();
2157 }
2158 
2159 size_t ShenandoahHeap::tlab_used(Thread* thread) const {
2160   return _free_set->used();
2161 }
2162 
2163 bool ShenandoahHeap::try_cancel_gc() {
2164   while (true) {
2165     jbyte prev = _cancelled_gc.cmpxchg(CANCELLED, CANCELLABLE);
2166     if (prev == CANCELLABLE) return true;
2167     else if (prev == CANCELLED) return false;
2168     assert(ShenandoahSuspendibleWorkers, "should not get here when not using suspendible workers");
2169     assert(prev == NOT_CANCELLED, "must be NOT_CANCELLED");
2170     if (Thread::current()->is_Java_thread()) {
2171       // We need to provide a safepoint here, otherwise we might
2172       // spin forever if a SP is pending.
2173       ThreadBlockInVM sp(JavaThread::current());
2174       SpinPause();
2175     }
2176   }
2177 }
2178 
2179 void ShenandoahHeap::cancel_gc(GCCause::Cause cause) {
2180   if (try_cancel_gc()) {
2181     FormatBuffer<> msg("Cancelling GC: %s", GCCause::to_string(cause));
2182     log_info(gc)("%s", msg.buffer());
2183     Events::log(Thread::current(), "%s", msg.buffer());
2184   }
2185 }
2186 
2187 uint ShenandoahHeap::max_workers() {
2188   return _max_workers;
2189 }
2190 
2191 void ShenandoahHeap::stop() {
2192   // The shutdown sequence should be able to terminate when GC is running.
2193 
2194   // Step 0. Notify policy to disable event recording.
2195   _shenandoah_policy->record_shutdown();
2196 
2197   // Step 1. Notify control thread that we are in shutdown.
2198   // Note that we cannot do that with stop(), because stop() is blocking and waits for the actual shutdown.
2199   // Doing stop() here would wait for the normal GC cycle to complete, never falling through to cancel below.
2200   control_thread()->prepare_for_graceful_shutdown();
2201 
2202   // Step 2. Notify GC workers that we are cancelling GC.
2203   cancel_gc(GCCause::_shenandoah_stop_vm);
2204 
2205   // Step 3. Wait until GC worker exits normally.
2206   control_thread()->stop();
2207 
2208   // Step 4. Stop String Dedup thread if it is active
2209   if (ShenandoahStringDedup::is_enabled()) {
2210     ShenandoahStringDedup::stop();
2211   }
2212 }
2213 
2214 void ShenandoahHeap::stw_unload_classes(bool full_gc) {
2215   if (!unload_classes()) return;
2216 
2217   // Unload classes and purge SystemDictionary.
2218   {
2219     ShenandoahGCPhase phase(full_gc ?
2220                             ShenandoahPhaseTimings::full_gc_purge_class_unload :
2221                             ShenandoahPhaseTimings::purge_class_unload);
2222     bool purged_class = SystemDictionary::do_unloading(gc_timer());
2223 
2224     ShenandoahIsAliveSelector is_alive;
2225     uint num_workers = _workers->active_workers();
2226     ShenandoahClassUnloadingTask unlink_task(is_alive.is_alive_closure(), num_workers, purged_class);
2227     _workers->run_task(&unlink_task);
2228   }
2229 
2230   {
2231     ShenandoahGCPhase phase(full_gc ?
2232                             ShenandoahPhaseTimings::full_gc_purge_cldg :
2233                             ShenandoahPhaseTimings::purge_cldg);
2234     ClassLoaderDataGraph::purge();
2235   }
2236   // Resize and verify metaspace
2237   MetaspaceGC::compute_new_size();
2238   MetaspaceUtils::verify_metrics();
2239 }
2240 
2241 // Weak roots are either pre-evacuated (final mark) or updated (final updaterefs),
2242 // so they should not have forwarded oops.
2243 // However, we do need to "null" dead oops in the roots, if can not be done
2244 // in concurrent cycles.
2245 void ShenandoahHeap::stw_process_weak_roots(bool full_gc) {
2246   ShenandoahGCPhase root_phase(full_gc ?
2247                                ShenandoahPhaseTimings::full_gc_purge :
2248                                ShenandoahPhaseTimings::purge);
2249   uint num_workers = _workers->active_workers();
2250   ShenandoahPhaseTimings::Phase timing_phase = full_gc ?
2251                                                ShenandoahPhaseTimings::full_gc_purge_weak_par :
2252                                                ShenandoahPhaseTimings::purge_weak_par;
2253   ShenandoahGCPhase phase(timing_phase);
2254   ShenandoahGCWorkerPhase worker_phase(timing_phase);
2255 
2256   // Cleanup weak roots
2257   if (has_forwarded_objects()) {
2258     ShenandoahForwardedIsAliveClosure is_alive;
2259     ShenandoahUpdateRefsClosure keep_alive;
2260     ShenandoahParallelWeakRootsCleaningTask<ShenandoahForwardedIsAliveClosure, ShenandoahUpdateRefsClosure>
2261       cleaning_task(timing_phase, &is_alive, &keep_alive, num_workers, !ShenandoahConcurrentRoots::should_do_concurrent_class_unloading());
2262     _workers->run_task(&cleaning_task);
2263   } else {
2264     ShenandoahIsAliveClosure is_alive;
2265 #ifdef ASSERT
2266     ShenandoahAssertNotForwardedClosure verify_cl;
2267     ShenandoahParallelWeakRootsCleaningTask<ShenandoahIsAliveClosure, ShenandoahAssertNotForwardedClosure>
2268       cleaning_task(timing_phase, &is_alive, &verify_cl, num_workers, !ShenandoahConcurrentRoots::should_do_concurrent_class_unloading());
2269 #else
2270     ShenandoahParallelWeakRootsCleaningTask<ShenandoahIsAliveClosure, DoNothingClosure>
2271       cleaning_task(timing_phase, &is_alive, &do_nothing_cl, num_workers, !ShenandoahConcurrentRoots::should_do_concurrent_class_unloading());
2272 #endif
2273     _workers->run_task(&cleaning_task);
2274   }
2275 }
2276 
2277 void ShenandoahHeap::parallel_cleaning(bool full_gc) {
2278   assert(SafepointSynchronize::is_at_safepoint(), "Must be at a safepoint");
2279   stw_process_weak_roots(full_gc);
2280   if (!ShenandoahConcurrentRoots::should_do_concurrent_class_unloading()) {
2281     stw_unload_classes(full_gc);
2282   }
2283 }
2284 
2285 void ShenandoahHeap::set_has_forwarded_objects(bool cond) {
2286   set_gc_state_mask(HAS_FORWARDED, cond);
2287 }
2288 
2289 void ShenandoahHeap::set_process_references(bool pr) {
2290   _process_references.set_cond(pr);
2291 }
2292 
2293 void ShenandoahHeap::set_unload_classes(bool uc) {
2294   _unload_classes.set_cond(uc);
2295 }
2296 
2297 bool ShenandoahHeap::process_references() const {
2298   return _process_references.is_set();
2299 }
2300 
2301 bool ShenandoahHeap::unload_classes() const {
2302   return _unload_classes.is_set();
2303 }
2304 
2305 address ShenandoahHeap::in_cset_fast_test_addr() {
2306   ShenandoahHeap* heap = ShenandoahHeap::heap();
2307   assert(heap->collection_set() != NULL, "Sanity");
2308   return (address) heap->collection_set()->biased_map_address();
2309 }
2310 
2311 address ShenandoahHeap::cancelled_gc_addr() {
2312   return (address) ShenandoahHeap::heap()->_cancelled_gc.addr_of();
2313 }
2314 
2315 address ShenandoahHeap::gc_state_addr() {
2316   return (address) ShenandoahHeap::heap()->_gc_state.addr_of();
2317 }
2318 
2319 size_t ShenandoahHeap::bytes_allocated_since_gc_start() {
2320   return Atomic::load_acquire(&_bytes_allocated_since_gc_start);
2321 }
2322 
2323 void ShenandoahHeap::reset_bytes_allocated_since_gc_start() {
2324   Atomic::release_store_fence(&_bytes_allocated_since_gc_start, (size_t)0);
2325 }
2326 
2327 void ShenandoahHeap::set_degenerated_gc_in_progress(bool in_progress) {
2328   _degenerated_gc_in_progress.set_cond(in_progress);
2329 }
2330 
2331 void ShenandoahHeap::set_full_gc_in_progress(bool in_progress) {
2332   _full_gc_in_progress.set_cond(in_progress);
2333 }
2334 
2335 void ShenandoahHeap::set_full_gc_move_in_progress(bool in_progress) {
2336   assert (is_full_gc_in_progress(), "should be");
2337   _full_gc_move_in_progress.set_cond(in_progress);
2338 }
2339 
2340 void ShenandoahHeap::set_update_refs_in_progress(bool in_progress) {
2341   set_gc_state_mask(UPDATEREFS, in_progress);
2342 }
2343 
2344 void ShenandoahHeap::register_nmethod(nmethod* nm) {
2345   ShenandoahCodeRoots::register_nmethod(nm);
2346 }
2347 
2348 void ShenandoahHeap::unregister_nmethod(nmethod* nm) {
2349   ShenandoahCodeRoots::unregister_nmethod(nm);
2350 }
2351 
2352 void ShenandoahHeap::flush_nmethod(nmethod* nm) {
2353   ShenandoahCodeRoots::flush_nmethod(nm);
2354 }
2355 
2356 oop ShenandoahHeap::pin_object(JavaThread* thr, oop o) {
2357   heap_region_containing(o)->record_pin();
2358   return o;
2359 }
2360 
2361 void ShenandoahHeap::unpin_object(JavaThread* thr, oop o) {
2362   heap_region_containing(o)->record_unpin();
2363 }
2364 
2365 void ShenandoahHeap::sync_pinned_region_status() {
2366   ShenandoahHeapLocker locker(lock());
2367 
2368   for (size_t i = 0; i < num_regions(); i++) {
2369     ShenandoahHeapRegion *r = get_region(i);
2370     if (r->is_active()) {
2371       if (r->is_pinned()) {
2372         if (r->pin_count() == 0) {
2373           r->make_unpinned();
2374         }
2375       } else {
2376         if (r->pin_count() > 0) {
2377           r->make_pinned();
2378         }
2379       }
2380     }
2381   }
2382 
2383   assert_pinned_region_status();
2384 }
2385 
2386 #ifdef ASSERT
2387 void ShenandoahHeap::assert_pinned_region_status() {
2388   for (size_t i = 0; i < num_regions(); i++) {
2389     ShenandoahHeapRegion* r = get_region(i);
2390     assert((r->is_pinned() && r->pin_count() > 0) || (!r->is_pinned() && r->pin_count() == 0),
2391            "Region " SIZE_FORMAT " pinning status is inconsistent", i);
2392   }
2393 }
2394 #endif
2395 
2396 ConcurrentGCTimer* ShenandoahHeap::gc_timer() const {
2397   return _gc_timer;
2398 }
2399 
2400 void ShenandoahHeap::prepare_concurrent_roots() {
2401   assert(SafepointSynchronize::is_at_safepoint(), "Must be at a safepoint");
2402   if (ShenandoahConcurrentRoots::should_do_concurrent_roots()) {
2403     set_concurrent_strong_root_in_progress(!collection_set()->is_empty());
2404     set_concurrent_weak_root_in_progress(true);
2405   }
2406 }
2407 
2408 void ShenandoahHeap::prepare_concurrent_unloading() {
2409   assert(SafepointSynchronize::is_at_safepoint(), "Must be at a safepoint");
2410   if (ShenandoahConcurrentRoots::should_do_concurrent_class_unloading()) {
2411     _unloader.prepare();
2412   }
2413 }
2414 
2415 void ShenandoahHeap::finish_concurrent_unloading() {
2416   assert(SafepointSynchronize::is_at_safepoint(), "Must be at a safepoint");
2417   if (ShenandoahConcurrentRoots::should_do_concurrent_class_unloading()) {
2418     _unloader.finish();
2419   }
2420 }
2421 
2422 #ifdef ASSERT
2423 void ShenandoahHeap::assert_gc_workers(uint nworkers) {
2424   assert(nworkers > 0 && nworkers <= max_workers(), "Sanity");
2425 
2426   if (ShenandoahSafepoint::is_at_shenandoah_safepoint()) {
2427     if (UseDynamicNumberOfGCThreads) {
2428       assert(nworkers <= ParallelGCThreads, "Cannot use more than it has");
2429     } else {
2430       // Use ParallelGCThreads inside safepoints
2431       assert(nworkers == ParallelGCThreads, "Use ParallelGCThreads within safepoints");
2432     }
2433   } else {
2434     if (UseDynamicNumberOfGCThreads) {
2435       assert(nworkers <= ConcGCThreads, "Cannot use more than it has");
2436     } else {
2437       // Use ConcGCThreads outside safepoints
2438       assert(nworkers == ConcGCThreads, "Use ConcGCThreads outside safepoints");
2439     }
2440   }
2441 }
2442 #endif
2443 
2444 ShenandoahVerifier* ShenandoahHeap::verifier() {
2445   guarantee(ShenandoahVerify, "Should be enabled");
2446   assert (_verifier != NULL, "sanity");
2447   return _verifier;
2448 }
2449 
2450 template<class T>
2451 class ShenandoahUpdateHeapRefsTask : public AbstractGangTask {
2452 private:
2453   T cl;
2454   ShenandoahHeap* _heap;
2455   ShenandoahRegionIterator* _regions;
2456   bool _concurrent;
2457 public:
2458   ShenandoahUpdateHeapRefsTask(ShenandoahRegionIterator* regions, bool concurrent) :
2459     AbstractGangTask("Concurrent Update References Task"),
2460     cl(T()),
2461     _heap(ShenandoahHeap::heap()),
2462     _regions(regions),
2463     _concurrent(concurrent) {
2464   }
2465 
2466   void work(uint worker_id) {
2467     if (_concurrent) {
2468       ShenandoahConcurrentWorkerSession worker_session(worker_id);
2469       ShenandoahSuspendibleThreadSetJoiner stsj(ShenandoahSuspendibleWorkers);
2470       do_work();
2471     } else {
2472       ShenandoahParallelWorkerSession worker_session(worker_id);
2473       do_work();
2474     }
2475   }
2476 
2477 private:
2478   void do_work() {
2479     ShenandoahHeapRegion* r = _regions->next();
2480     ShenandoahMarkingContext* const ctx = _heap->complete_marking_context();
2481     while (r != NULL) {
2482       HeapWord* update_watermark = r->get_update_watermark();
2483       assert (update_watermark >= r->bottom(), "sanity");
2484       if (r->is_active() && !r->is_cset()) {
2485         _heap->marked_object_oop_iterate(r, &cl, update_watermark);
2486       }
2487       if (ShenandoahPacing) {
2488         _heap->pacer()->report_updaterefs(pointer_delta(update_watermark, r->bottom()));
2489       }
2490       if (_heap->check_cancelled_gc_and_yield(_concurrent)) {
2491         return;
2492       }
2493       r = _regions->next();
2494     }
2495   }
2496 };
2497 
2498 void ShenandoahHeap::update_heap_references(bool concurrent) {
2499   ShenandoahUpdateHeapRefsTask<ShenandoahUpdateHeapRefsClosure> task(&_update_refs_iterator, concurrent);
2500   workers()->run_task(&task);
2501 }
2502 
2503 void ShenandoahHeap::op_init_updaterefs() {
2504   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "must be at safepoint");
2505 
2506   set_evacuation_in_progress(false);
2507 
2508   {
2509     ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_update_refs_retire_gclabs);
2510     retire_and_reset_gclabs();
2511   }
2512 
2513   if (ShenandoahVerify) {
2514     if (!is_degenerated_gc_in_progress()) {
2515       verifier()->verify_roots_in_to_space_except(ShenandoahRootVerifier::ThreadRoots);
2516     }
2517     verifier()->verify_before_updaterefs();
2518   }
2519 
2520   set_update_refs_in_progress(true);
2521 
2522   _update_refs_iterator.reset();
2523 
2524   if (ShenandoahPacing) {
2525     pacer()->setup_for_updaterefs();
2526   }
2527 }
2528 
2529 class ShenandoahFinalUpdateRefsUpdateRegionStateClosure : public ShenandoahHeapRegionClosure {
2530 private:
2531   ShenandoahHeapLock* const _lock;
2532 
2533 public:
2534   ShenandoahFinalUpdateRefsUpdateRegionStateClosure() : _lock(ShenandoahHeap::heap()->lock()) {}
2535 
2536   void heap_region_do(ShenandoahHeapRegion* r) {
2537     // Drop unnecessary "pinned" state from regions that does not have CP marks
2538     // anymore, as this would allow trashing them.
2539 
2540     if (r->is_active()) {
2541       if (r->is_pinned()) {
2542         if (r->pin_count() == 0) {
2543           ShenandoahHeapLocker locker(_lock);
2544           r->make_unpinned();
2545         }
2546       } else {
2547         if (r->pin_count() > 0) {
2548           ShenandoahHeapLocker locker(_lock);
2549           r->make_pinned();
2550         }
2551       }
2552     }
2553   }
2554 
2555   bool is_thread_safe() { return true; }
2556 };
2557 
2558 void ShenandoahHeap::op_final_updaterefs() {
2559   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "must be at safepoint");
2560 
2561   finish_concurrent_unloading();
2562 
2563   // Check if there is left-over work, and finish it
2564   if (_update_refs_iterator.has_next()) {
2565     ShenandoahGCPhase phase(ShenandoahPhaseTimings::final_update_refs_finish_work);
2566 
2567     // Finish updating references where we left off.
2568     clear_cancelled_gc();
2569     update_heap_references(false);
2570   }
2571 
2572   // Clear cancelled GC, if set. On cancellation path, the block before would handle
2573   // everything. On degenerated paths, cancelled gc would not be set anyway.
2574   if (cancelled_gc()) {
2575     clear_cancelled_gc();
2576   }
2577   assert(!cancelled_gc(), "Should have been done right before");
2578 
2579   if (ShenandoahVerify && !is_degenerated_gc_in_progress()) {
2580     verifier()->verify_roots_in_to_space_except(ShenandoahRootVerifier::ThreadRoots);
2581   }
2582 
2583   if (is_degenerated_gc_in_progress()) {
2584     concurrent_mark()->update_roots(ShenandoahPhaseTimings::degen_gc_update_roots);
2585   } else {
2586     concurrent_mark()->update_thread_roots(ShenandoahPhaseTimings::final_update_refs_roots);
2587   }
2588 
2589   // Has to be done before cset is clear
2590   if (ShenandoahVerify) {
2591     verifier()->verify_roots_in_to_space();
2592   }
2593 
2594   {
2595     ShenandoahGCPhase phase(ShenandoahPhaseTimings::final_update_refs_update_region_states);
2596     ShenandoahFinalUpdateRefsUpdateRegionStateClosure cl;
2597     parallel_heap_region_iterate(&cl);
2598 
2599     assert_pinned_region_status();
2600   }
2601 
2602   {
2603     ShenandoahGCPhase phase(ShenandoahPhaseTimings::final_update_refs_trash_cset);
2604     trash_cset_regions();
2605   }
2606 
2607   set_has_forwarded_objects(false);
2608   set_update_refs_in_progress(false);
2609 
2610   if (ShenandoahVerify) {
2611     verifier()->verify_after_updaterefs();
2612   }
2613 
2614   if (VerifyAfterGC) {
2615     Universe::verify();
2616   }
2617 
2618   {
2619     ShenandoahGCPhase phase(ShenandoahPhaseTimings::final_update_refs_rebuild_freeset);
2620     ShenandoahHeapLocker locker(lock());
2621     _free_set->rebuild();
2622   }
2623 }
2624 
2625 void ShenandoahHeap::print_extended_on(outputStream *st) const {
2626   print_on(st);
2627   print_heap_regions_on(st);
2628 }
2629 
2630 bool ShenandoahHeap::is_bitmap_slice_committed(ShenandoahHeapRegion* r, bool skip_self) {
2631   size_t slice = r->index() / _bitmap_regions_per_slice;
2632 
2633   size_t regions_from = _bitmap_regions_per_slice * slice;
2634   size_t regions_to   = MIN2(num_regions(), _bitmap_regions_per_slice * (slice + 1));
2635   for (size_t g = regions_from; g < regions_to; g++) {
2636     assert (g / _bitmap_regions_per_slice == slice, "same slice");
2637     if (skip_self && g == r->index()) continue;
2638     if (get_region(g)->is_committed()) {
2639       return true;
2640     }
2641   }
2642   return false;
2643 }
2644 
2645 bool ShenandoahHeap::commit_bitmap_slice(ShenandoahHeapRegion* r) {
2646   shenandoah_assert_heaplocked();
2647 
2648   // Bitmaps in special regions do not need commits
2649   if (_bitmap_region_special) {
2650     return true;
2651   }
2652 
2653   if (is_bitmap_slice_committed(r, true)) {
2654     // Some other region from the group is already committed, meaning the bitmap
2655     // slice is already committed, we exit right away.
2656     return true;
2657   }
2658 
2659   // Commit the bitmap slice:
2660   size_t slice = r->index() / _bitmap_regions_per_slice;
2661   size_t off = _bitmap_bytes_per_slice * slice;
2662   size_t len = _bitmap_bytes_per_slice;
2663   if (!os::commit_memory((char*)_bitmap_region.start() + off, len, false)) {
2664     return false;
2665   }
2666   return true;
2667 }
2668 
2669 bool ShenandoahHeap::uncommit_bitmap_slice(ShenandoahHeapRegion *r) {
2670   shenandoah_assert_heaplocked();
2671 
2672   // Bitmaps in special regions do not need uncommits
2673   if (_bitmap_region_special) {
2674     return true;
2675   }
2676 
2677   if (is_bitmap_slice_committed(r, true)) {
2678     // Some other region from the group is still committed, meaning the bitmap
2679     // slice is should stay committed, exit right away.
2680     return true;
2681   }
2682 
2683   // Uncommit the bitmap slice:
2684   size_t slice = r->index() / _bitmap_regions_per_slice;
2685   size_t off = _bitmap_bytes_per_slice * slice;
2686   size_t len = _bitmap_bytes_per_slice;
2687   if (!os::uncommit_memory((char*)_bitmap_region.start() + off, len)) {
2688     return false;
2689   }
2690   return true;
2691 }
2692 
2693 void ShenandoahHeap::safepoint_synchronize_begin() {
2694   if (ShenandoahSuspendibleWorkers || UseStringDeduplication) {
2695     SuspendibleThreadSet::synchronize();
2696   }
2697 }
2698 
2699 void ShenandoahHeap::safepoint_synchronize_end() {
2700   if (ShenandoahSuspendibleWorkers || UseStringDeduplication) {
2701     SuspendibleThreadSet::desynchronize();
2702   }
2703 }
2704 
2705 void ShenandoahHeap::vmop_entry_init_mark() {
2706   TraceCollectorStats tcs(monitoring_support()->stw_collection_counters());
2707   ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::init_mark_gross);
2708 
2709   try_inject_alloc_failure();
2710   VM_ShenandoahInitMark op;
2711   VMThread::execute(&op); // jump to entry_init_mark() under safepoint
2712 }
2713 
2714 void ShenandoahHeap::vmop_entry_final_mark() {
2715   TraceCollectorStats tcs(monitoring_support()->stw_collection_counters());
2716   ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_mark_gross);
2717 
2718   try_inject_alloc_failure();
2719   VM_ShenandoahFinalMarkStartEvac op;
2720   VMThread::execute(&op); // jump to entry_final_mark under safepoint
2721 }
2722 
2723 void ShenandoahHeap::vmop_entry_init_updaterefs() {
2724   TraceCollectorStats tcs(monitoring_support()->stw_collection_counters());
2725   ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::init_update_refs_gross);
2726 
2727   try_inject_alloc_failure();
2728   VM_ShenandoahInitUpdateRefs op;
2729   VMThread::execute(&op);
2730 }
2731 
2732 void ShenandoahHeap::vmop_entry_final_updaterefs() {
2733   TraceCollectorStats tcs(monitoring_support()->stw_collection_counters());
2734   ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_update_refs_gross);
2735 
2736   try_inject_alloc_failure();
2737   VM_ShenandoahFinalUpdateRefs op;
2738   VMThread::execute(&op);
2739 }
2740 
2741 void ShenandoahHeap::vmop_entry_full(GCCause::Cause cause) {
2742   TraceCollectorStats tcs(monitoring_support()->full_stw_collection_counters());
2743   ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::full_gc_gross);
2744 
2745   try_inject_alloc_failure();
2746   VM_ShenandoahFullGC op(cause);
2747   VMThread::execute(&op);
2748 }
2749 
2750 void ShenandoahHeap::vmop_degenerated(ShenandoahDegenPoint point) {
2751   TraceCollectorStats tcs(monitoring_support()->full_stw_collection_counters());
2752   ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::degen_gc_gross);
2753 
2754   VM_ShenandoahDegeneratedGC degenerated_gc((int)point);
2755   VMThread::execute(&degenerated_gc);
2756 }
2757 
2758 void ShenandoahHeap::entry_init_mark() {
2759   const char* msg = init_mark_event_message();
2760   ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::init_mark);
2761   EventMark em("%s", msg);
2762 
2763   ShenandoahWorkerScope scope(workers(),
2764                               ShenandoahWorkerPolicy::calc_workers_for_init_marking(),
2765                               "init marking");
2766 
2767   op_init_mark();
2768 }
2769 
2770 void ShenandoahHeap::entry_final_mark() {
2771   const char* msg = final_mark_event_message();
2772   ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::final_mark);
2773   EventMark em("%s", msg);
2774 
2775   ShenandoahWorkerScope scope(workers(),
2776                               ShenandoahWorkerPolicy::calc_workers_for_final_marking(),
2777                               "final marking");
2778 
2779   op_final_mark();
2780 }
2781 
2782 void ShenandoahHeap::entry_init_updaterefs() {
2783   static const char* msg = "Pause Init Update Refs";
2784   ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::init_update_refs);
2785   EventMark em("%s", msg);
2786 
2787   // No workers used in this phase, no setup required
2788 
2789   op_init_updaterefs();
2790 }
2791 
2792 void ShenandoahHeap::entry_final_updaterefs() {
2793   static const char* msg = "Pause Final Update Refs";
2794   ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::final_update_refs);
2795   EventMark em("%s", msg);
2796 
2797   ShenandoahWorkerScope scope(workers(),
2798                               ShenandoahWorkerPolicy::calc_workers_for_final_update_ref(),
2799                               "final reference update");
2800 
2801   op_final_updaterefs();
2802 }
2803 
2804 void ShenandoahHeap::entry_full(GCCause::Cause cause) {
2805   static const char* msg = "Pause Full";
2806   ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::full_gc, true /* log_heap_usage */);
2807   EventMark em("%s", msg);
2808 
2809   ShenandoahWorkerScope scope(workers(),
2810                               ShenandoahWorkerPolicy::calc_workers_for_fullgc(),
2811                               "full gc");
2812 
2813   op_full(cause);
2814 }
2815 
2816 void ShenandoahHeap::entry_degenerated(int point) {
2817   ShenandoahDegenPoint dpoint = (ShenandoahDegenPoint)point;
2818   const char* msg = degen_event_message(dpoint);
2819   ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::degen_gc, true /* log_heap_usage */);
2820   EventMark em("%s", msg);
2821 
2822   ShenandoahWorkerScope scope(workers(),
2823                               ShenandoahWorkerPolicy::calc_workers_for_stw_degenerated(),
2824                               "stw degenerated gc");
2825 
2826   set_degenerated_gc_in_progress(true);
2827   op_degenerated(dpoint);
2828   set_degenerated_gc_in_progress(false);
2829 }
2830 
2831 void ShenandoahHeap::entry_mark() {
2832   TraceCollectorStats tcs(monitoring_support()->concurrent_collection_counters());
2833 
2834   const char* msg = conc_mark_event_message();
2835   ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_mark);
2836   EventMark em("%s", msg);
2837 
2838   ShenandoahWorkerScope scope(workers(),
2839                               ShenandoahWorkerPolicy::calc_workers_for_conc_marking(),
2840                               "concurrent marking");
2841 
2842   try_inject_alloc_failure();
2843   op_mark();
2844 }
2845 
2846 void ShenandoahHeap::entry_evac() {
2847   TraceCollectorStats tcs(monitoring_support()->concurrent_collection_counters());
2848 
2849   static const char* msg = "Concurrent evacuation";
2850   ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_evac);
2851   EventMark em("%s", msg);
2852 
2853   ShenandoahWorkerScope scope(workers(),
2854                               ShenandoahWorkerPolicy::calc_workers_for_conc_evac(),
2855                               "concurrent evacuation");
2856 
2857   try_inject_alloc_failure();
2858   op_conc_evac();
2859 }
2860 
2861 void ShenandoahHeap::entry_updaterefs() {
2862   static const char* msg = "Concurrent update references";
2863   ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_update_refs);
2864   EventMark em("%s", msg);
2865 
2866   ShenandoahWorkerScope scope(workers(),
2867                               ShenandoahWorkerPolicy::calc_workers_for_conc_update_ref(),
2868                               "concurrent reference update");
2869 
2870   try_inject_alloc_failure();
2871   op_updaterefs();
2872 }
2873 
2874 void ShenandoahHeap::entry_weak_roots() {
2875   static const char* msg = "Concurrent weak roots";
2876   ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_weak_roots);
2877   EventMark em("%s", msg);
2878 
2879   ShenandoahWorkerScope scope(workers(),
2880                               ShenandoahWorkerPolicy::calc_workers_for_conc_root_processing(),
2881                               "concurrent weak root");
2882 
2883   try_inject_alloc_failure();
2884   op_weak_roots();
2885 }
2886 
2887 void ShenandoahHeap::entry_class_unloading() {
2888   static const char* msg = "Concurrent class unloading";
2889   ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_class_unload);
2890   EventMark em("%s", msg);
2891 
2892   ShenandoahWorkerScope scope(workers(),
2893                               ShenandoahWorkerPolicy::calc_workers_for_conc_root_processing(),
2894                               "concurrent class unloading");
2895 
2896   try_inject_alloc_failure();
2897   op_class_unloading();
2898 }
2899 
2900 void ShenandoahHeap::entry_strong_roots() {
2901   static const char* msg = "Concurrent strong roots";
2902   ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_strong_roots);
2903   EventMark em("%s", msg);
2904 
2905   ShenandoahGCWorkerPhase worker_phase(ShenandoahPhaseTimings::conc_strong_roots);
2906 
2907   ShenandoahWorkerScope scope(workers(),
2908                               ShenandoahWorkerPolicy::calc_workers_for_conc_root_processing(),
2909                               "concurrent strong root");
2910 
2911   try_inject_alloc_failure();
2912   op_strong_roots();
2913 }
2914 
2915 void ShenandoahHeap::entry_cleanup_early() {
2916   static const char* msg = "Concurrent cleanup";
2917   ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_cleanup_early, true /* log_heap_usage */);
2918   EventMark em("%s", msg);
2919 
2920   // This phase does not use workers, no need for setup
2921 
2922   try_inject_alloc_failure();
2923   op_cleanup_early();
2924 }
2925 
2926 void ShenandoahHeap::entry_cleanup_complete() {
2927   static const char* msg = "Concurrent cleanup";
2928   ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_cleanup_complete, true /* log_heap_usage */);
2929   EventMark em("%s", msg);
2930 
2931   // This phase does not use workers, no need for setup
2932 
2933   try_inject_alloc_failure();
2934   op_cleanup_complete();
2935 }
2936 
2937 void ShenandoahHeap::entry_reset() {
2938   static const char* msg = "Concurrent reset";
2939   ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_reset);
2940   EventMark em("%s", msg);
2941 
2942   ShenandoahWorkerScope scope(workers(),
2943                               ShenandoahWorkerPolicy::calc_workers_for_conc_reset(),
2944                               "concurrent reset");
2945 
2946   try_inject_alloc_failure();
2947   op_reset();
2948 }
2949 
2950 void ShenandoahHeap::entry_preclean() {
2951   if (ShenandoahPreclean && process_references()) {
2952     static const char* msg = "Concurrent precleaning";
2953     ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_preclean);
2954     EventMark em("%s", msg);
2955 
2956     ShenandoahWorkerScope scope(workers(),
2957                                 ShenandoahWorkerPolicy::calc_workers_for_conc_preclean(),
2958                                 "concurrent preclean",
2959                                 /* check_workers = */ false);
2960 
2961     try_inject_alloc_failure();
2962     op_preclean();
2963   }
2964 }
2965 
2966 void ShenandoahHeap::entry_uncommit(double shrink_before) {
2967   static const char *msg = "Concurrent uncommit";
2968   ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_uncommit, true /* log_heap_usage */);
2969   EventMark em("%s", msg);
2970 
2971   op_uncommit(shrink_before);
2972 }
2973 
2974 void ShenandoahHeap::try_inject_alloc_failure() {
2975   if (ShenandoahAllocFailureALot && !cancelled_gc() && ((os::random() % 1000) > 950)) {
2976     _inject_alloc_failure.set();
2977     os::naked_short_sleep(1);
2978     if (cancelled_gc()) {
2979       log_info(gc)("Allocation failure was successfully injected");
2980     }
2981   }
2982 }
2983 
2984 bool ShenandoahHeap::should_inject_alloc_failure() {
2985   return _inject_alloc_failure.is_set() && _inject_alloc_failure.try_unset();
2986 }
2987 
2988 void ShenandoahHeap::initialize_serviceability() {
2989   _memory_pool = new ShenandoahMemoryPool(this);
2990   _cycle_memory_manager.add_pool(_memory_pool);
2991   _stw_memory_manager.add_pool(_memory_pool);
2992 }
2993 
2994 GrowableArray<GCMemoryManager*> ShenandoahHeap::memory_managers() {
2995   GrowableArray<GCMemoryManager*> memory_managers(2);
2996   memory_managers.append(&_cycle_memory_manager);
2997   memory_managers.append(&_stw_memory_manager);
2998   return memory_managers;
2999 }
3000 
3001 GrowableArray<MemoryPool*> ShenandoahHeap::memory_pools() {
3002   GrowableArray<MemoryPool*> memory_pools(1);
3003   memory_pools.append(_memory_pool);
3004   return memory_pools;
3005 }
3006 
3007 MemoryUsage ShenandoahHeap::memory_usage() {
3008   return _memory_pool->get_memory_usage();
3009 }
3010 
3011 void ShenandoahHeap::enter_evacuation() {
3012   _oom_evac_handler.enter_evacuation();
3013 }
3014 
3015 void ShenandoahHeap::leave_evacuation() {
3016   _oom_evac_handler.leave_evacuation();
3017 }
3018 
3019 ShenandoahRegionIterator::ShenandoahRegionIterator() :
3020   _heap(ShenandoahHeap::heap()),
3021   _index(0) {}
3022 
3023 ShenandoahRegionIterator::ShenandoahRegionIterator(ShenandoahHeap* heap) :
3024   _heap(heap),
3025   _index(0) {}
3026 
3027 void ShenandoahRegionIterator::reset() {
3028   _index = 0;
3029 }
3030 
3031 bool ShenandoahRegionIterator::has_next() const {
3032   return _index < _heap->num_regions();
3033 }
3034 
3035 char ShenandoahHeap::gc_state() const {
3036   return _gc_state.raw_value();
3037 }
3038 
3039 void ShenandoahHeap::deduplicate_string(oop str) {
3040   assert(java_lang_String::is_instance(str), "invariant");
3041 
3042   if (ShenandoahStringDedup::is_enabled()) {
3043     ShenandoahStringDedup::deduplicate(str);
3044   }
3045 }
3046 
3047 const char* ShenandoahHeap::init_mark_event_message() const {
3048   assert(!has_forwarded_objects(), "Should not have forwarded objects here");
3049 
3050   bool proc_refs = process_references();
3051   bool unload_cls = unload_classes();
3052 
3053   if (proc_refs && unload_cls) {
3054     return "Pause Init Mark (process weakrefs) (unload classes)";
3055   } else if (proc_refs) {
3056     return "Pause Init Mark (process weakrefs)";
3057   } else if (unload_cls) {
3058     return "Pause Init Mark (unload classes)";
3059   } else {
3060     return "Pause Init Mark";
3061   }
3062 }
3063 
3064 const char* ShenandoahHeap::final_mark_event_message() const {
3065   assert(!has_forwarded_objects(), "Should not have forwarded objects here");
3066 
3067   bool proc_refs = process_references();
3068   bool unload_cls = unload_classes();
3069 
3070   if (proc_refs && unload_cls) {
3071     return "Pause Final Mark (process weakrefs) (unload classes)";
3072   } else if (proc_refs) {
3073     return "Pause Final Mark (process weakrefs)";
3074   } else if (unload_cls) {
3075     return "Pause Final Mark (unload classes)";
3076   } else {
3077     return "Pause Final Mark";
3078   }
3079 }
3080 
3081 const char* ShenandoahHeap::conc_mark_event_message() const {
3082   assert(!has_forwarded_objects(), "Should not have forwarded objects here");
3083 
3084   bool proc_refs = process_references();
3085   bool unload_cls = unload_classes();
3086 
3087   if (proc_refs && unload_cls) {
3088     return "Concurrent marking (process weakrefs) (unload classes)";
3089   } else if (proc_refs) {
3090     return "Concurrent marking (process weakrefs)";
3091   } else if (unload_cls) {
3092     return "Concurrent marking (unload classes)";
3093   } else {
3094     return "Concurrent marking";
3095   }
3096 }
3097 
3098 const char* ShenandoahHeap::degen_event_message(ShenandoahDegenPoint point) const {
3099   switch (point) {
3100     case _degenerated_unset:
3101       return "Pause Degenerated GC (<UNSET>)";
3102     case _degenerated_outside_cycle:
3103       return "Pause Degenerated GC (Outside of Cycle)";
3104     case _degenerated_mark:
3105       return "Pause Degenerated GC (Mark)";
3106     case _degenerated_evac:
3107       return "Pause Degenerated GC (Evacuation)";
3108     case _degenerated_updaterefs:
3109       return "Pause Degenerated GC (Update Refs)";
3110     default:
3111       ShouldNotReachHere();
3112       return "ERROR";
3113   }
3114 }
3115 
3116 ShenandoahLiveData* ShenandoahHeap::get_liveness_cache(uint worker_id) {
3117 #ifdef ASSERT
3118   assert(_liveness_cache != NULL, "sanity");
3119   assert(worker_id < _max_workers, "sanity");
3120   for (uint i = 0; i < num_regions(); i++) {
3121     assert(_liveness_cache[worker_id][i] == 0, "liveness cache should be empty");
3122   }
3123 #endif
3124   return _liveness_cache[worker_id];
3125 }
3126 
3127 void ShenandoahHeap::flush_liveness_cache(uint worker_id) {
3128   assert(worker_id < _max_workers, "sanity");
3129   assert(_liveness_cache != NULL, "sanity");
3130   ShenandoahLiveData* ld = _liveness_cache[worker_id];
3131   for (uint i = 0; i < num_regions(); i++) {
3132     ShenandoahLiveData live = ld[i];
3133     if (live > 0) {
3134       ShenandoahHeapRegion* r = get_region(i);
3135       r->increase_live_data_gc_words(live);
3136       ld[i] = 0;
3137     }
3138   }
3139 }