1 /*
   2  * Copyright (c) 2013, 2020, Red Hat, Inc. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "memory/allocation.hpp"
  27 #include "memory/universe.hpp"
  28 
  29 #include "gc/shared/gcArguments.hpp"
  30 #include "gc/shared/gcTimer.hpp"
  31 #include "gc/shared/gcTraceTime.inline.hpp"
  32 #include "gc/shared/locationPrinter.inline.hpp"
  33 #include "gc/shared/memAllocator.hpp"
  34 #include "gc/shared/plab.hpp"
  35 
  36 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
  37 #include "gc/shenandoah/shenandoahClosures.inline.hpp"
  38 #include "gc/shenandoah/shenandoahCollectionSet.hpp"
  39 #include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
  40 #include "gc/shenandoah/shenandoahConcurrentMark.inline.hpp"
  41 #include "gc/shenandoah/shenandoahConcurrentRoots.hpp"
  42 #include "gc/shenandoah/shenandoahControlThread.hpp"
  43 #include "gc/shenandoah/shenandoahFreeSet.hpp"
  44 #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
  45 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
  46 #include "gc/shenandoah/shenandoahHeapRegion.inline.hpp"
  47 #include "gc/shenandoah/shenandoahHeapRegionSet.hpp"
  48 #include "gc/shenandoah/shenandoahInitLogger.hpp"
  49 #include "gc/shenandoah/shenandoahMarkCompact.hpp"
  50 #include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"
  51 #include "gc/shenandoah/shenandoahMemoryPool.hpp"
  52 #include "gc/shenandoah/shenandoahMetrics.hpp"
  53 #include "gc/shenandoah/shenandoahMonitoringSupport.hpp"
  54 #include "gc/shenandoah/shenandoahOopClosures.inline.hpp"
  55 #include "gc/shenandoah/shenandoahPacer.inline.hpp"
  56 #include "gc/shenandoah/shenandoahPadding.hpp"
  57 #include "gc/shenandoah/shenandoahParallelCleaning.inline.hpp"
  58 #include "gc/shenandoah/shenandoahRootProcessor.inline.hpp"
  59 #include "gc/shenandoah/shenandoahStringDedup.hpp"
  60 #include "gc/shenandoah/shenandoahTaskqueue.hpp"
  61 #include "gc/shenandoah/shenandoahUtils.hpp"
  62 #include "gc/shenandoah/shenandoahVerifier.hpp"
  63 #include "gc/shenandoah/shenandoahCodeRoots.hpp"
  64 #include "gc/shenandoah/shenandoahVMOperations.hpp"
  65 #include "gc/shenandoah/shenandoahWorkGroup.hpp"
  66 #include "gc/shenandoah/shenandoahWorkerPolicy.hpp"
  67 #include "gc/shenandoah/mode/shenandoahIUMode.hpp"
  68 #include "gc/shenandoah/mode/shenandoahPassiveMode.hpp"
  69 #include "gc/shenandoah/mode/shenandoahSATBMode.hpp"
  70 #if INCLUDE_JFR
  71 #include "gc/shenandoah/shenandoahJfrSupport.hpp"
  72 #endif
  73 
  74 #include "memory/metaspace.hpp"
  75 #include "oops/compressedOops.inline.hpp"
  76 #include "runtime/atomic.hpp"
  77 #include "runtime/globals.hpp"
  78 #include "runtime/interfaceSupport.inline.hpp"
  79 #include "runtime/orderAccess.hpp"
  80 #include "runtime/safepointMechanism.hpp"
  81 #include "runtime/vmThread.hpp"
  82 #include "services/mallocTracker.hpp"
  83 #include "utilities/powerOfTwo.hpp"
  84 
  85 #ifdef ASSERT
  86 template <class T>
  87 void ShenandoahAssertToSpaceClosure::do_oop_work(T* p) {
  88   T o = RawAccess<>::oop_load(p);
  89   if (! CompressedOops::is_null(o)) {
  90     oop obj = CompressedOops::decode_not_null(o);
  91     shenandoah_assert_not_forwarded(p, obj);
  92   }
  93 }
  94 
  95 void ShenandoahAssertToSpaceClosure::do_oop(narrowOop* p) { do_oop_work(p); }
  96 void ShenandoahAssertToSpaceClosure::do_oop(oop* p)       { do_oop_work(p); }
  97 #endif
  98 
  99 class ShenandoahPretouchHeapTask : public AbstractGangTask {
 100 private:
 101   ShenandoahRegionIterator _regions;
 102   const size_t _page_size;
 103 public:
 104   ShenandoahPretouchHeapTask(size_t page_size) :
 105     AbstractGangTask("Shenandoah Pretouch Heap"),
 106     _page_size(page_size) {}
 107 
 108   virtual void work(uint worker_id) {
 109     ShenandoahHeapRegion* r = _regions.next();
 110     while (r != NULL) {
 111       if (r->is_committed()) {
 112         os::pretouch_memory(r->bottom(), r->end(), _page_size);
 113       }
 114       r = _regions.next();
 115     }
 116   }
 117 };
 118 
 119 class ShenandoahPretouchBitmapTask : public AbstractGangTask {
 120 private:
 121   ShenandoahRegionIterator _regions;
 122   char* _bitmap_base;
 123   const size_t _bitmap_size;
 124   const size_t _page_size;
 125 public:
 126   ShenandoahPretouchBitmapTask(char* bitmap_base, size_t bitmap_size, size_t page_size) :
 127     AbstractGangTask("Shenandoah Pretouch Bitmap"),
 128     _bitmap_base(bitmap_base),
 129     _bitmap_size(bitmap_size),
 130     _page_size(page_size) {}
 131 
 132   virtual void work(uint worker_id) {
 133     ShenandoahHeapRegion* r = _regions.next();
 134     while (r != NULL) {
 135       size_t start = r->index()       * ShenandoahHeapRegion::region_size_bytes() / MarkBitMap::heap_map_factor();
 136       size_t end   = (r->index() + 1) * ShenandoahHeapRegion::region_size_bytes() / MarkBitMap::heap_map_factor();
 137       assert (end <= _bitmap_size, "end is sane: " SIZE_FORMAT " < " SIZE_FORMAT, end, _bitmap_size);
 138 
 139       if (r->is_committed()) {
 140         os::pretouch_memory(_bitmap_base + start, _bitmap_base + end, _page_size);
 141       }
 142 
 143       r = _regions.next();
 144     }
 145   }
 146 };
 147 
 148 jint ShenandoahHeap::initialize() {
 149   //
 150   // Figure out heap sizing
 151   //
 152 
 153   size_t init_byte_size = InitialHeapSize;
 154   size_t min_byte_size  = MinHeapSize;
 155   size_t max_byte_size  = MaxHeapSize;
 156   size_t heap_alignment = HeapAlignment;
 157 
 158   size_t reg_size_bytes = ShenandoahHeapRegion::region_size_bytes();
 159 
 160   Universe::check_alignment(max_byte_size,  reg_size_bytes, "Shenandoah heap");
 161   Universe::check_alignment(init_byte_size, reg_size_bytes, "Shenandoah heap");
 162 
 163   _num_regions = ShenandoahHeapRegion::region_count();
 164 
 165   // Now we know the number of regions, initialize the heuristics.
 166   initialize_heuristics();
 167 
 168   size_t num_committed_regions = init_byte_size / reg_size_bytes;
 169   num_committed_regions = MIN2(num_committed_regions, _num_regions);
 170   assert(num_committed_regions <= _num_regions, "sanity");
 171   _initial_size = num_committed_regions * reg_size_bytes;
 172 
 173   size_t num_min_regions = min_byte_size / reg_size_bytes;
 174   num_min_regions = MIN2(num_min_regions, _num_regions);
 175   assert(num_min_regions <= _num_regions, "sanity");
 176   _minimum_size = num_min_regions * reg_size_bytes;
 177 
 178   _committed = _initial_size;
 179 
 180   size_t heap_page_size   = UseLargePages ? (size_t)os::large_page_size() : (size_t)os::vm_page_size();
 181   size_t bitmap_page_size = UseLargePages ? (size_t)os::large_page_size() : (size_t)os::vm_page_size();
 182   size_t region_page_size = UseLargePages ? (size_t)os::large_page_size() : (size_t)os::vm_page_size();
 183 
 184   //
 185   // Reserve and commit memory for heap
 186   //
 187 
 188   ReservedHeapSpace heap_rs = Universe::reserve_heap(max_byte_size, heap_alignment);
 189   initialize_reserved_region(heap_rs);
 190   _heap_region = MemRegion((HeapWord*)heap_rs.base(), heap_rs.size() / HeapWordSize);
 191   _heap_region_special = heap_rs.special();
 192 
 193   assert((((size_t) base()) & ShenandoahHeapRegion::region_size_bytes_mask()) == 0,
 194          "Misaligned heap: " PTR_FORMAT, p2i(base()));
 195 
 196 #if SHENANDOAH_OPTIMIZED_OBJTASK
 197   // The optimized ObjArrayChunkedTask takes some bits away from the full object bits.
 198   // Fail if we ever attempt to address more than we can.
 199   if ((uintptr_t)heap_rs.end() >= ObjArrayChunkedTask::max_addressable()) {
 200     FormatBuffer<512> buf("Shenandoah reserved [" PTR_FORMAT ", " PTR_FORMAT") for the heap, \n"
 201                           "but max object address is " PTR_FORMAT ". Try to reduce heap size, or try other \n"
 202                           "VM options that allocate heap at lower addresses (HeapBaseMinAddress, AllocateHeapAt, etc).",
 203                 p2i(heap_rs.base()), p2i(heap_rs.end()), ObjArrayChunkedTask::max_addressable());
 204     vm_exit_during_initialization("Fatal Error", buf);
 205   }
 206 #endif
 207 
 208   ReservedSpace sh_rs = heap_rs.first_part(max_byte_size);
 209   if (!_heap_region_special) {
 210     os::commit_memory_or_exit(sh_rs.base(), _initial_size, heap_alignment, false,
 211                               "Cannot commit heap memory");
 212   }
 213 
 214   //
 215   // Reserve and commit memory for bitmap(s)
 216   //
 217 
 218   _bitmap_size = MarkBitMap::compute_size(heap_rs.size());
 219   _bitmap_size = align_up(_bitmap_size, bitmap_page_size);
 220 
 221   size_t bitmap_bytes_per_region = reg_size_bytes / MarkBitMap::heap_map_factor();
 222 
 223   guarantee(bitmap_bytes_per_region != 0,
 224             "Bitmap bytes per region should not be zero");
 225   guarantee(is_power_of_2(bitmap_bytes_per_region),
 226             "Bitmap bytes per region should be power of two: " SIZE_FORMAT, bitmap_bytes_per_region);
 227 
 228   if (bitmap_page_size > bitmap_bytes_per_region) {
 229     _bitmap_regions_per_slice = bitmap_page_size / bitmap_bytes_per_region;
 230     _bitmap_bytes_per_slice = bitmap_page_size;
 231   } else {
 232     _bitmap_regions_per_slice = 1;
 233     _bitmap_bytes_per_slice = bitmap_bytes_per_region;
 234   }
 235 
 236   guarantee(_bitmap_regions_per_slice >= 1,
 237             "Should have at least one region per slice: " SIZE_FORMAT,
 238             _bitmap_regions_per_slice);
 239 
 240   guarantee(((_bitmap_bytes_per_slice) % bitmap_page_size) == 0,
 241             "Bitmap slices should be page-granular: bps = " SIZE_FORMAT ", page size = " SIZE_FORMAT,
 242             _bitmap_bytes_per_slice, bitmap_page_size);
 243 
 244   ReservedSpace bitmap(_bitmap_size, bitmap_page_size);
 245   MemTracker::record_virtual_memory_type(bitmap.base(), mtGC);
 246   _bitmap_region = MemRegion((HeapWord*) bitmap.base(), bitmap.size() / HeapWordSize);
 247   _bitmap_region_special = bitmap.special();
 248 
 249   size_t bitmap_init_commit = _bitmap_bytes_per_slice *
 250                               align_up(num_committed_regions, _bitmap_regions_per_slice) / _bitmap_regions_per_slice;
 251   bitmap_init_commit = MIN2(_bitmap_size, bitmap_init_commit);
 252   if (!_bitmap_region_special) {
 253     os::commit_memory_or_exit((char *) _bitmap_region.start(), bitmap_init_commit, bitmap_page_size, false,
 254                               "Cannot commit bitmap memory");
 255   }
 256 
 257   _marking_context = new ShenandoahMarkingContext(_heap_region, _bitmap_region, _num_regions);
 258 
 259   if (ShenandoahVerify) {
 260     ReservedSpace verify_bitmap(_bitmap_size, bitmap_page_size);
 261     if (!verify_bitmap.special()) {
 262       os::commit_memory_or_exit(verify_bitmap.base(), verify_bitmap.size(), bitmap_page_size, false,
 263                                 "Cannot commit verification bitmap memory");
 264     }
 265     MemTracker::record_virtual_memory_type(verify_bitmap.base(), mtGC);
 266     MemRegion verify_bitmap_region = MemRegion((HeapWord *) verify_bitmap.base(), verify_bitmap.size() / HeapWordSize);
 267     _verification_bit_map.initialize(_heap_region, verify_bitmap_region);
 268     _verifier = new ShenandoahVerifier(this, &_verification_bit_map);
 269   }
 270 
 271   // Reserve aux bitmap for use in object_iterate(). We don't commit it here.
 272   ReservedSpace aux_bitmap(_bitmap_size, bitmap_page_size);
 273   MemTracker::record_virtual_memory_type(aux_bitmap.base(), mtGC);
 274   _aux_bitmap_region = MemRegion((HeapWord*) aux_bitmap.base(), aux_bitmap.size() / HeapWordSize);
 275   _aux_bitmap_region_special = aux_bitmap.special();
 276   _aux_bit_map.initialize(_heap_region, _aux_bitmap_region);
 277 
 278   //
 279   // Create regions and region sets
 280   //
 281   size_t region_align = align_up(sizeof(ShenandoahHeapRegion), SHENANDOAH_CACHE_LINE_SIZE);
 282   size_t region_storage_size = align_up(region_align * _num_regions, region_page_size);
 283   region_storage_size = align_up(region_storage_size, os::vm_allocation_granularity());
 284 
 285   ReservedSpace region_storage(region_storage_size, region_page_size);
 286   MemTracker::record_virtual_memory_type(region_storage.base(), mtGC);
 287   if (!region_storage.special()) {
 288     os::commit_memory_or_exit(region_storage.base(), region_storage_size, region_page_size, false,
 289                               "Cannot commit region memory");
 290   }
 291 
 292   // Try to fit the collection set bitmap at lower addresses. This optimizes code generation for cset checks.
 293   // Go up until a sensible limit (subject to encoding constraints) and try to reserve the space there.
 294   // If not successful, bite a bullet and allocate at whatever address.
 295   {
 296     size_t cset_align = MAX2<size_t>(os::vm_page_size(), os::vm_allocation_granularity());
 297     size_t cset_size = align_up(((size_t) sh_rs.base() + sh_rs.size()) >> ShenandoahHeapRegion::region_size_bytes_shift(), cset_align);
 298 
 299     uintptr_t min = round_up_power_of_2(cset_align);
 300     uintptr_t max = (1u << 30u);
 301 
 302     for (uintptr_t addr = min; addr <= max; addr <<= 1u) {
 303       char* req_addr = (char*)addr;
 304       assert(is_aligned(req_addr, cset_align), "Should be aligned");
 305       ReservedSpace cset_rs(cset_size, cset_align, false, req_addr);
 306       if (cset_rs.is_reserved()) {
 307         assert(cset_rs.base() == req_addr, "Allocated where requested: " PTR_FORMAT ", " PTR_FORMAT, p2i(cset_rs.base()), addr);
 308         _collection_set = new ShenandoahCollectionSet(this, cset_rs, sh_rs.base());
 309         break;
 310       }
 311     }
 312 
 313     if (_collection_set == NULL) {
 314       ReservedSpace cset_rs(cset_size, cset_align, false);
 315       _collection_set = new ShenandoahCollectionSet(this, cset_rs, sh_rs.base());
 316     }
 317   }
 318 
 319   _regions = NEW_C_HEAP_ARRAY(ShenandoahHeapRegion*, _num_regions, mtGC);
 320   _free_set = new ShenandoahFreeSet(this, _num_regions);
 321 
 322   {
 323     ShenandoahHeapLocker locker(lock());
 324 
 325     for (size_t i = 0; i < _num_regions; i++) {
 326       HeapWord* start = (HeapWord*)sh_rs.base() + ShenandoahHeapRegion::region_size_words() * i;
 327       bool is_committed = i < num_committed_regions;
 328       void* loc = region_storage.base() + i * region_align;
 329 
 330       ShenandoahHeapRegion* r = new (loc) ShenandoahHeapRegion(start, i, is_committed);
 331       assert(is_aligned(r, SHENANDOAH_CACHE_LINE_SIZE), "Sanity");
 332 
 333       _marking_context->initialize_top_at_mark_start(r);
 334       _regions[i] = r;
 335       assert(!collection_set()->is_in(i), "New region should not be in collection set");
 336     }
 337 
 338     // Initialize to complete
 339     _marking_context->mark_complete();
 340 
 341     _free_set->rebuild();
 342   }
 343 
 344   if (AlwaysPreTouch) {
 345     // For NUMA, it is important to pre-touch the storage under bitmaps with worker threads,
 346     // before initialize() below zeroes it with initializing thread. For any given region,
 347     // we touch the region and the corresponding bitmaps from the same thread.
 348     ShenandoahPushWorkerScope scope(workers(), _max_workers, false);
 349 
 350     _pretouch_heap_page_size = heap_page_size;
 351     _pretouch_bitmap_page_size = bitmap_page_size;
 352 
 353 #ifdef LINUX
 354     // UseTransparentHugePages would madvise that backing memory can be coalesced into huge
 355     // pages. But, the kernel needs to know that every small page is used, in order to coalesce
 356     // them into huge one. Therefore, we need to pretouch with smaller pages.
 357     if (UseTransparentHugePages) {
 358       _pretouch_heap_page_size = (size_t)os::vm_page_size();
 359       _pretouch_bitmap_page_size = (size_t)os::vm_page_size();
 360     }
 361 #endif
 362 
 363     // OS memory managers may want to coalesce back-to-back pages. Make their jobs
 364     // simpler by pre-touching continuous spaces (heap and bitmap) separately.
 365 
 366     ShenandoahPretouchBitmapTask bcl(bitmap.base(), _bitmap_size, _pretouch_bitmap_page_size);
 367     _workers->run_task(&bcl);
 368 
 369     ShenandoahPretouchHeapTask hcl(_pretouch_heap_page_size);
 370     _workers->run_task(&hcl);
 371   }
 372 
 373   //
 374   // Initialize the rest of GC subsystems
 375   //
 376 
 377   _liveness_cache = NEW_C_HEAP_ARRAY(ShenandoahLiveData*, _max_workers, mtGC);
 378   for (uint worker = 0; worker < _max_workers; worker++) {
 379     _liveness_cache[worker] = NEW_C_HEAP_ARRAY(ShenandoahLiveData, _num_regions, mtGC);
 380     Copy::fill_to_bytes(_liveness_cache[worker], _num_regions * sizeof(ShenandoahLiveData));
 381   }
 382 
 383   // There should probably be Shenandoah-specific options for these,
 384   // just as there are G1-specific options.
 385   {
 386     ShenandoahSATBMarkQueueSet& satbqs = ShenandoahBarrierSet::satb_mark_queue_set();
 387     satbqs.set_process_completed_buffers_threshold(20); // G1SATBProcessCompletedThreshold
 388     satbqs.set_buffer_enqueue_threshold_percentage(60); // G1SATBBufferEnqueueingThresholdPercent
 389   }
 390 
 391   _monitoring_support = new ShenandoahMonitoringSupport(this);
 392   _phase_timings = new ShenandoahPhaseTimings(max_workers());
 393   ShenandoahStringDedup::initialize();
 394   ShenandoahCodeRoots::initialize();
 395 
 396   if (ShenandoahPacing) {
 397     _pacer = new ShenandoahPacer(this);
 398     _pacer->setup_for_idle();
 399   } else {
 400     _pacer = NULL;
 401   }
 402 
 403   _control_thread = new ShenandoahControlThread();
 404 
 405   _ref_proc_mt_processing = ParallelRefProcEnabled && (ParallelGCThreads > 1);
 406   _ref_proc_mt_discovery = _max_workers > 1;
 407 
 408   ShenandoahInitLogger::print();
 409 
 410   return JNI_OK;
 411 }
 412 
 413 void ShenandoahHeap::initialize_heuristics() {
 414   if (ShenandoahGCMode != NULL) {
 415     if (strcmp(ShenandoahGCMode, "satb") == 0) {
 416       _gc_mode = new ShenandoahSATBMode();
 417     } else if (strcmp(ShenandoahGCMode, "iu") == 0) {
 418       _gc_mode = new ShenandoahIUMode();
 419     } else if (strcmp(ShenandoahGCMode, "passive") == 0) {
 420       _gc_mode = new ShenandoahPassiveMode();
 421     } else {
 422       vm_exit_during_initialization("Unknown -XX:ShenandoahGCMode option");
 423     }
 424   } else {
 425     ShouldNotReachHere();
 426   }
 427   _gc_mode->initialize_flags();
 428   if (_gc_mode->is_diagnostic() && !UnlockDiagnosticVMOptions) {
 429     vm_exit_during_initialization(
 430             err_msg("GC mode \"%s\" is diagnostic, and must be enabled via -XX:+UnlockDiagnosticVMOptions.",
 431                     _gc_mode->name()));
 432   }
 433   if (_gc_mode->is_experimental() && !UnlockExperimentalVMOptions) {
 434     vm_exit_during_initialization(
 435             err_msg("GC mode \"%s\" is experimental, and must be enabled via -XX:+UnlockExperimentalVMOptions.",
 436                     _gc_mode->name()));
 437   }
 438 
 439   _heuristics = _gc_mode->initialize_heuristics();
 440 
 441   if (_heuristics->is_diagnostic() && !UnlockDiagnosticVMOptions) {
 442     vm_exit_during_initialization(
 443             err_msg("Heuristics \"%s\" is diagnostic, and must be enabled via -XX:+UnlockDiagnosticVMOptions.",
 444                     _heuristics->name()));
 445   }
 446   if (_heuristics->is_experimental() && !UnlockExperimentalVMOptions) {
 447     vm_exit_during_initialization(
 448             err_msg("Heuristics \"%s\" is experimental, and must be enabled via -XX:+UnlockExperimentalVMOptions.",
 449                     _heuristics->name()));
 450   }
 451 }
 452 
 453 #ifdef _MSC_VER
 454 #pragma warning( push )
 455 #pragma warning( disable:4355 ) // 'this' : used in base member initializer list
 456 #endif
 457 
 458 ShenandoahHeap::ShenandoahHeap(ShenandoahCollectorPolicy* policy) :
 459   CollectedHeap(),
 460   _initial_size(0),
 461   _used(0),
 462   _committed(0),
 463   _bytes_allocated_since_gc_start(0),
 464   _max_workers(MAX2(ConcGCThreads, ParallelGCThreads)),
 465   _workers(NULL),
 466   _safepoint_workers(NULL),
 467   _heap_region_special(false),
 468   _num_regions(0),
 469   _regions(NULL),
 470   _update_refs_iterator(this),
 471   _control_thread(NULL),
 472   _shenandoah_policy(policy),
 473   _heuristics(NULL),
 474   _free_set(NULL),
 475   _scm(new ShenandoahConcurrentMark()),
 476   _full_gc(new ShenandoahMarkCompact()),
 477   _pacer(NULL),
 478   _verifier(NULL),
 479   _phase_timings(NULL),
 480   _monitoring_support(NULL),
 481   _memory_pool(NULL),
 482   _stw_memory_manager("Shenandoah Pauses", "end of GC pause"),
 483   _cycle_memory_manager("Shenandoah Cycles", "end of GC cycle"),
 484   _gc_timer(new (ResourceObj::C_HEAP, mtGC) ConcurrentGCTimer()),
 485   _soft_ref_policy(),
 486   _log_min_obj_alignment_in_bytes(LogMinObjAlignmentInBytes),
 487   _ref_processor(NULL),
 488   _marking_context(NULL),
 489   _bitmap_size(0),
 490   _bitmap_regions_per_slice(0),
 491   _bitmap_bytes_per_slice(0),
 492   _bitmap_region_special(false),
 493   _aux_bitmap_region_special(false),
 494   _liveness_cache(NULL),
 495   _collection_set(NULL)
 496 {
 497   BarrierSet::set_barrier_set(new ShenandoahBarrierSet(this));
 498 
 499   _max_workers = MAX2(_max_workers, 1U);
 500   _workers = new ShenandoahWorkGang("Shenandoah GC Threads", _max_workers,
 501                             /* are_GC_task_threads */ true,
 502                             /* are_ConcurrentGC_threads */ true);
 503   if (_workers == NULL) {
 504     vm_exit_during_initialization("Failed necessary allocation.");
 505   } else {
 506     _workers->initialize_workers();
 507   }
 508 
 509   if (ParallelGCThreads > 1) {
 510     _safepoint_workers = new ShenandoahWorkGang("Safepoint Cleanup Thread",
 511                                                 ParallelGCThreads,
 512                       /* are_GC_task_threads */ false,
 513                  /* are_ConcurrentGC_threads */ false);
 514     _safepoint_workers->initialize_workers();
 515   }
 516 }
 517 
 518 #ifdef _MSC_VER
 519 #pragma warning( pop )
 520 #endif
 521 
 522 class ShenandoahResetBitmapTask : public AbstractGangTask {
 523 private:
 524   ShenandoahRegionIterator _regions;
 525 
 526 public:
 527   ShenandoahResetBitmapTask() :
 528     AbstractGangTask("Parallel Reset Bitmap Task") {}
 529 
 530   void work(uint worker_id) {
 531     ShenandoahHeapRegion* region = _regions.next();
 532     ShenandoahHeap* heap = ShenandoahHeap::heap();
 533     ShenandoahMarkingContext* const ctx = heap->marking_context();
 534     while (region != NULL) {
 535       if (heap->is_bitmap_slice_committed(region)) {
 536         ctx->clear_bitmap(region);
 537       }
 538       region = _regions.next();
 539     }
 540   }
 541 };
 542 
 543 void ShenandoahHeap::reset_mark_bitmap() {
 544   assert_gc_workers(_workers->active_workers());
 545   mark_incomplete_marking_context();
 546 
 547   ShenandoahResetBitmapTask task;
 548   _workers->run_task(&task);
 549 }
 550 
 551 void ShenandoahHeap::print_on(outputStream* st) const {
 552   st->print_cr("Shenandoah Heap");
 553   st->print_cr(" " SIZE_FORMAT "%s total, " SIZE_FORMAT "%s committed, " SIZE_FORMAT "%s used",
 554                byte_size_in_proper_unit(max_capacity()), proper_unit_for_byte_size(max_capacity()),
 555                byte_size_in_proper_unit(committed()),    proper_unit_for_byte_size(committed()),
 556                byte_size_in_proper_unit(used()),         proper_unit_for_byte_size(used()));
 557   st->print_cr(" " SIZE_FORMAT " x " SIZE_FORMAT"%s regions",
 558                num_regions(),
 559                byte_size_in_proper_unit(ShenandoahHeapRegion::region_size_bytes()),
 560                proper_unit_for_byte_size(ShenandoahHeapRegion::region_size_bytes()));
 561 
 562   st->print("Status: ");
 563   if (has_forwarded_objects())                 st->print("has forwarded objects, ");
 564   if (is_concurrent_mark_in_progress())        st->print("marking, ");
 565   if (is_evacuation_in_progress())             st->print("evacuating, ");
 566   if (is_update_refs_in_progress())            st->print("updating refs, ");
 567   if (is_degenerated_gc_in_progress())         st->print("degenerated gc, ");
 568   if (is_full_gc_in_progress())                st->print("full gc, ");
 569   if (is_full_gc_move_in_progress())           st->print("full gc move, ");
 570   if (is_concurrent_weak_root_in_progress())   st->print("concurrent weak roots, ");
 571   if (is_concurrent_strong_root_in_progress() &&
 572       !is_concurrent_weak_root_in_progress())  st->print("concurrent strong roots, ");
 573 
 574   if (cancelled_gc()) {
 575     st->print("cancelled");
 576   } else {
 577     st->print("not cancelled");
 578   }
 579   st->cr();
 580 
 581   st->print_cr("Reserved region:");
 582   st->print_cr(" - [" PTR_FORMAT ", " PTR_FORMAT ") ",
 583                p2i(reserved_region().start()),
 584                p2i(reserved_region().end()));
 585 
 586   ShenandoahCollectionSet* cset = collection_set();
 587   st->print_cr("Collection set:");
 588   if (cset != NULL) {
 589     st->print_cr(" - map (vanilla): " PTR_FORMAT, p2i(cset->map_address()));
 590     st->print_cr(" - map (biased):  " PTR_FORMAT, p2i(cset->biased_map_address()));
 591   } else {
 592     st->print_cr(" (NULL)");
 593   }
 594 
 595   st->cr();
 596   MetaspaceUtils::print_on(st);
 597 
 598   if (Verbose) {
 599     print_heap_regions_on(st);
 600   }
 601 }
 602 
 603 class ShenandoahInitWorkerGCLABClosure : public ThreadClosure {
 604 public:
 605   void do_thread(Thread* thread) {
 606     assert(thread != NULL, "Sanity");
 607     assert(thread->is_Worker_thread(), "Only worker thread expected");
 608     ShenandoahThreadLocalData::initialize_gclab(thread);
 609   }
 610 };
 611 
 612 void ShenandoahHeap::post_initialize() {
 613   CollectedHeap::post_initialize();
 614   MutexLocker ml(Threads_lock);
 615 
 616   ShenandoahInitWorkerGCLABClosure init_gclabs;
 617   _workers->threads_do(&init_gclabs);
 618 
 619   // gclab can not be initialized early during VM startup, as it can not determinate its max_size.
 620   // Now, we will let WorkGang to initialize gclab when new worker is created.
 621   _workers->set_initialize_gclab();
 622 
 623   _scm->initialize(_max_workers);
 624   _full_gc->initialize(_gc_timer);
 625 
 626   ref_processing_init();
 627 
 628   _heuristics->initialize();
 629 
 630   JFR_ONLY(ShenandoahJFRSupport::register_jfr_type_serializers());
 631 }
 632 
 633 size_t ShenandoahHeap::used() const {
 634   return Atomic::load_acquire(&_used);
 635 }
 636 
 637 size_t ShenandoahHeap::committed() const {
 638   OrderAccess::acquire();
 639   return _committed;
 640 }
 641 
 642 void ShenandoahHeap::increase_committed(size_t bytes) {
 643   shenandoah_assert_heaplocked_or_safepoint();
 644   _committed += bytes;
 645 }
 646 
 647 void ShenandoahHeap::decrease_committed(size_t bytes) {
 648   shenandoah_assert_heaplocked_or_safepoint();
 649   _committed -= bytes;
 650 }
 651 
 652 void ShenandoahHeap::increase_used(size_t bytes) {
 653   Atomic::add(&_used, bytes);
 654 }
 655 
 656 void ShenandoahHeap::set_used(size_t bytes) {
 657   Atomic::release_store_fence(&_used, bytes);
 658 }
 659 
 660 void ShenandoahHeap::decrease_used(size_t bytes) {
 661   assert(used() >= bytes, "never decrease heap size by more than we've left");
 662   Atomic::sub(&_used, bytes);
 663 }
 664 
 665 void ShenandoahHeap::increase_allocated(size_t bytes) {
 666   Atomic::add(&_bytes_allocated_since_gc_start, bytes);
 667 }
 668 
 669 void ShenandoahHeap::notify_mutator_alloc_words(size_t words, bool waste) {
 670   size_t bytes = words * HeapWordSize;
 671   if (!waste) {
 672     increase_used(bytes);
 673   }
 674   increase_allocated(bytes);
 675   if (ShenandoahPacing) {
 676     control_thread()->pacing_notify_alloc(words);
 677     if (waste) {
 678       pacer()->claim_for_alloc(words, true);
 679     }
 680   }
 681 }
 682 
 683 size_t ShenandoahHeap::capacity() const {
 684   return committed();
 685 }
 686 
 687 size_t ShenandoahHeap::max_capacity() const {
 688   return _num_regions * ShenandoahHeapRegion::region_size_bytes();
 689 }
 690 
 691 size_t ShenandoahHeap::min_capacity() const {
 692   return _minimum_size;
 693 }
 694 
 695 size_t ShenandoahHeap::initial_capacity() const {
 696   return _initial_size;
 697 }
 698 
 699 bool ShenandoahHeap::is_in(const void* p) const {
 700   HeapWord* heap_base = (HeapWord*) base();
 701   HeapWord* last_region_end = heap_base + ShenandoahHeapRegion::region_size_words() * num_regions();
 702   return p >= heap_base && p < last_region_end;
 703 }
 704 
 705 void ShenandoahHeap::op_uncommit(double shrink_before) {
 706   assert (ShenandoahUncommit, "should be enabled");
 707 
 708   // Application allocates from the beginning of the heap, and GC allocates at
 709   // the end of it. It is more efficient to uncommit from the end, so that applications
 710   // could enjoy the near committed regions. GC allocations are much less frequent,
 711   // and therefore can accept the committing costs.
 712 
 713   size_t count = 0;
 714   for (size_t i = num_regions(); i > 0; i--) { // care about size_t underflow
 715     ShenandoahHeapRegion* r = get_region(i - 1);
 716     if (r->is_empty_committed() && (r->empty_time() < shrink_before)) {
 717       ShenandoahHeapLocker locker(lock());
 718       if (r->is_empty_committed()) {
 719         // Do not uncommit below minimal capacity
 720         if (committed() < min_capacity() + ShenandoahHeapRegion::region_size_bytes()) {
 721           break;
 722         }
 723 
 724         r->make_uncommitted();
 725         count++;
 726       }
 727     }
 728     SpinPause(); // allow allocators to take the lock
 729   }
 730 
 731   if (count > 0) {
 732     control_thread()->notify_heap_changed();
 733   }
 734 }
 735 
 736 HeapWord* ShenandoahHeap::allocate_from_gclab_slow(Thread* thread, size_t size) {
 737   // New object should fit the GCLAB size
 738   size_t min_size = MAX2(size, PLAB::min_size());
 739 
 740   // Figure out size of new GCLAB, looking back at heuristics. Expand aggressively.
 741   size_t new_size = ShenandoahThreadLocalData::gclab_size(thread) * 2;
 742   new_size = MIN2(new_size, PLAB::max_size());
 743   new_size = MAX2(new_size, PLAB::min_size());
 744 
 745   // Record new heuristic value even if we take any shortcut. This captures
 746   // the case when moderately-sized objects always take a shortcut. At some point,
 747   // heuristics should catch up with them.
 748   ShenandoahThreadLocalData::set_gclab_size(thread, new_size);
 749 
 750   if (new_size < size) {
 751     // New size still does not fit the object. Fall back to shared allocation.
 752     // This avoids retiring perfectly good GCLABs, when we encounter a large object.
 753     return NULL;
 754   }
 755 
 756   // Retire current GCLAB, and allocate a new one.
 757   PLAB* gclab = ShenandoahThreadLocalData::gclab(thread);
 758   gclab->retire();
 759 
 760   size_t actual_size = 0;
 761   HeapWord* gclab_buf = allocate_new_gclab(min_size, new_size, &actual_size);
 762   if (gclab_buf == NULL) {
 763     return NULL;
 764   }
 765 
 766   assert (size <= actual_size, "allocation should fit");
 767 
 768   if (ZeroTLAB) {
 769     // ..and clear it.
 770     Copy::zero_to_words(gclab_buf, actual_size);
 771   } else {
 772     // ...and zap just allocated object.
 773 #ifdef ASSERT
 774     // Skip mangling the space corresponding to the object header to
 775     // ensure that the returned space is not considered parsable by
 776     // any concurrent GC thread.
 777     size_t hdr_size = oopDesc::header_size();
 778     Copy::fill_to_words(gclab_buf + hdr_size, actual_size - hdr_size, badHeapWordVal);
 779 #endif // ASSERT
 780   }
 781   gclab->set_buf(gclab_buf, actual_size);
 782   return gclab->allocate(size);
 783 }
 784 
 785 HeapWord* ShenandoahHeap::allocate_new_tlab(size_t min_size,
 786                                             size_t requested_size,
 787                                             size_t* actual_size) {
 788   ShenandoahAllocRequest req = ShenandoahAllocRequest::for_tlab(min_size, requested_size);
 789   HeapWord* res = allocate_memory(req);
 790   if (res != NULL) {
 791     *actual_size = req.actual_size();
 792   } else {
 793     *actual_size = 0;
 794   }
 795   return res;
 796 }
 797 
 798 HeapWord* ShenandoahHeap::allocate_new_gclab(size_t min_size,
 799                                              size_t word_size,
 800                                              size_t* actual_size) {
 801   ShenandoahAllocRequest req = ShenandoahAllocRequest::for_gclab(min_size, word_size);
 802   HeapWord* res = allocate_memory(req);
 803   if (res != NULL) {
 804     *actual_size = req.actual_size();
 805   } else {
 806     *actual_size = 0;
 807   }
 808   return res;
 809 }
 810 
 811 HeapWord* ShenandoahHeap::allocate_memory(ShenandoahAllocRequest& req) {
 812   intptr_t pacer_epoch = 0;
 813   bool in_new_region = false;
 814   HeapWord* result = NULL;
 815 
 816   if (req.is_mutator_alloc()) {
 817     if (ShenandoahPacing) {
 818       pacer()->pace_for_alloc(req.size());
 819       pacer_epoch = pacer()->epoch();
 820     }
 821 
 822     if (!ShenandoahAllocFailureALot || !should_inject_alloc_failure()) {
 823       result = allocate_memory_under_lock(req, in_new_region);
 824     }
 825 
 826     // Allocation failed, block until control thread reacted, then retry allocation.
 827     //
 828     // It might happen that one of the threads requesting allocation would unblock
 829     // way later after GC happened, only to fail the second allocation, because
 830     // other threads have already depleted the free storage. In this case, a better
 831     // strategy is to try again, as long as GC makes progress.
 832     //
 833     // Then, we need to make sure the allocation was retried after at least one
 834     // Full GC, which means we want to try more than ShenandoahFullGCThreshold times.
 835 
 836     size_t tries = 0;
 837 
 838     while (result == NULL && _progress_last_gc.is_set()) {
 839       tries++;
 840       control_thread()->handle_alloc_failure(req);
 841       result = allocate_memory_under_lock(req, in_new_region);
 842     }
 843 
 844     while (result == NULL && tries <= ShenandoahFullGCThreshold) {
 845       tries++;
 846       control_thread()->handle_alloc_failure(req);
 847       result = allocate_memory_under_lock(req, in_new_region);
 848     }
 849 
 850   } else {
 851     assert(req.is_gc_alloc(), "Can only accept GC allocs here");
 852     result = allocate_memory_under_lock(req, in_new_region);
 853     // Do not call handle_alloc_failure() here, because we cannot block.
 854     // The allocation failure would be handled by the LRB slowpath with handle_alloc_failure_evac().
 855   }
 856 
 857   if (in_new_region) {
 858     control_thread()->notify_heap_changed();
 859   }
 860 
 861   if (result != NULL) {
 862     size_t requested = req.size();
 863     size_t actual = req.actual_size();
 864 
 865     assert (req.is_lab_alloc() || (requested == actual),
 866             "Only LAB allocations are elastic: %s, requested = " SIZE_FORMAT ", actual = " SIZE_FORMAT,
 867             ShenandoahAllocRequest::alloc_type_to_string(req.type()), requested, actual);
 868 
 869     if (req.is_mutator_alloc()) {
 870       notify_mutator_alloc_words(actual, false);
 871 
 872       // If we requested more than we were granted, give the rest back to pacer.
 873       // This only matters if we are in the same pacing epoch: do not try to unpace
 874       // over the budget for the other phase.
 875       if (ShenandoahPacing && (pacer_epoch > 0) && (requested > actual)) {
 876         pacer()->unpace_for_alloc(pacer_epoch, requested - actual);
 877       }
 878     } else {
 879       increase_used(actual*HeapWordSize);
 880     }
 881   }
 882 
 883   return result;
 884 }
 885 
 886 HeapWord* ShenandoahHeap::allocate_memory_under_lock(ShenandoahAllocRequest& req, bool& in_new_region) {
 887   ShenandoahHeapLocker locker(lock());
 888   return _free_set->allocate(req, in_new_region);
 889 }
 890 
 891 HeapWord* ShenandoahHeap::mem_allocate(size_t size,
 892                                         bool*  gc_overhead_limit_was_exceeded) {
 893   ShenandoahAllocRequest req = ShenandoahAllocRequest::for_shared(size);
 894   return allocate_memory(req);
 895 }
 896 
 897 MetaWord* ShenandoahHeap::satisfy_failed_metadata_allocation(ClassLoaderData* loader_data,
 898                                                              size_t size,
 899                                                              Metaspace::MetadataType mdtype) {
 900   MetaWord* result;
 901 
 902   // Inform metaspace OOM to GC heuristics if class unloading is possible.
 903   if (heuristics()->can_unload_classes()) {
 904     ShenandoahHeuristics* h = heuristics();
 905     h->record_metaspace_oom();
 906   }
 907 
 908   // Expand and retry allocation
 909   result = loader_data->metaspace_non_null()->expand_and_allocate(size, mdtype);
 910   if (result != NULL) {
 911     return result;
 912   }
 913 
 914   // Start full GC
 915   collect(GCCause::_metadata_GC_clear_soft_refs);
 916 
 917   // Retry allocation
 918   result = loader_data->metaspace_non_null()->allocate(size, mdtype);
 919   if (result != NULL) {
 920     return result;
 921   }
 922 
 923   // Expand and retry allocation
 924   result = loader_data->metaspace_non_null()->expand_and_allocate(size, mdtype);
 925   if (result != NULL) {
 926     return result;
 927   }
 928 
 929   // Out of memory
 930   return NULL;
 931 }
 932 
 933 class ShenandoahConcurrentEvacuateRegionObjectClosure : public ObjectClosure {
 934 private:
 935   ShenandoahHeap* const _heap;
 936   Thread* const _thread;
 937 public:
 938   ShenandoahConcurrentEvacuateRegionObjectClosure(ShenandoahHeap* heap) :
 939     _heap(heap), _thread(Thread::current()) {}
 940 
 941   void do_object(oop p) {
 942     shenandoah_assert_marked(NULL, p);
 943     if (!p->is_forwarded()) {
 944       _heap->evacuate_object(p, _thread);
 945     }
 946   }
 947 };
 948 
 949 class ShenandoahEvacuationTask : public AbstractGangTask {
 950 private:
 951   ShenandoahHeap* const _sh;
 952   ShenandoahCollectionSet* const _cs;
 953   bool _concurrent;
 954 public:
 955   ShenandoahEvacuationTask(ShenandoahHeap* sh,
 956                            ShenandoahCollectionSet* cs,
 957                            bool concurrent) :
 958     AbstractGangTask("Parallel Evacuation Task"),
 959     _sh(sh),
 960     _cs(cs),
 961     _concurrent(concurrent)
 962   {}
 963 
 964   void work(uint worker_id) {
 965     if (_concurrent) {
 966       ShenandoahConcurrentWorkerSession worker_session(worker_id);
 967       ShenandoahSuspendibleThreadSetJoiner stsj(ShenandoahSuspendibleWorkers);
 968       ShenandoahEvacOOMScope oom_evac_scope;
 969       do_work();
 970     } else {
 971       ShenandoahParallelWorkerSession worker_session(worker_id);
 972       ShenandoahEvacOOMScope oom_evac_scope;
 973       do_work();
 974     }
 975   }
 976 
 977 private:
 978   void do_work() {
 979     ShenandoahConcurrentEvacuateRegionObjectClosure cl(_sh);
 980     ShenandoahHeapRegion* r;
 981     while ((r =_cs->claim_next()) != NULL) {
 982       assert(r->has_live(), "Region " SIZE_FORMAT " should have been reclaimed early", r->index());
 983       _sh->marked_object_iterate(r, &cl);
 984 
 985       if (ShenandoahPacing) {
 986         _sh->pacer()->report_evac(r->used() >> LogHeapWordSize);
 987       }
 988 
 989       if (_sh->check_cancelled_gc_and_yield(_concurrent)) {
 990         break;
 991       }
 992     }
 993   }
 994 };
 995 
 996 void ShenandoahHeap::trash_cset_regions() {
 997   ShenandoahHeapLocker locker(lock());
 998 
 999   ShenandoahCollectionSet* set = collection_set();
1000   ShenandoahHeapRegion* r;
1001   set->clear_current_index();
1002   while ((r = set->next()) != NULL) {
1003     r->make_trash();
1004   }
1005   collection_set()->clear();
1006 }
1007 
1008 void ShenandoahHeap::print_heap_regions_on(outputStream* st) const {
1009   st->print_cr("Heap Regions:");
1010   st->print_cr("EU=empty-uncommitted, EC=empty-committed, R=regular, H=humongous start, HC=humongous continuation, CS=collection set, T=trash, P=pinned");
1011   st->print_cr("BTE=bottom/top/end, U=used, T=TLAB allocs, G=GCLAB allocs, S=shared allocs, L=live data");
1012   st->print_cr("R=root, CP=critical pins, TAMS=top-at-mark-start, UWM=update watermark");
1013   st->print_cr("SN=alloc sequence number");
1014 
1015   for (size_t i = 0; i < num_regions(); i++) {
1016     get_region(i)->print_on(st);
1017   }
1018 }
1019 
1020 void ShenandoahHeap::trash_humongous_region_at(ShenandoahHeapRegion* start) {
1021   assert(start->is_humongous_start(), "reclaim regions starting with the first one");
1022 
1023   oop humongous_obj = oop(start->bottom());
1024   size_t size = humongous_obj->size();
1025   size_t required_regions = ShenandoahHeapRegion::required_regions(size * HeapWordSize);
1026   size_t index = start->index() + required_regions - 1;
1027 
1028   assert(!start->has_live(), "liveness must be zero");
1029 
1030   for(size_t i = 0; i < required_regions; i++) {
1031     // Reclaim from tail. Otherwise, assertion fails when printing region to trace log,
1032     // as it expects that every region belongs to a humongous region starting with a humongous start region.
1033     ShenandoahHeapRegion* region = get_region(index --);
1034 
1035     assert(region->is_humongous(), "expect correct humongous start or continuation");
1036     assert(!region->is_cset(), "Humongous region should not be in collection set");
1037 
1038     region->make_trash_immediate();
1039   }
1040 }
1041 
1042 class ShenandoahCheckCleanGCLABClosure : public ThreadClosure {
1043 public:
1044   ShenandoahCheckCleanGCLABClosure() {}
1045   void do_thread(Thread* thread) {
1046     PLAB* gclab = ShenandoahThreadLocalData::gclab(thread);
1047     assert(gclab != NULL, "GCLAB should be initialized for %s", thread->name());
1048     assert(gclab->words_remaining() == 0, "GCLAB should not need retirement");
1049   }
1050 };
1051 
1052 class ShenandoahRetireGCLABClosure : public ThreadClosure {
1053 private:
1054   bool const _resize;
1055 public:
1056   ShenandoahRetireGCLABClosure(bool resize) : _resize(resize) {}
1057   void do_thread(Thread* thread) {
1058     PLAB* gclab = ShenandoahThreadLocalData::gclab(thread);
1059     assert(gclab != NULL, "GCLAB should be initialized for %s", thread->name());
1060     gclab->retire();
1061     if (_resize && ShenandoahThreadLocalData::gclab_size(thread) > 0) {
1062       ShenandoahThreadLocalData::set_gclab_size(thread, 0);
1063     }
1064   }
1065 };
1066 
1067 void ShenandoahHeap::labs_make_parsable() {
1068   assert(UseTLAB, "Only call with UseTLAB");
1069 
1070   ShenandoahRetireGCLABClosure cl(false);
1071 
1072   for (JavaThreadIteratorWithHandle jtiwh; JavaThread *t = jtiwh.next(); ) {
1073     ThreadLocalAllocBuffer& tlab = t->tlab();
1074     tlab.make_parsable();
1075     cl.do_thread(t);
1076   }
1077 
1078   workers()->threads_do(&cl);
1079 }
1080 
1081 void ShenandoahHeap::tlabs_retire(bool resize) {
1082   assert(UseTLAB, "Only call with UseTLAB");
1083   assert(!resize || ResizeTLAB, "Only call for resize when ResizeTLAB is enabled");
1084 
1085   ThreadLocalAllocStats stats;
1086 
1087   for (JavaThreadIteratorWithHandle jtiwh; JavaThread *t = jtiwh.next(); ) {
1088     ThreadLocalAllocBuffer& tlab = t->tlab();
1089     tlab.retire(&stats);
1090     if (resize) {
1091       tlab.resize();
1092     }
1093   }
1094 
1095   stats.publish();
1096 
1097 #ifdef ASSERT
1098   ShenandoahCheckCleanGCLABClosure cl;
1099   for (JavaThreadIteratorWithHandle jtiwh; JavaThread *t = jtiwh.next(); ) {
1100     cl.do_thread(t);
1101   }
1102   workers()->threads_do(&cl);
1103 #endif
1104 }
1105 
1106 void ShenandoahHeap::gclabs_retire(bool resize) {
1107   assert(UseTLAB, "Only call with UseTLAB");
1108   assert(!resize || ResizeTLAB, "Only call for resize when ResizeTLAB is enabled");
1109 
1110   ShenandoahRetireGCLABClosure cl(resize);
1111   for (JavaThreadIteratorWithHandle jtiwh; JavaThread *t = jtiwh.next(); ) {
1112     cl.do_thread(t);
1113   }
1114   workers()->threads_do(&cl);
1115 }
1116 
1117 class ShenandoahEvacuateUpdateRootsTask : public AbstractGangTask {
1118 private:
1119   ShenandoahRootEvacuator* _rp;
1120 
1121 public:
1122   ShenandoahEvacuateUpdateRootsTask(ShenandoahRootEvacuator* rp) :
1123     AbstractGangTask("Shenandoah evacuate and update roots"),
1124     _rp(rp) {}
1125 
1126   void work(uint worker_id) {
1127     ShenandoahParallelWorkerSession worker_session(worker_id);
1128     ShenandoahEvacOOMScope oom_evac_scope;
1129     ShenandoahEvacuateUpdateRootsClosure<> cl;
1130     MarkingCodeBlobClosure blobsCl(&cl, CodeBlobToOopClosure::FixRelocations);
1131     _rp->roots_do(worker_id, &cl);
1132   }
1133 };
1134 
1135 void ShenandoahHeap::evacuate_and_update_roots() {
1136 #if COMPILER2_OR_JVMCI
1137   DerivedPointerTable::clear();
1138 #endif
1139   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Only iterate roots while world is stopped");
1140   {
1141     // Include concurrent roots if current cycle can not process those roots concurrently
1142     ShenandoahRootEvacuator rp(workers()->active_workers(),
1143                                ShenandoahPhaseTimings::init_evac,
1144                                !ShenandoahConcurrentRoots::should_do_concurrent_roots(),
1145                                !ShenandoahConcurrentRoots::should_do_concurrent_class_unloading());
1146     ShenandoahEvacuateUpdateRootsTask roots_task(&rp);
1147     workers()->run_task(&roots_task);
1148   }
1149 
1150 #if COMPILER2_OR_JVMCI
1151   DerivedPointerTable::update_pointers();
1152 #endif
1153 }
1154 
1155 // Returns size in bytes
1156 size_t ShenandoahHeap::unsafe_max_tlab_alloc(Thread *thread) const {
1157   if (ShenandoahElasticTLAB) {
1158     // With Elastic TLABs, return the max allowed size, and let the allocation path
1159     // figure out the safe size for current allocation.
1160     return ShenandoahHeapRegion::max_tlab_size_bytes();
1161   } else {
1162     return MIN2(_free_set->unsafe_peek_free(), ShenandoahHeapRegion::max_tlab_size_bytes());
1163   }
1164 }
1165 
1166 size_t ShenandoahHeap::max_tlab_size() const {
1167   // Returns size in words
1168   return ShenandoahHeapRegion::max_tlab_size_words();
1169 }
1170 
1171 void ShenandoahHeap::collect(GCCause::Cause cause) {
1172   control_thread()->request_gc(cause);
1173 }
1174 
1175 void ShenandoahHeap::do_full_collection(bool clear_all_soft_refs) {
1176   //assert(false, "Shouldn't need to do full collections");
1177 }
1178 
1179 HeapWord* ShenandoahHeap::block_start(const void* addr) const {
1180   ShenandoahHeapRegion* r = heap_region_containing(addr);
1181   if (r != NULL) {
1182     return r->block_start(addr);
1183   }
1184   return NULL;
1185 }
1186 
1187 bool ShenandoahHeap::block_is_obj(const HeapWord* addr) const {
1188   ShenandoahHeapRegion* r = heap_region_containing(addr);
1189   return r->block_is_obj(addr);
1190 }
1191 
1192 bool ShenandoahHeap::print_location(outputStream* st, void* addr) const {
1193   return BlockLocationPrinter<ShenandoahHeap>::print_location(st, addr);
1194 }
1195 
1196 jlong ShenandoahHeap::millis_since_last_gc() {
1197   double v = heuristics()->time_since_last_gc() * 1000;
1198   assert(0 <= v && v <= max_jlong, "value should fit: %f", v);
1199   return (jlong)v;
1200 }
1201 
1202 void ShenandoahHeap::prepare_for_verify() {
1203   if (SafepointSynchronize::is_at_safepoint() && UseTLAB) {
1204     labs_make_parsable();
1205   }
1206 }
1207 
1208 void ShenandoahHeap::gc_threads_do(ThreadClosure* tcl) const {
1209   workers()->threads_do(tcl);
1210   if (_safepoint_workers != NULL) {
1211     _safepoint_workers->threads_do(tcl);
1212   }
1213   if (ShenandoahStringDedup::is_enabled()) {
1214     ShenandoahStringDedup::threads_do(tcl);
1215   }
1216 }
1217 
1218 void ShenandoahHeap::print_tracing_info() const {
1219   LogTarget(Info, gc, stats) lt;
1220   if (lt.is_enabled()) {
1221     ResourceMark rm;
1222     LogStream ls(lt);
1223 
1224     phase_timings()->print_global_on(&ls);
1225 
1226     ls.cr();
1227     ls.cr();
1228 
1229     shenandoah_policy()->print_gc_stats(&ls);
1230 
1231     ls.cr();
1232     ls.cr();
1233   }
1234 }
1235 
1236 void ShenandoahHeap::verify(VerifyOption vo) {
1237   if (ShenandoahSafepoint::is_at_shenandoah_safepoint()) {
1238     if (ShenandoahVerify) {
1239       verifier()->verify_generic(vo);
1240     } else {
1241       // TODO: Consider allocating verification bitmaps on demand,
1242       // and turn this on unconditionally.
1243     }
1244   }
1245 }
1246 size_t ShenandoahHeap::tlab_capacity(Thread *thr) const {
1247   return _free_set->capacity();
1248 }
1249 
1250 class ObjectIterateScanRootClosure : public BasicOopIterateClosure {
1251 private:
1252   MarkBitMap* _bitmap;
1253   Stack<oop,mtGC>* _oop_stack;
1254   ShenandoahHeap* const _heap;
1255   ShenandoahMarkingContext* const _marking_context;
1256 
1257   template <class T>
1258   void do_oop_work(T* p) {
1259     T o = RawAccess<>::oop_load(p);
1260     if (!CompressedOops::is_null(o)) {
1261       oop obj = CompressedOops::decode_not_null(o);
1262       if (_heap->is_concurrent_weak_root_in_progress() && !_marking_context->is_marked(obj)) {
1263         // There may be dead oops in weak roots in concurrent root phase, do not touch them.
1264         return;
1265       }
1266       obj = ShenandoahBarrierSet::resolve_forwarded_not_null(obj);
1267 
1268       assert(oopDesc::is_oop(obj), "must be a valid oop");
1269       if (!_bitmap->is_marked(obj)) {
1270         _bitmap->mark(obj);
1271         _oop_stack->push(obj);
1272       }
1273     }
1274   }
1275 public:
1276   ObjectIterateScanRootClosure(MarkBitMap* bitmap, Stack<oop,mtGC>* oop_stack) :
1277     _bitmap(bitmap), _oop_stack(oop_stack), _heap(ShenandoahHeap::heap()),
1278     _marking_context(_heap->marking_context()) {}
1279   void do_oop(oop* p)       { do_oop_work(p); }
1280   void do_oop(narrowOop* p) { do_oop_work(p); }
1281 };
1282 
1283 /*
1284  * This is public API, used in preparation of object_iterate().
1285  * Since we don't do linear scan of heap in object_iterate() (see comment below), we don't
1286  * need to make the heap parsable. For Shenandoah-internal linear heap scans that we can
1287  * control, we call SH::tlabs_retire, SH::gclabs_retire.
1288  */
1289 void ShenandoahHeap::ensure_parsability(bool retire_tlabs) {
1290   // No-op.
1291 }
1292 
1293 /*
1294  * Iterates objects in the heap. This is public API, used for, e.g., heap dumping.
1295  *
1296  * We cannot safely iterate objects by doing a linear scan at random points in time. Linear
1297  * scanning needs to deal with dead objects, which may have dead Klass* pointers (e.g.
1298  * calling oopDesc::size() would crash) or dangling reference fields (crashes) etc. Linear
1299  * scanning therefore depends on having a valid marking bitmap to support it. However, we only
1300  * have a valid marking bitmap after successful marking. In particular, we *don't* have a valid
1301  * marking bitmap during marking, after aborted marking or during/after cleanup (when we just
1302  * wiped the bitmap in preparation for next marking).
1303  *
1304  * For all those reasons, we implement object iteration as a single marking traversal, reporting
1305  * objects as we mark+traverse through the heap, starting from GC roots. JVMTI IterateThroughHeap
1306  * is allowed to report dead objects, but is not required to do so.
1307  */
1308 void ShenandoahHeap::object_iterate(ObjectClosure* cl) {
1309   assert(SafepointSynchronize::is_at_safepoint(), "safe iteration is only available during safepoints");
1310   if (!_aux_bitmap_region_special && !os::commit_memory((char*)_aux_bitmap_region.start(), _aux_bitmap_region.byte_size(), false)) {
1311     log_warning(gc)("Could not commit native memory for auxiliary marking bitmap for heap iteration");
1312     return;
1313   }
1314 
1315   // Reset bitmap
1316   _aux_bit_map.clear();
1317 
1318   Stack<oop,mtGC> oop_stack;
1319 
1320   ObjectIterateScanRootClosure oops(&_aux_bit_map, &oop_stack);
1321 
1322   {
1323     // First, we process GC roots according to current GC cycle.
1324     // This populates the work stack with initial objects.
1325     // It is important to relinquish the associated locks before diving
1326     // into heap dumper.
1327     ShenandoahHeapIterationRootScanner rp;
1328     rp.roots_do(&oops);
1329   }
1330 
1331   // Work through the oop stack to traverse heap.
1332   while (! oop_stack.is_empty()) {
1333     oop obj = oop_stack.pop();
1334     assert(oopDesc::is_oop(obj), "must be a valid oop");
1335     cl->do_object(obj);
1336     obj->oop_iterate(&oops);
1337   }
1338 
1339   assert(oop_stack.is_empty(), "should be empty");
1340 
1341   if (!_aux_bitmap_region_special && !os::uncommit_memory((char*)_aux_bitmap_region.start(), _aux_bitmap_region.byte_size())) {
1342     log_warning(gc)("Could not uncommit native memory for auxiliary marking bitmap for heap iteration");
1343   }
1344 }
1345 
1346 void ShenandoahHeap::run_task(AbstractGangTask* task) {
1347   workers()->run_task(task, workers()->active_workers());
1348 }
1349 
1350 // Keep alive an object that was loaded with AS_NO_KEEPALIVE.
1351 void ShenandoahHeap::keep_alive(oop obj) {
1352   if (is_concurrent_mark_in_progress()) {
1353     ShenandoahBarrierSet::barrier_set()->enqueue(obj);
1354   }
1355 }
1356 
1357 void ShenandoahHeap::heap_region_iterate(ShenandoahHeapRegionClosure* blk) const {
1358   for (size_t i = 0; i < num_regions(); i++) {
1359     ShenandoahHeapRegion* current = get_region(i);
1360     blk->heap_region_do(current);
1361   }
1362 }
1363 
1364 class ShenandoahParallelHeapRegionTask : public AbstractGangTask {
1365 private:
1366   ShenandoahHeap* const _heap;
1367   ShenandoahHeapRegionClosure* const _blk;
1368 
1369   shenandoah_padding(0);
1370   volatile size_t _index;
1371   shenandoah_padding(1);
1372 
1373 public:
1374   ShenandoahParallelHeapRegionTask(ShenandoahHeapRegionClosure* blk) :
1375           AbstractGangTask("Parallel Region Task"),
1376           _heap(ShenandoahHeap::heap()), _blk(blk), _index(0) {}
1377 
1378   void work(uint worker_id) {
1379     ShenandoahParallelWorkerSession worker_session(worker_id);
1380     size_t stride = ShenandoahParallelRegionStride;
1381 
1382     size_t max = _heap->num_regions();
1383     while (_index < max) {
1384       size_t cur = Atomic::fetch_and_add(&_index, stride);
1385       size_t start = cur;
1386       size_t end = MIN2(cur + stride, max);
1387       if (start >= max) break;
1388 
1389       for (size_t i = cur; i < end; i++) {
1390         ShenandoahHeapRegion* current = _heap->get_region(i);
1391         _blk->heap_region_do(current);
1392       }
1393     }
1394   }
1395 };
1396 
1397 void ShenandoahHeap::parallel_heap_region_iterate(ShenandoahHeapRegionClosure* blk) const {
1398   assert(blk->is_thread_safe(), "Only thread-safe closures here");
1399   if (num_regions() > ShenandoahParallelRegionStride) {
1400     ShenandoahParallelHeapRegionTask task(blk);
1401     workers()->run_task(&task);
1402   } else {
1403     heap_region_iterate(blk);
1404   }
1405 }
1406 
1407 class ShenandoahInitMarkUpdateRegionStateClosure : public ShenandoahHeapRegionClosure {
1408 private:
1409   ShenandoahMarkingContext* const _ctx;
1410 public:
1411   ShenandoahInitMarkUpdateRegionStateClosure() : _ctx(ShenandoahHeap::heap()->marking_context()) {}
1412 
1413   void heap_region_do(ShenandoahHeapRegion* r) {
1414     assert(!r->has_live(), "Region " SIZE_FORMAT " should have no live data", r->index());
1415     if (r->is_active()) {
1416       // Check if region needs updating its TAMS. We have updated it already during concurrent
1417       // reset, so it is very likely we don't need to do another write here.
1418       if (_ctx->top_at_mark_start(r) != r->top()) {
1419         _ctx->capture_top_at_mark_start(r);
1420       }
1421     } else {
1422       assert(_ctx->top_at_mark_start(r) == r->top(),
1423              "Region " SIZE_FORMAT " should already have correct TAMS", r->index());
1424     }
1425   }
1426 
1427   bool is_thread_safe() { return true; }
1428 };
1429 
1430 void ShenandoahHeap::op_init_mark() {
1431   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Should be at safepoint");
1432   assert(Thread::current()->is_VM_thread(), "can only do this in VMThread");
1433 
1434   assert(marking_context()->is_bitmap_clear(), "need clear marking bitmap");
1435   assert(!marking_context()->is_complete(), "should not be complete");
1436   assert(!has_forwarded_objects(), "No forwarded objects on this path");
1437 
1438   if (ShenandoahVerify) {
1439     verifier()->verify_before_concmark();
1440   }
1441 
1442   if (VerifyBeforeGC) {
1443     Universe::verify();
1444   }
1445 
1446   set_concurrent_mark_in_progress(true);
1447 
1448   // We need to reset all TLABs because they might be below the TAMS, and we need to mark
1449   // the objects in them. Do not let mutators allocate any new objects in their current TLABs.
1450   // It is also a good place to resize the TLAB sizes for future allocations.
1451   if (UseTLAB) {
1452     ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_manage_tlabs);
1453     tlabs_retire(ResizeTLAB);
1454   }
1455 
1456   {
1457     ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_update_region_states);
1458     ShenandoahInitMarkUpdateRegionStateClosure cl;
1459     parallel_heap_region_iterate(&cl);
1460   }
1461 
1462   // Make above changes visible to worker threads
1463   OrderAccess::fence();
1464 
1465   concurrent_mark()->mark_roots(ShenandoahPhaseTimings::scan_roots);
1466 
1467   if (ShenandoahPacing) {
1468     pacer()->setup_for_mark();
1469   }
1470 
1471   // Arm nmethods for concurrent marking. When a nmethod is about to be executed,
1472   // we need to make sure that all its metadata are marked. alternative is to remark
1473   // thread roots at final mark pause, but it can be potential latency killer.
1474   if (ShenandoahConcurrentRoots::should_do_concurrent_class_unloading()) {
1475     ShenandoahCodeRoots::arm_nmethods();
1476   }
1477 }
1478 
1479 void ShenandoahHeap::op_mark() {
1480   concurrent_mark()->mark_from_roots();
1481 }
1482 
1483 class ShenandoahFinalMarkUpdateRegionStateClosure : public ShenandoahHeapRegionClosure {
1484 private:
1485   ShenandoahMarkingContext* const _ctx;
1486   ShenandoahHeapLock* const _lock;
1487 
1488 public:
1489   ShenandoahFinalMarkUpdateRegionStateClosure() :
1490     _ctx(ShenandoahHeap::heap()->complete_marking_context()), _lock(ShenandoahHeap::heap()->lock()) {}
1491 
1492   void heap_region_do(ShenandoahHeapRegion* r) {
1493     if (r->is_active()) {
1494       // All allocations past TAMS are implicitly live, adjust the region data.
1495       // Bitmaps/TAMS are swapped at this point, so we need to poll complete bitmap.
1496       HeapWord *tams = _ctx->top_at_mark_start(r);
1497       HeapWord *top = r->top();
1498       if (top > tams) {
1499         r->increase_live_data_alloc_words(pointer_delta(top, tams));
1500       }
1501 
1502       // We are about to select the collection set, make sure it knows about
1503       // current pinning status. Also, this allows trashing more regions that
1504       // now have their pinning status dropped.
1505       if (r->is_pinned()) {
1506         if (r->pin_count() == 0) {
1507           ShenandoahHeapLocker locker(_lock);
1508           r->make_unpinned();
1509         }
1510       } else {
1511         if (r->pin_count() > 0) {
1512           ShenandoahHeapLocker locker(_lock);
1513           r->make_pinned();
1514         }
1515       }
1516 
1517       // Remember limit for updating refs. It's guaranteed that we get no
1518       // from-space-refs written from here on.
1519       r->set_update_watermark_at_safepoint(r->top());
1520     } else {
1521       assert(!r->has_live(), "Region " SIZE_FORMAT " should have no live data", r->index());
1522       assert(_ctx->top_at_mark_start(r) == r->top(),
1523              "Region " SIZE_FORMAT " should have correct TAMS", r->index());
1524     }
1525   }
1526 
1527   bool is_thread_safe() { return true; }
1528 };
1529 
1530 void ShenandoahHeap::op_final_mark() {
1531   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Should be at safepoint");
1532   assert(!has_forwarded_objects(), "No forwarded objects on this path");
1533 
1534   // It is critical that we
1535   // evacuate roots right after finishing marking, so that we don't
1536   // get unmarked objects in the roots.
1537 
1538   if (!cancelled_gc()) {
1539     concurrent_mark()->finish_mark_from_roots(/* full_gc = */ false);
1540 
1541     // Marking is completed, deactivate SATB barrier
1542     set_concurrent_mark_in_progress(false);
1543     mark_complete_marking_context();
1544 
1545     parallel_cleaning(false /* full gc*/);
1546 
1547     if (ShenandoahVerify) {
1548       verifier()->verify_roots_no_forwarded();
1549     }
1550 
1551     {
1552       ShenandoahGCPhase phase(ShenandoahPhaseTimings::final_update_region_states);
1553       ShenandoahFinalMarkUpdateRegionStateClosure cl;
1554       parallel_heap_region_iterate(&cl);
1555 
1556       assert_pinned_region_status();
1557     }
1558 
1559     // Retire the TLABs, which will force threads to reacquire their TLABs after the pause.
1560     // This is needed for two reasons. Strong one: new allocations would be with new freeset,
1561     // which would be outside the collection set, so no cset writes would happen there.
1562     // Weaker one: new allocations would happen past update watermark, and so less work would
1563     // be needed for reference updates (would update the large filler instead).
1564     if (UseTLAB) {
1565       ShenandoahGCPhase phase(ShenandoahPhaseTimings::final_manage_labs);
1566       tlabs_retire(false);
1567     }
1568 
1569     {
1570       ShenandoahGCPhase phase(ShenandoahPhaseTimings::choose_cset);
1571       ShenandoahHeapLocker locker(lock());
1572       _collection_set->clear();
1573       heuristics()->choose_collection_set(_collection_set);
1574     }
1575 
1576     {
1577       ShenandoahGCPhase phase(ShenandoahPhaseTimings::final_rebuild_freeset);
1578       ShenandoahHeapLocker locker(lock());
1579       _free_set->rebuild();
1580     }
1581 
1582     if (!is_degenerated_gc_in_progress()) {
1583       prepare_concurrent_roots();
1584       prepare_concurrent_unloading();
1585     }
1586 
1587     // If collection set has candidates, start evacuation.
1588     // Otherwise, bypass the rest of the cycle.
1589     if (!collection_set()->is_empty()) {
1590       ShenandoahGCPhase init_evac(ShenandoahPhaseTimings::init_evac);
1591 
1592       if (ShenandoahVerify) {
1593         verifier()->verify_before_evacuation();
1594       }
1595 
1596       set_evacuation_in_progress(true);
1597       // From here on, we need to update references.
1598       set_has_forwarded_objects(true);
1599 
1600       if (!is_degenerated_gc_in_progress()) {
1601         if (ShenandoahConcurrentRoots::should_do_concurrent_class_unloading()) {
1602           ShenandoahCodeRoots::arm_nmethods();
1603         }
1604         evacuate_and_update_roots();
1605       }
1606 
1607       if (ShenandoahPacing) {
1608         pacer()->setup_for_evac();
1609       }
1610 
1611       if (ShenandoahVerify) {
1612         // If OOM while evacuating/updating of roots, there is no guarantee of their consistencies
1613         if (!cancelled_gc()) {
1614           ShenandoahRootVerifier::RootTypes types = ShenandoahRootVerifier::None;
1615           if (ShenandoahConcurrentRoots::should_do_concurrent_roots()) {
1616             types = ShenandoahRootVerifier::combine(ShenandoahRootVerifier::JNIHandleRoots, ShenandoahRootVerifier::WeakRoots);
1617             types = ShenandoahRootVerifier::combine(types, ShenandoahRootVerifier::CLDGRoots);
1618             types = ShenandoahRootVerifier::combine(types, ShenandoahRootVerifier::StringDedupRoots);
1619           }
1620 
1621           if (ShenandoahConcurrentRoots::should_do_concurrent_class_unloading()) {
1622             types = ShenandoahRootVerifier::combine(types, ShenandoahRootVerifier::CodeRoots);
1623           }
1624           verifier()->verify_roots_no_forwarded_except(types);
1625         }
1626         verifier()->verify_during_evacuation();
1627       }
1628     } else {
1629       if (ShenandoahVerify) {
1630         verifier()->verify_after_concmark();
1631       }
1632 
1633       if (VerifyAfterGC) {
1634         Universe::verify();
1635       }
1636     }
1637 
1638   } else {
1639     // If this cycle was updating references, we need to keep the has_forwarded_objects
1640     // flag on, for subsequent phases to deal with it.
1641     concurrent_mark()->cancel();
1642     set_concurrent_mark_in_progress(false);
1643 
1644     if (process_references()) {
1645       // Abandon reference processing right away: pre-cleaning must have failed.
1646       ReferenceProcessor *rp = ref_processor();
1647       rp->disable_discovery();
1648       rp->abandon_partial_discovery();
1649       rp->verify_no_references_recorded();
1650     }
1651   }
1652 }
1653 
1654 void ShenandoahHeap::op_conc_evac() {
1655   ShenandoahEvacuationTask task(this, _collection_set, true);
1656   workers()->run_task(&task);
1657 }
1658 
1659 void ShenandoahHeap::op_stw_evac() {
1660   ShenandoahEvacuationTask task(this, _collection_set, false);
1661   workers()->run_task(&task);
1662 }
1663 
1664 void ShenandoahHeap::op_updaterefs() {
1665   update_heap_references(true);
1666 }
1667 
1668 void ShenandoahHeap::op_cleanup_early() {
1669   free_set()->recycle_trash();
1670 }
1671 
1672 void ShenandoahHeap::op_cleanup_complete() {
1673   free_set()->recycle_trash();
1674 }
1675 
1676 class ShenandoahConcurrentRootsEvacUpdateTask : public AbstractGangTask {
1677 private:
1678   ShenandoahVMRoots<true /*concurrent*/>        _vm_roots;
1679   ShenandoahClassLoaderDataRoots<true /*concurrent*/, false /*single threaded*/> _cld_roots;
1680 
1681 public:
1682   ShenandoahConcurrentRootsEvacUpdateTask(ShenandoahPhaseTimings::Phase phase) :
1683     AbstractGangTask("Shenandoah Evacuate/Update Concurrent Strong Roots Task"),
1684     _vm_roots(phase),
1685     _cld_roots(phase, ShenandoahHeap::heap()->workers()->active_workers()) {}
1686 
1687   void work(uint worker_id) {
1688     ShenandoahConcurrentWorkerSession worker_session(worker_id);
1689     ShenandoahEvacOOMScope oom;
1690     {
1691       // vm_roots and weak_roots are OopStorage backed roots, concurrent iteration
1692       // may race against OopStorage::release() calls.
1693       ShenandoahEvacUpdateOopStorageRootsClosure cl;
1694       _vm_roots.oops_do<ShenandoahEvacUpdateOopStorageRootsClosure>(&cl, worker_id);
1695     }
1696 
1697     {
1698       ShenandoahEvacuateUpdateRootsClosure<> cl;
1699       CLDToOopClosure clds(&cl, ClassLoaderData::_claim_strong);
1700       _cld_roots.cld_do(&clds, worker_id);
1701     }
1702   }
1703 };
1704 
1705 class ShenandoahEvacUpdateCleanupOopStorageRootsClosure : public BasicOopIterateClosure {
1706 private:
1707   ShenandoahHeap* const _heap;
1708   ShenandoahMarkingContext* const _mark_context;
1709   bool  _evac_in_progress;
1710   Thread* const _thread;
1711 
1712 public:
1713   ShenandoahEvacUpdateCleanupOopStorageRootsClosure();
1714   void do_oop(oop* p);
1715   void do_oop(narrowOop* p);
1716 };
1717 
1718 ShenandoahEvacUpdateCleanupOopStorageRootsClosure::ShenandoahEvacUpdateCleanupOopStorageRootsClosure() :
1719   _heap(ShenandoahHeap::heap()),
1720   _mark_context(ShenandoahHeap::heap()->marking_context()),
1721   _evac_in_progress(ShenandoahHeap::heap()->is_evacuation_in_progress()),
1722   _thread(Thread::current()) {
1723 }
1724 
1725 void ShenandoahEvacUpdateCleanupOopStorageRootsClosure::do_oop(oop* p) {
1726   const oop obj = RawAccess<>::oop_load(p);
1727   if (!CompressedOops::is_null(obj)) {
1728     if (!_mark_context->is_marked(obj)) {
1729       shenandoah_assert_correct(p, obj);
1730       Atomic::cmpxchg(p, obj, oop(NULL));
1731     } else if (_evac_in_progress && _heap->in_collection_set(obj)) {
1732       oop resolved = ShenandoahBarrierSet::resolve_forwarded_not_null(obj);
1733       if (resolved == obj) {
1734         resolved = _heap->evacuate_object(obj, _thread);
1735       }
1736       Atomic::cmpxchg(p, obj, resolved);
1737       assert(_heap->cancelled_gc() ||
1738              _mark_context->is_marked(resolved) && !_heap->in_collection_set(resolved),
1739              "Sanity");
1740     }
1741   }
1742 }
1743 
1744 void ShenandoahEvacUpdateCleanupOopStorageRootsClosure::do_oop(narrowOop* p) {
1745   ShouldNotReachHere();
1746 }
1747 
1748 class ShenandoahIsCLDAliveClosure : public CLDClosure {
1749 public:
1750   void do_cld(ClassLoaderData* cld) {
1751     cld->is_alive();
1752   }
1753 };
1754 
1755 class ShenandoahIsNMethodAliveClosure: public NMethodClosure {
1756 public:
1757   void do_nmethod(nmethod* n) {
1758     n->is_unloading();
1759   }
1760 };
1761 
1762 // This task not only evacuates/updates marked weak roots, but also "NULL"
1763 // dead weak roots.
1764 class ShenandoahConcurrentWeakRootsEvacUpdateTask : public AbstractGangTask {
1765 private:
1766   ShenandoahVMWeakRoots<true /*concurrent*/> _vm_roots;
1767 
1768   // Roots related to concurrent class unloading
1769   ShenandoahClassLoaderDataRoots<true /* concurrent */, false /* single thread*/>
1770                                              _cld_roots;
1771   ShenandoahConcurrentNMethodIterator        _nmethod_itr;
1772   ShenandoahConcurrentStringDedupRoots       _dedup_roots;
1773   bool                                       _concurrent_class_unloading;
1774 
1775 public:
1776   ShenandoahConcurrentWeakRootsEvacUpdateTask(ShenandoahPhaseTimings::Phase phase) :
1777     AbstractGangTask("Shenandoah Concurrent Weak Root Task"),
1778     _vm_roots(phase),
1779     _cld_roots(phase, ShenandoahHeap::heap()->workers()->active_workers()),
1780     _nmethod_itr(ShenandoahCodeRoots::table()),
1781     _dedup_roots(phase),
1782     _concurrent_class_unloading(ShenandoahConcurrentRoots::should_do_concurrent_class_unloading()) {
1783     if (_concurrent_class_unloading) {
1784       MutexLocker mu(CodeCache_lock, Mutex::_no_safepoint_check_flag);
1785       _nmethod_itr.nmethods_do_begin();
1786     }
1787   }
1788 
1789   ~ShenandoahConcurrentWeakRootsEvacUpdateTask() {
1790     if (_concurrent_class_unloading) {
1791       MutexLocker mu(CodeCache_lock, Mutex::_no_safepoint_check_flag);
1792       _nmethod_itr.nmethods_do_end();
1793     }
1794     // Notify runtime data structures of potentially dead oops
1795     _vm_roots.report_num_dead();
1796   }
1797 
1798   void work(uint worker_id) {
1799     ShenandoahConcurrentWorkerSession worker_session(worker_id);
1800     {
1801       ShenandoahEvacOOMScope oom;
1802       // jni_roots and weak_roots are OopStorage backed roots, concurrent iteration
1803       // may race against OopStorage::release() calls.
1804       ShenandoahEvacUpdateCleanupOopStorageRootsClosure cl;
1805       _vm_roots.oops_do(&cl, worker_id);
1806 
1807       // String dedup weak roots
1808       ShenandoahForwardedIsAliveClosure is_alive;
1809       ShenandoahEvacuateUpdateRootsClosure<MO_RELEASE> keep_alive;
1810       _dedup_roots.oops_do(&is_alive, &keep_alive, worker_id);
1811     }
1812 
1813     // If we are going to perform concurrent class unloading later on, we need to
1814     // cleanup the weak oops in CLD and determinate nmethod's unloading state, so that we
1815     // can cleanup immediate garbage sooner.
1816     if (_concurrent_class_unloading) {
1817       // Applies ShenandoahIsCLDAlive closure to CLDs, native barrier will either NULL the
1818       // CLD's holder or evacuate it.
1819       ShenandoahIsCLDAliveClosure is_cld_alive;
1820       _cld_roots.cld_do(&is_cld_alive, worker_id);
1821 
1822       // Applies ShenandoahIsNMethodAliveClosure to registered nmethods.
1823       // The closure calls nmethod->is_unloading(). The is_unloading
1824       // state is cached, therefore, during concurrent class unloading phase,
1825       // we will not touch the metadata of unloading nmethods
1826       ShenandoahIsNMethodAliveClosure is_nmethod_alive;
1827       _nmethod_itr.nmethods_do(&is_nmethod_alive);
1828     }
1829   }
1830 };
1831 
1832 void ShenandoahHeap::op_weak_roots() {
1833   if (is_concurrent_weak_root_in_progress()) {
1834     // Concurrent weak root processing
1835     {
1836       ShenandoahTimingsTracker t(ShenandoahPhaseTimings::conc_weak_roots_work);
1837       ShenandoahGCWorkerPhase worker_phase(ShenandoahPhaseTimings::conc_weak_roots_work);
1838       ShenandoahConcurrentWeakRootsEvacUpdateTask task(ShenandoahPhaseTimings::conc_weak_roots_work);
1839       workers()->run_task(&task);
1840       if (!ShenandoahConcurrentRoots::should_do_concurrent_class_unloading()) {
1841         set_concurrent_weak_root_in_progress(false);
1842       }
1843     }
1844 
1845     // Perform handshake to flush out dead oops
1846     {
1847       ShenandoahTimingsTracker t(ShenandoahPhaseTimings::conc_weak_roots_rendezvous);
1848       ShenandoahRendezvousClosure cl;
1849       Handshake::execute(&cl);
1850     }
1851   }
1852 }
1853 
1854 void ShenandoahHeap::op_class_unloading() {
1855   assert (is_concurrent_weak_root_in_progress() &&
1856           ShenandoahConcurrentRoots::should_do_concurrent_class_unloading(),
1857           "Checked by caller");
1858   _unloader.unload();
1859   set_concurrent_weak_root_in_progress(false);
1860 }
1861 
1862 void ShenandoahHeap::op_strong_roots() {
1863   assert(is_concurrent_strong_root_in_progress(), "Checked by caller");
1864   ShenandoahConcurrentRootsEvacUpdateTask task(ShenandoahPhaseTimings::conc_strong_roots);
1865   workers()->run_task(&task);
1866   set_concurrent_strong_root_in_progress(false);
1867 }
1868 
1869 class ShenandoahResetUpdateRegionStateClosure : public ShenandoahHeapRegionClosure {
1870 private:
1871   ShenandoahMarkingContext* const _ctx;
1872 public:
1873   ShenandoahResetUpdateRegionStateClosure() : _ctx(ShenandoahHeap::heap()->marking_context()) {}
1874 
1875   void heap_region_do(ShenandoahHeapRegion* r) {
1876     if (r->is_active()) {
1877       // Reset live data and set TAMS optimistically. We would recheck these under the pause
1878       // anyway to capture any updates that happened since now.
1879       r->clear_live_data();
1880       _ctx->capture_top_at_mark_start(r);
1881     }
1882   }
1883 
1884   bool is_thread_safe() { return true; }
1885 };
1886 
1887 void ShenandoahHeap::op_reset() {
1888   if (ShenandoahPacing) {
1889     pacer()->setup_for_reset();
1890   }
1891   reset_mark_bitmap();
1892 
1893   ShenandoahResetUpdateRegionStateClosure cl;
1894   parallel_heap_region_iterate(&cl);
1895 }
1896 
1897 void ShenandoahHeap::op_preclean() {
1898   if (ShenandoahPacing) {
1899     pacer()->setup_for_preclean();
1900   }
1901   concurrent_mark()->preclean_weak_refs();
1902 }
1903 
1904 void ShenandoahHeap::op_full(GCCause::Cause cause) {
1905   ShenandoahMetricsSnapshot metrics;
1906   metrics.snap_before();
1907 
1908   full_gc()->do_it(cause);
1909 
1910   metrics.snap_after();
1911 
1912   if (metrics.is_good_progress()) {
1913     _progress_last_gc.set();
1914   } else {
1915     // Nothing to do. Tell the allocation path that we have failed to make
1916     // progress, and it can finally fail.
1917     _progress_last_gc.unset();
1918   }
1919 }
1920 
1921 void ShenandoahHeap::op_degenerated(ShenandoahDegenPoint point) {
1922   // Degenerated GC is STW, but it can also fail. Current mechanics communicates
1923   // GC failure via cancelled_concgc() flag. So, if we detect the failure after
1924   // some phase, we have to upgrade the Degenerate GC to Full GC.
1925 
1926   clear_cancelled_gc();
1927 
1928   ShenandoahMetricsSnapshot metrics;
1929   metrics.snap_before();
1930 
1931   switch (point) {
1932     // The cases below form the Duff's-like device: it describes the actual GC cycle,
1933     // but enters it at different points, depending on which concurrent phase had
1934     // degenerated.
1935 
1936     case _degenerated_outside_cycle:
1937       // We have degenerated from outside the cycle, which means something is bad with
1938       // the heap, most probably heavy humongous fragmentation, or we are very low on free
1939       // space. It makes little sense to wait for Full GC to reclaim as much as it can, when
1940       // we can do the most aggressive degen cycle, which includes processing references and
1941       // class unloading, unless those features are explicitly disabled.
1942       //
1943       // Note that we can only do this for "outside-cycle" degens, otherwise we would risk
1944       // changing the cycle parameters mid-cycle during concurrent -> degenerated handover.
1945       set_process_references(heuristics()->can_process_references());
1946       set_unload_classes(heuristics()->can_unload_classes());
1947 
1948       op_reset();
1949 
1950       op_init_mark();
1951       if (cancelled_gc()) {
1952         op_degenerated_fail();
1953         return;
1954       }
1955 
1956     case _degenerated_mark:
1957       op_final_mark();
1958       if (cancelled_gc()) {
1959         op_degenerated_fail();
1960         return;
1961       }
1962 
1963       if (!has_forwarded_objects() && ShenandoahConcurrentRoots::can_do_concurrent_class_unloading()) {
1964         // Disarm nmethods that armed for concurrent mark. On normal cycle, it would
1965         // be disarmed while conc-roots phase is running.
1966         // TODO: Call op_conc_roots() here instead
1967         ShenandoahCodeRoots::disarm_nmethods();
1968       }
1969 
1970       op_cleanup_early();
1971 
1972     case _degenerated_evac:
1973       // If heuristics thinks we should do the cycle, this flag would be set,
1974       // and we can do evacuation. Otherwise, it would be the shortcut cycle.
1975       if (is_evacuation_in_progress()) {
1976 
1977         // Degeneration under oom-evac protocol might have left some objects in
1978         // collection set un-evacuated. Restart evacuation from the beginning to
1979         // capture all objects. For all the objects that are already evacuated,
1980         // it would be a simple check, which is supposed to be fast. This is also
1981         // safe to do even without degeneration, as CSet iterator is at beginning
1982         // in preparation for evacuation anyway.
1983         //
1984         // Before doing that, we need to make sure we never had any cset-pinned
1985         // regions. This may happen if allocation failure happened when evacuating
1986         // the about-to-be-pinned object, oom-evac protocol left the object in
1987         // the collection set, and then the pin reached the cset region. If we continue
1988         // the cycle here, we would trash the cset and alive objects in it. To avoid
1989         // it, we fail degeneration right away and slide into Full GC to recover.
1990 
1991         {
1992           sync_pinned_region_status();
1993           collection_set()->clear_current_index();
1994 
1995           ShenandoahHeapRegion* r;
1996           while ((r = collection_set()->next()) != NULL) {
1997             if (r->is_pinned()) {
1998               cancel_gc(GCCause::_shenandoah_upgrade_to_full_gc);
1999               op_degenerated_fail();
2000               return;
2001             }
2002           }
2003 
2004           collection_set()->clear_current_index();
2005         }
2006 
2007         op_stw_evac();
2008         if (cancelled_gc()) {
2009           op_degenerated_fail();
2010           return;
2011         }
2012       }
2013 
2014       // If heuristics thinks we should do the cycle, this flag would be set,
2015       // and we need to do update-refs. Otherwise, it would be the shortcut cycle.
2016       if (has_forwarded_objects()) {
2017         op_init_updaterefs();
2018         if (cancelled_gc()) {
2019           op_degenerated_fail();
2020           return;
2021         }
2022       }
2023 
2024     case _degenerated_updaterefs:
2025       if (has_forwarded_objects()) {
2026         op_final_updaterefs();
2027         if (cancelled_gc()) {
2028           op_degenerated_fail();
2029           return;
2030         }
2031       }
2032 
2033       op_cleanup_complete();
2034       break;
2035 
2036     default:
2037       ShouldNotReachHere();
2038   }
2039 
2040   if (ShenandoahVerify) {
2041     verifier()->verify_after_degenerated();
2042   }
2043 
2044   if (VerifyAfterGC) {
2045     Universe::verify();
2046   }
2047 
2048   metrics.snap_after();
2049 
2050   // Check for futility and fail. There is no reason to do several back-to-back Degenerated cycles,
2051   // because that probably means the heap is overloaded and/or fragmented.
2052   if (!metrics.is_good_progress()) {
2053     _progress_last_gc.unset();
2054     cancel_gc(GCCause::_shenandoah_upgrade_to_full_gc);
2055     op_degenerated_futile();
2056   } else {
2057     _progress_last_gc.set();
2058   }
2059 }
2060 
2061 void ShenandoahHeap::op_degenerated_fail() {
2062   log_info(gc)("Cannot finish degeneration, upgrading to Full GC");
2063   shenandoah_policy()->record_degenerated_upgrade_to_full();
2064   op_full(GCCause::_shenandoah_upgrade_to_full_gc);
2065 }
2066 
2067 void ShenandoahHeap::op_degenerated_futile() {
2068   shenandoah_policy()->record_degenerated_upgrade_to_full();
2069   op_full(GCCause::_shenandoah_upgrade_to_full_gc);
2070 }
2071 
2072 void ShenandoahHeap::force_satb_flush_all_threads() {
2073   if (!is_concurrent_mark_in_progress()) {
2074     // No need to flush SATBs
2075     return;
2076   }
2077 
2078   for (JavaThreadIteratorWithHandle jtiwh; JavaThread *t = jtiwh.next(); ) {
2079     ShenandoahThreadLocalData::set_force_satb_flush(t, true);
2080   }
2081   // The threads are not "acquiring" their thread-local data, but it does not
2082   // hurt to "release" the updates here anyway.
2083   OrderAccess::fence();
2084 }
2085 
2086 void ShenandoahHeap::set_gc_state_all_threads(char state) {
2087   for (JavaThreadIteratorWithHandle jtiwh; JavaThread *t = jtiwh.next(); ) {
2088     ShenandoahThreadLocalData::set_gc_state(t, state);
2089   }
2090 }
2091 
2092 void ShenandoahHeap::set_gc_state_mask(uint mask, bool value) {
2093   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Should really be Shenandoah safepoint");
2094   _gc_state.set_cond(mask, value);
2095   set_gc_state_all_threads(_gc_state.raw_value());
2096 }
2097 
2098 void ShenandoahHeap::set_concurrent_mark_in_progress(bool in_progress) {
2099   if (has_forwarded_objects()) {
2100     set_gc_state_mask(MARKING | UPDATEREFS, in_progress);
2101   } else {
2102     set_gc_state_mask(MARKING, in_progress);
2103   }
2104   ShenandoahBarrierSet::satb_mark_queue_set().set_active_all_threads(in_progress, !in_progress);
2105 }
2106 
2107 void ShenandoahHeap::set_evacuation_in_progress(bool in_progress) {
2108   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Only call this at safepoint");
2109   set_gc_state_mask(EVACUATION, in_progress);
2110 }
2111 
2112 void ShenandoahHeap::set_concurrent_strong_root_in_progress(bool in_progress) {
2113   assert(ShenandoahConcurrentRoots::can_do_concurrent_roots(), "Why set the flag?");
2114   if (in_progress) {
2115     _concurrent_strong_root_in_progress.set();
2116   } else {
2117     _concurrent_strong_root_in_progress.unset();
2118   }
2119 }
2120 
2121 void ShenandoahHeap::set_concurrent_weak_root_in_progress(bool in_progress) {
2122   assert(ShenandoahConcurrentRoots::can_do_concurrent_roots(), "Why set the flag?");
2123   if (in_progress) {
2124     _concurrent_weak_root_in_progress.set();
2125   } else {
2126     _concurrent_weak_root_in_progress.unset();
2127   }
2128 }
2129 
2130 void ShenandoahHeap::ref_processing_init() {
2131   assert(_max_workers > 0, "Sanity");
2132 
2133   _ref_processor =
2134     new ReferenceProcessor(&_subject_to_discovery,  // is_subject_to_discovery
2135                            _ref_proc_mt_processing, // MT processing
2136                            _max_workers,            // Degree of MT processing
2137                            _ref_proc_mt_discovery,  // MT discovery
2138                            _max_workers,            // Degree of MT discovery
2139                            false,                   // Reference discovery is not atomic
2140                            NULL,                    // No closure, should be installed before use
2141                            true);                   // Scale worker threads
2142 
2143   shenandoah_assert_rp_isalive_not_installed();
2144 }
2145 
2146 GCTracer* ShenandoahHeap::tracer() {
2147   return shenandoah_policy()->tracer();
2148 }
2149 
2150 size_t ShenandoahHeap::tlab_used(Thread* thread) const {
2151   return _free_set->used();
2152 }
2153 
2154 bool ShenandoahHeap::try_cancel_gc() {
2155   while (true) {
2156     jbyte prev = _cancelled_gc.cmpxchg(CANCELLED, CANCELLABLE);
2157     if (prev == CANCELLABLE) return true;
2158     else if (prev == CANCELLED) return false;
2159     assert(ShenandoahSuspendibleWorkers, "should not get here when not using suspendible workers");
2160     assert(prev == NOT_CANCELLED, "must be NOT_CANCELLED");
2161     if (Thread::current()->is_Java_thread()) {
2162       // We need to provide a safepoint here, otherwise we might
2163       // spin forever if a SP is pending.
2164       ThreadBlockInVM sp(JavaThread::current());
2165       SpinPause();
2166     }
2167   }
2168 }
2169 
2170 void ShenandoahHeap::cancel_gc(GCCause::Cause cause) {
2171   if (try_cancel_gc()) {
2172     FormatBuffer<> msg("Cancelling GC: %s", GCCause::to_string(cause));
2173     log_info(gc)("%s", msg.buffer());
2174     Events::log(Thread::current(), "%s", msg.buffer());
2175   }
2176 }
2177 
2178 uint ShenandoahHeap::max_workers() {
2179   return _max_workers;
2180 }
2181 
2182 void ShenandoahHeap::stop() {
2183   // The shutdown sequence should be able to terminate when GC is running.
2184 
2185   // Step 0. Notify policy to disable event recording.
2186   _shenandoah_policy->record_shutdown();
2187 
2188   // Step 1. Notify control thread that we are in shutdown.
2189   // Note that we cannot do that with stop(), because stop() is blocking and waits for the actual shutdown.
2190   // Doing stop() here would wait for the normal GC cycle to complete, never falling through to cancel below.
2191   control_thread()->prepare_for_graceful_shutdown();
2192 
2193   // Step 2. Notify GC workers that we are cancelling GC.
2194   cancel_gc(GCCause::_shenandoah_stop_vm);
2195 
2196   // Step 3. Wait until GC worker exits normally.
2197   control_thread()->stop();
2198 
2199   // Step 4. Stop String Dedup thread if it is active
2200   if (ShenandoahStringDedup::is_enabled()) {
2201     ShenandoahStringDedup::stop();
2202   }
2203 }
2204 
2205 void ShenandoahHeap::stw_unload_classes(bool full_gc) {
2206   if (!unload_classes()) return;
2207 
2208   // Unload classes and purge SystemDictionary.
2209   {
2210     ShenandoahGCPhase phase(full_gc ?
2211                             ShenandoahPhaseTimings::full_gc_purge_class_unload :
2212                             ShenandoahPhaseTimings::purge_class_unload);
2213     bool purged_class = SystemDictionary::do_unloading(gc_timer());
2214 
2215     ShenandoahIsAliveSelector is_alive;
2216     uint num_workers = _workers->active_workers();
2217     ShenandoahClassUnloadingTask unlink_task(is_alive.is_alive_closure(), num_workers, purged_class);
2218     _workers->run_task(&unlink_task);
2219   }
2220 
2221   {
2222     ShenandoahGCPhase phase(full_gc ?
2223                             ShenandoahPhaseTimings::full_gc_purge_cldg :
2224                             ShenandoahPhaseTimings::purge_cldg);
2225     ClassLoaderDataGraph::purge();
2226   }
2227   // Resize and verify metaspace
2228   MetaspaceGC::compute_new_size();
2229   MetaspaceUtils::verify_metrics();
2230 }
2231 
2232 // Weak roots are either pre-evacuated (final mark) or updated (final updaterefs),
2233 // so they should not have forwarded oops.
2234 // However, we do need to "null" dead oops in the roots, if can not be done
2235 // in concurrent cycles.
2236 void ShenandoahHeap::stw_process_weak_roots(bool full_gc) {
2237   ShenandoahGCPhase root_phase(full_gc ?
2238                                ShenandoahPhaseTimings::full_gc_purge :
2239                                ShenandoahPhaseTimings::purge);
2240   uint num_workers = _workers->active_workers();
2241   ShenandoahPhaseTimings::Phase timing_phase = full_gc ?
2242                                                ShenandoahPhaseTimings::full_gc_purge_weak_par :
2243                                                ShenandoahPhaseTimings::purge_weak_par;
2244   ShenandoahGCPhase phase(timing_phase);
2245   ShenandoahGCWorkerPhase worker_phase(timing_phase);
2246 
2247   // Cleanup weak roots
2248   if (has_forwarded_objects()) {
2249     ShenandoahForwardedIsAliveClosure is_alive;
2250     ShenandoahUpdateRefsClosure keep_alive;
2251     ShenandoahParallelWeakRootsCleaningTask<ShenandoahForwardedIsAliveClosure, ShenandoahUpdateRefsClosure>
2252       cleaning_task(timing_phase, &is_alive, &keep_alive, num_workers, !ShenandoahConcurrentRoots::should_do_concurrent_class_unloading());
2253     _workers->run_task(&cleaning_task);
2254   } else {
2255     ShenandoahIsAliveClosure is_alive;
2256 #ifdef ASSERT
2257     ShenandoahAssertNotForwardedClosure verify_cl;
2258     ShenandoahParallelWeakRootsCleaningTask<ShenandoahIsAliveClosure, ShenandoahAssertNotForwardedClosure>
2259       cleaning_task(timing_phase, &is_alive, &verify_cl, num_workers, !ShenandoahConcurrentRoots::should_do_concurrent_class_unloading());
2260 #else
2261     ShenandoahParallelWeakRootsCleaningTask<ShenandoahIsAliveClosure, DoNothingClosure>
2262       cleaning_task(timing_phase, &is_alive, &do_nothing_cl, num_workers, !ShenandoahConcurrentRoots::should_do_concurrent_class_unloading());
2263 #endif
2264     _workers->run_task(&cleaning_task);
2265   }
2266 }
2267 
2268 void ShenandoahHeap::parallel_cleaning(bool full_gc) {
2269   assert(SafepointSynchronize::is_at_safepoint(), "Must be at a safepoint");
2270   stw_process_weak_roots(full_gc);
2271   if (!ShenandoahConcurrentRoots::should_do_concurrent_class_unloading()) {
2272     stw_unload_classes(full_gc);
2273   }
2274 }
2275 
2276 void ShenandoahHeap::set_has_forwarded_objects(bool cond) {
2277   set_gc_state_mask(HAS_FORWARDED, cond);
2278 }
2279 
2280 void ShenandoahHeap::set_process_references(bool pr) {
2281   _process_references.set_cond(pr);
2282 }
2283 
2284 void ShenandoahHeap::set_unload_classes(bool uc) {
2285   _unload_classes.set_cond(uc);
2286 }
2287 
2288 bool ShenandoahHeap::process_references() const {
2289   return _process_references.is_set();
2290 }
2291 
2292 bool ShenandoahHeap::unload_classes() const {
2293   return _unload_classes.is_set();
2294 }
2295 
2296 address ShenandoahHeap::in_cset_fast_test_addr() {
2297   ShenandoahHeap* heap = ShenandoahHeap::heap();
2298   assert(heap->collection_set() != NULL, "Sanity");
2299   return (address) heap->collection_set()->biased_map_address();
2300 }
2301 
2302 address ShenandoahHeap::cancelled_gc_addr() {
2303   return (address) ShenandoahHeap::heap()->_cancelled_gc.addr_of();
2304 }
2305 
2306 address ShenandoahHeap::gc_state_addr() {
2307   return (address) ShenandoahHeap::heap()->_gc_state.addr_of();
2308 }
2309 
2310 size_t ShenandoahHeap::bytes_allocated_since_gc_start() {
2311   return Atomic::load_acquire(&_bytes_allocated_since_gc_start);
2312 }
2313 
2314 void ShenandoahHeap::reset_bytes_allocated_since_gc_start() {
2315   Atomic::release_store_fence(&_bytes_allocated_since_gc_start, (size_t)0);
2316 }
2317 
2318 void ShenandoahHeap::set_degenerated_gc_in_progress(bool in_progress) {
2319   _degenerated_gc_in_progress.set_cond(in_progress);
2320 }
2321 
2322 void ShenandoahHeap::set_full_gc_in_progress(bool in_progress) {
2323   _full_gc_in_progress.set_cond(in_progress);
2324 }
2325 
2326 void ShenandoahHeap::set_full_gc_move_in_progress(bool in_progress) {
2327   assert (is_full_gc_in_progress(), "should be");
2328   _full_gc_move_in_progress.set_cond(in_progress);
2329 }
2330 
2331 void ShenandoahHeap::set_update_refs_in_progress(bool in_progress) {
2332   set_gc_state_mask(UPDATEREFS, in_progress);
2333 }
2334 
2335 void ShenandoahHeap::register_nmethod(nmethod* nm) {
2336   ShenandoahCodeRoots::register_nmethod(nm);
2337 }
2338 
2339 void ShenandoahHeap::unregister_nmethod(nmethod* nm) {
2340   ShenandoahCodeRoots::unregister_nmethod(nm);
2341 }
2342 
2343 void ShenandoahHeap::flush_nmethod(nmethod* nm) {
2344   ShenandoahCodeRoots::flush_nmethod(nm);
2345 }
2346 
2347 oop ShenandoahHeap::pin_object(JavaThread* thr, oop o) {
2348   heap_region_containing(o)->record_pin();
2349   return o;
2350 }
2351 
2352 void ShenandoahHeap::unpin_object(JavaThread* thr, oop o) {
2353   heap_region_containing(o)->record_unpin();
2354 }
2355 
2356 void ShenandoahHeap::sync_pinned_region_status() {
2357   ShenandoahHeapLocker locker(lock());
2358 
2359   for (size_t i = 0; i < num_regions(); i++) {
2360     ShenandoahHeapRegion *r = get_region(i);
2361     if (r->is_active()) {
2362       if (r->is_pinned()) {
2363         if (r->pin_count() == 0) {
2364           r->make_unpinned();
2365         }
2366       } else {
2367         if (r->pin_count() > 0) {
2368           r->make_pinned();
2369         }
2370       }
2371     }
2372   }
2373 
2374   assert_pinned_region_status();
2375 }
2376 
2377 #ifdef ASSERT
2378 void ShenandoahHeap::assert_pinned_region_status() {
2379   for (size_t i = 0; i < num_regions(); i++) {
2380     ShenandoahHeapRegion* r = get_region(i);
2381     assert((r->is_pinned() && r->pin_count() > 0) || (!r->is_pinned() && r->pin_count() == 0),
2382            "Region " SIZE_FORMAT " pinning status is inconsistent", i);
2383   }
2384 }
2385 #endif
2386 
2387 ConcurrentGCTimer* ShenandoahHeap::gc_timer() const {
2388   return _gc_timer;
2389 }
2390 
2391 void ShenandoahHeap::prepare_concurrent_roots() {
2392   assert(SafepointSynchronize::is_at_safepoint(), "Must be at a safepoint");
2393   if (ShenandoahConcurrentRoots::should_do_concurrent_roots()) {
2394     set_concurrent_strong_root_in_progress(!collection_set()->is_empty());
2395     set_concurrent_weak_root_in_progress(true);
2396   }
2397 }
2398 
2399 void ShenandoahHeap::prepare_concurrent_unloading() {
2400   assert(SafepointSynchronize::is_at_safepoint(), "Must be at a safepoint");
2401   if (ShenandoahConcurrentRoots::should_do_concurrent_class_unloading()) {
2402     _unloader.prepare();
2403   }
2404 }
2405 
2406 void ShenandoahHeap::finish_concurrent_unloading() {
2407   assert(SafepointSynchronize::is_at_safepoint(), "Must be at a safepoint");
2408   if (ShenandoahConcurrentRoots::should_do_concurrent_class_unloading()) {
2409     _unloader.finish();
2410   }
2411 }
2412 
2413 #ifdef ASSERT
2414 void ShenandoahHeap::assert_gc_workers(uint nworkers) {
2415   assert(nworkers > 0 && nworkers <= max_workers(), "Sanity");
2416 
2417   if (ShenandoahSafepoint::is_at_shenandoah_safepoint()) {
2418     if (UseDynamicNumberOfGCThreads) {
2419       assert(nworkers <= ParallelGCThreads, "Cannot use more than it has");
2420     } else {
2421       // Use ParallelGCThreads inside safepoints
2422       assert(nworkers == ParallelGCThreads, "Use ParallelGCThreads within safepoints");
2423     }
2424   } else {
2425     if (UseDynamicNumberOfGCThreads) {
2426       assert(nworkers <= ConcGCThreads, "Cannot use more than it has");
2427     } else {
2428       // Use ConcGCThreads outside safepoints
2429       assert(nworkers == ConcGCThreads, "Use ConcGCThreads outside safepoints");
2430     }
2431   }
2432 }
2433 #endif
2434 
2435 ShenandoahVerifier* ShenandoahHeap::verifier() {
2436   guarantee(ShenandoahVerify, "Should be enabled");
2437   assert (_verifier != NULL, "sanity");
2438   return _verifier;
2439 }
2440 
2441 template<class T>
2442 class ShenandoahUpdateHeapRefsTask : public AbstractGangTask {
2443 private:
2444   T cl;
2445   ShenandoahHeap* _heap;
2446   ShenandoahRegionIterator* _regions;
2447   bool _concurrent;
2448 public:
2449   ShenandoahUpdateHeapRefsTask(ShenandoahRegionIterator* regions, bool concurrent) :
2450     AbstractGangTask("Concurrent Update References Task"),
2451     cl(T()),
2452     _heap(ShenandoahHeap::heap()),
2453     _regions(regions),
2454     _concurrent(concurrent) {
2455   }
2456 
2457   void work(uint worker_id) {
2458     if (_concurrent) {
2459       ShenandoahConcurrentWorkerSession worker_session(worker_id);
2460       ShenandoahSuspendibleThreadSetJoiner stsj(ShenandoahSuspendibleWorkers);
2461       do_work();
2462     } else {
2463       ShenandoahParallelWorkerSession worker_session(worker_id);
2464       do_work();
2465     }
2466   }
2467 
2468 private:
2469   void do_work() {
2470     ShenandoahHeapRegion* r = _regions->next();
2471     ShenandoahMarkingContext* const ctx = _heap->complete_marking_context();
2472     while (r != NULL) {
2473       HeapWord* update_watermark = r->get_update_watermark();
2474       assert (update_watermark >= r->bottom(), "sanity");
2475       if (r->is_active() && !r->is_cset()) {
2476         _heap->marked_object_oop_iterate(r, &cl, update_watermark);
2477       }
2478       if (ShenandoahPacing) {
2479         _heap->pacer()->report_updaterefs(pointer_delta(update_watermark, r->bottom()));
2480       }
2481       if (_heap->check_cancelled_gc_and_yield(_concurrent)) {
2482         return;
2483       }
2484       r = _regions->next();
2485     }
2486   }
2487 };
2488 
2489 void ShenandoahHeap::update_heap_references(bool concurrent) {
2490   ShenandoahUpdateHeapRefsTask<ShenandoahUpdateHeapRefsClosure> task(&_update_refs_iterator, concurrent);
2491   workers()->run_task(&task);
2492 }
2493 
2494 void ShenandoahHeap::op_init_updaterefs() {
2495   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "must be at safepoint");
2496 
2497   set_evacuation_in_progress(false);
2498 
2499   // Evacuation is over, no GCLABs are needed anymore. GCLABs are under URWM, so we need to
2500   // make them parsable for update code to work correctly. Plus, we can compute new sizes
2501   // for future GCLABs here.
2502   if (UseTLAB) {
2503     ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_update_refs_manage_gclabs);
2504     gclabs_retire(ResizeTLAB);
2505   }
2506 
2507   if (ShenandoahVerify) {
2508     if (!is_degenerated_gc_in_progress()) {
2509       verifier()->verify_roots_in_to_space_except(ShenandoahRootVerifier::ThreadRoots);
2510     }
2511     verifier()->verify_before_updaterefs();
2512   }
2513 
2514   set_update_refs_in_progress(true);
2515 
2516   _update_refs_iterator.reset();
2517 
2518   if (ShenandoahPacing) {
2519     pacer()->setup_for_updaterefs();
2520   }
2521 }
2522 
2523 class ShenandoahFinalUpdateRefsUpdateRegionStateClosure : public ShenandoahHeapRegionClosure {
2524 private:
2525   ShenandoahHeapLock* const _lock;
2526 
2527 public:
2528   ShenandoahFinalUpdateRefsUpdateRegionStateClosure() : _lock(ShenandoahHeap::heap()->lock()) {}
2529 
2530   void heap_region_do(ShenandoahHeapRegion* r) {
2531     // Drop unnecessary "pinned" state from regions that does not have CP marks
2532     // anymore, as this would allow trashing them.
2533 
2534     if (r->is_active()) {
2535       if (r->is_pinned()) {
2536         if (r->pin_count() == 0) {
2537           ShenandoahHeapLocker locker(_lock);
2538           r->make_unpinned();
2539         }
2540       } else {
2541         if (r->pin_count() > 0) {
2542           ShenandoahHeapLocker locker(_lock);
2543           r->make_pinned();
2544         }
2545       }
2546     }
2547   }
2548 
2549   bool is_thread_safe() { return true; }
2550 };
2551 
2552 void ShenandoahHeap::op_final_updaterefs() {
2553   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "must be at safepoint");
2554 
2555   finish_concurrent_unloading();
2556 
2557   // Check if there is left-over work, and finish it
2558   if (_update_refs_iterator.has_next()) {
2559     ShenandoahGCPhase phase(ShenandoahPhaseTimings::final_update_refs_finish_work);
2560 
2561     // Finish updating references where we left off.
2562     clear_cancelled_gc();
2563     update_heap_references(false);
2564   }
2565 
2566   // Clear cancelled GC, if set. On cancellation path, the block before would handle
2567   // everything. On degenerated paths, cancelled gc would not be set anyway.
2568   if (cancelled_gc()) {
2569     clear_cancelled_gc();
2570   }
2571   assert(!cancelled_gc(), "Should have been done right before");
2572 
2573   if (ShenandoahVerify && !is_degenerated_gc_in_progress()) {
2574     verifier()->verify_roots_in_to_space_except(ShenandoahRootVerifier::ThreadRoots);
2575   }
2576 
2577   if (is_degenerated_gc_in_progress()) {
2578     concurrent_mark()->update_roots(ShenandoahPhaseTimings::degen_gc_update_roots);
2579   } else {
2580     concurrent_mark()->update_thread_roots(ShenandoahPhaseTimings::final_update_refs_roots);
2581   }
2582 
2583   // Has to be done before cset is clear
2584   if (ShenandoahVerify) {
2585     verifier()->verify_roots_in_to_space();
2586   }
2587 
2588   {
2589     ShenandoahGCPhase phase(ShenandoahPhaseTimings::final_update_refs_update_region_states);
2590     ShenandoahFinalUpdateRefsUpdateRegionStateClosure cl;
2591     parallel_heap_region_iterate(&cl);
2592 
2593     assert_pinned_region_status();
2594   }
2595 
2596   {
2597     ShenandoahGCPhase phase(ShenandoahPhaseTimings::final_update_refs_trash_cset);
2598     trash_cset_regions();
2599   }
2600 
2601   set_has_forwarded_objects(false);
2602   set_update_refs_in_progress(false);
2603 
2604   if (ShenandoahVerify) {
2605     verifier()->verify_after_updaterefs();
2606   }
2607 
2608   if (VerifyAfterGC) {
2609     Universe::verify();
2610   }
2611 
2612   {
2613     ShenandoahGCPhase phase(ShenandoahPhaseTimings::final_update_refs_rebuild_freeset);
2614     ShenandoahHeapLocker locker(lock());
2615     _free_set->rebuild();
2616   }
2617 }
2618 
2619 void ShenandoahHeap::print_extended_on(outputStream *st) const {
2620   print_on(st);
2621   print_heap_regions_on(st);
2622 }
2623 
2624 bool ShenandoahHeap::is_bitmap_slice_committed(ShenandoahHeapRegion* r, bool skip_self) {
2625   size_t slice = r->index() / _bitmap_regions_per_slice;
2626 
2627   size_t regions_from = _bitmap_regions_per_slice * slice;
2628   size_t regions_to   = MIN2(num_regions(), _bitmap_regions_per_slice * (slice + 1));
2629   for (size_t g = regions_from; g < regions_to; g++) {
2630     assert (g / _bitmap_regions_per_slice == slice, "same slice");
2631     if (skip_self && g == r->index()) continue;
2632     if (get_region(g)->is_committed()) {
2633       return true;
2634     }
2635   }
2636   return false;
2637 }
2638 
2639 bool ShenandoahHeap::commit_bitmap_slice(ShenandoahHeapRegion* r) {
2640   shenandoah_assert_heaplocked();
2641 
2642   // Bitmaps in special regions do not need commits
2643   if (_bitmap_region_special) {
2644     return true;
2645   }
2646 
2647   if (is_bitmap_slice_committed(r, true)) {
2648     // Some other region from the group is already committed, meaning the bitmap
2649     // slice is already committed, we exit right away.
2650     return true;
2651   }
2652 
2653   // Commit the bitmap slice:
2654   size_t slice = r->index() / _bitmap_regions_per_slice;
2655   size_t off = _bitmap_bytes_per_slice * slice;
2656   size_t len = _bitmap_bytes_per_slice;
2657   char* start = (char*) _bitmap_region.start() + off;
2658 
2659   if (!os::commit_memory(start, len, false)) {
2660     return false;
2661   }
2662 
2663   if (AlwaysPreTouch) {
2664     os::pretouch_memory(start, start + len, _pretouch_bitmap_page_size);
2665   }
2666 
2667   return true;
2668 }
2669 
2670 bool ShenandoahHeap::uncommit_bitmap_slice(ShenandoahHeapRegion *r) {
2671   shenandoah_assert_heaplocked();
2672 
2673   // Bitmaps in special regions do not need uncommits
2674   if (_bitmap_region_special) {
2675     return true;
2676   }
2677 
2678   if (is_bitmap_slice_committed(r, true)) {
2679     // Some other region from the group is still committed, meaning the bitmap
2680     // slice is should stay committed, exit right away.
2681     return true;
2682   }
2683 
2684   // Uncommit the bitmap slice:
2685   size_t slice = r->index() / _bitmap_regions_per_slice;
2686   size_t off = _bitmap_bytes_per_slice * slice;
2687   size_t len = _bitmap_bytes_per_slice;
2688   if (!os::uncommit_memory((char*)_bitmap_region.start() + off, len)) {
2689     return false;
2690   }
2691   return true;
2692 }
2693 
2694 void ShenandoahHeap::safepoint_synchronize_begin() {
2695   if (ShenandoahSuspendibleWorkers || UseStringDeduplication) {
2696     SuspendibleThreadSet::synchronize();
2697   }
2698 }
2699 
2700 void ShenandoahHeap::safepoint_synchronize_end() {
2701   if (ShenandoahSuspendibleWorkers || UseStringDeduplication) {
2702     SuspendibleThreadSet::desynchronize();
2703   }
2704 }
2705 
2706 void ShenandoahHeap::vmop_entry_init_mark() {
2707   TraceCollectorStats tcs(monitoring_support()->stw_collection_counters());
2708   ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::init_mark_gross);
2709 
2710   try_inject_alloc_failure();
2711   VM_ShenandoahInitMark op;
2712   VMThread::execute(&op); // jump to entry_init_mark() under safepoint
2713 }
2714 
2715 void ShenandoahHeap::vmop_entry_final_mark() {
2716   TraceCollectorStats tcs(monitoring_support()->stw_collection_counters());
2717   ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_mark_gross);
2718 
2719   try_inject_alloc_failure();
2720   VM_ShenandoahFinalMarkStartEvac op;
2721   VMThread::execute(&op); // jump to entry_final_mark under safepoint
2722 }
2723 
2724 void ShenandoahHeap::vmop_entry_init_updaterefs() {
2725   TraceCollectorStats tcs(monitoring_support()->stw_collection_counters());
2726   ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::init_update_refs_gross);
2727 
2728   try_inject_alloc_failure();
2729   VM_ShenandoahInitUpdateRefs op;
2730   VMThread::execute(&op);
2731 }
2732 
2733 void ShenandoahHeap::vmop_entry_final_updaterefs() {
2734   TraceCollectorStats tcs(monitoring_support()->stw_collection_counters());
2735   ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_update_refs_gross);
2736 
2737   try_inject_alloc_failure();
2738   VM_ShenandoahFinalUpdateRefs op;
2739   VMThread::execute(&op);
2740 }
2741 
2742 void ShenandoahHeap::vmop_entry_full(GCCause::Cause cause) {
2743   TraceCollectorStats tcs(monitoring_support()->full_stw_collection_counters());
2744   ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::full_gc_gross);
2745 
2746   try_inject_alloc_failure();
2747   VM_ShenandoahFullGC op(cause);
2748   VMThread::execute(&op);
2749 }
2750 
2751 void ShenandoahHeap::vmop_degenerated(ShenandoahDegenPoint point) {
2752   TraceCollectorStats tcs(monitoring_support()->full_stw_collection_counters());
2753   ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::degen_gc_gross);
2754 
2755   VM_ShenandoahDegeneratedGC degenerated_gc((int)point);
2756   VMThread::execute(&degenerated_gc);
2757 }
2758 
2759 void ShenandoahHeap::entry_init_mark() {
2760   const char* msg = init_mark_event_message();
2761   ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::init_mark);
2762   EventMark em("%s", msg);
2763 
2764   ShenandoahWorkerScope scope(workers(),
2765                               ShenandoahWorkerPolicy::calc_workers_for_init_marking(),
2766                               "init marking");
2767 
2768   op_init_mark();
2769 }
2770 
2771 void ShenandoahHeap::entry_final_mark() {
2772   const char* msg = final_mark_event_message();
2773   ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::final_mark);
2774   EventMark em("%s", msg);
2775 
2776   ShenandoahWorkerScope scope(workers(),
2777                               ShenandoahWorkerPolicy::calc_workers_for_final_marking(),
2778                               "final marking");
2779 
2780   op_final_mark();
2781 }
2782 
2783 void ShenandoahHeap::entry_init_updaterefs() {
2784   static const char* msg = "Pause Init Update Refs";
2785   ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::init_update_refs);
2786   EventMark em("%s", msg);
2787 
2788   // No workers used in this phase, no setup required
2789 
2790   op_init_updaterefs();
2791 }
2792 
2793 void ShenandoahHeap::entry_final_updaterefs() {
2794   static const char* msg = "Pause Final Update Refs";
2795   ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::final_update_refs);
2796   EventMark em("%s", msg);
2797 
2798   ShenandoahWorkerScope scope(workers(),
2799                               ShenandoahWorkerPolicy::calc_workers_for_final_update_ref(),
2800                               "final reference update");
2801 
2802   op_final_updaterefs();
2803 }
2804 
2805 void ShenandoahHeap::entry_full(GCCause::Cause cause) {
2806   static const char* msg = "Pause Full";
2807   ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::full_gc, true /* log_heap_usage */);
2808   EventMark em("%s", msg);
2809 
2810   ShenandoahWorkerScope scope(workers(),
2811                               ShenandoahWorkerPolicy::calc_workers_for_fullgc(),
2812                               "full gc");
2813 
2814   op_full(cause);
2815 }
2816 
2817 void ShenandoahHeap::entry_degenerated(int point) {
2818   ShenandoahDegenPoint dpoint = (ShenandoahDegenPoint)point;
2819   const char* msg = degen_event_message(dpoint);
2820   ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::degen_gc, true /* log_heap_usage */);
2821   EventMark em("%s", msg);
2822 
2823   ShenandoahWorkerScope scope(workers(),
2824                               ShenandoahWorkerPolicy::calc_workers_for_stw_degenerated(),
2825                               "stw degenerated gc");
2826 
2827   set_degenerated_gc_in_progress(true);
2828   op_degenerated(dpoint);
2829   set_degenerated_gc_in_progress(false);
2830 }
2831 
2832 void ShenandoahHeap::entry_mark() {
2833   TraceCollectorStats tcs(monitoring_support()->concurrent_collection_counters());
2834 
2835   const char* msg = conc_mark_event_message();
2836   ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_mark);
2837   EventMark em("%s", msg);
2838 
2839   ShenandoahWorkerScope scope(workers(),
2840                               ShenandoahWorkerPolicy::calc_workers_for_conc_marking(),
2841                               "concurrent marking");
2842 
2843   try_inject_alloc_failure();
2844   op_mark();
2845 }
2846 
2847 void ShenandoahHeap::entry_evac() {
2848   TraceCollectorStats tcs(monitoring_support()->concurrent_collection_counters());
2849 
2850   static const char* msg = "Concurrent evacuation";
2851   ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_evac);
2852   EventMark em("%s", msg);
2853 
2854   ShenandoahWorkerScope scope(workers(),
2855                               ShenandoahWorkerPolicy::calc_workers_for_conc_evac(),
2856                               "concurrent evacuation");
2857 
2858   try_inject_alloc_failure();
2859   op_conc_evac();
2860 }
2861 
2862 void ShenandoahHeap::entry_updaterefs() {
2863   static const char* msg = "Concurrent update references";
2864   ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_update_refs);
2865   EventMark em("%s", msg);
2866 
2867   ShenandoahWorkerScope scope(workers(),
2868                               ShenandoahWorkerPolicy::calc_workers_for_conc_update_ref(),
2869                               "concurrent reference update");
2870 
2871   try_inject_alloc_failure();
2872   op_updaterefs();
2873 }
2874 
2875 void ShenandoahHeap::entry_weak_roots() {
2876   static const char* msg = "Concurrent weak roots";
2877   ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_weak_roots);
2878   EventMark em("%s", msg);
2879 
2880   ShenandoahWorkerScope scope(workers(),
2881                               ShenandoahWorkerPolicy::calc_workers_for_conc_root_processing(),
2882                               "concurrent weak root");
2883 
2884   try_inject_alloc_failure();
2885   op_weak_roots();
2886 }
2887 
2888 void ShenandoahHeap::entry_class_unloading() {
2889   static const char* msg = "Concurrent class unloading";
2890   ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_class_unload);
2891   EventMark em("%s", msg);
2892 
2893   ShenandoahWorkerScope scope(workers(),
2894                               ShenandoahWorkerPolicy::calc_workers_for_conc_root_processing(),
2895                               "concurrent class unloading");
2896 
2897   try_inject_alloc_failure();
2898   op_class_unloading();
2899 }
2900 
2901 void ShenandoahHeap::entry_strong_roots() {
2902   static const char* msg = "Concurrent strong roots";
2903   ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_strong_roots);
2904   EventMark em("%s", msg);
2905 
2906   ShenandoahGCWorkerPhase worker_phase(ShenandoahPhaseTimings::conc_strong_roots);
2907 
2908   ShenandoahWorkerScope scope(workers(),
2909                               ShenandoahWorkerPolicy::calc_workers_for_conc_root_processing(),
2910                               "concurrent strong root");
2911 
2912   try_inject_alloc_failure();
2913   op_strong_roots();
2914 }
2915 
2916 void ShenandoahHeap::entry_cleanup_early() {
2917   static const char* msg = "Concurrent cleanup";
2918   ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_cleanup_early, true /* log_heap_usage */);
2919   EventMark em("%s", msg);
2920 
2921   // This phase does not use workers, no need for setup
2922 
2923   try_inject_alloc_failure();
2924   op_cleanup_early();
2925 }
2926 
2927 void ShenandoahHeap::entry_cleanup_complete() {
2928   static const char* msg = "Concurrent cleanup";
2929   ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_cleanup_complete, true /* log_heap_usage */);
2930   EventMark em("%s", msg);
2931 
2932   // This phase does not use workers, no need for setup
2933 
2934   try_inject_alloc_failure();
2935   op_cleanup_complete();
2936 }
2937 
2938 void ShenandoahHeap::entry_reset() {
2939   static const char* msg = "Concurrent reset";
2940   ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_reset);
2941   EventMark em("%s", msg);
2942 
2943   ShenandoahWorkerScope scope(workers(),
2944                               ShenandoahWorkerPolicy::calc_workers_for_conc_reset(),
2945                               "concurrent reset");
2946 
2947   try_inject_alloc_failure();
2948   op_reset();
2949 }
2950 
2951 void ShenandoahHeap::entry_preclean() {
2952   if (ShenandoahPreclean && process_references()) {
2953     static const char* msg = "Concurrent precleaning";
2954     ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_preclean);
2955     EventMark em("%s", msg);
2956 
2957     ShenandoahWorkerScope scope(workers(),
2958                                 ShenandoahWorkerPolicy::calc_workers_for_conc_preclean(),
2959                                 "concurrent preclean",
2960                                 /* check_workers = */ false);
2961 
2962     try_inject_alloc_failure();
2963     op_preclean();
2964   }
2965 }
2966 
2967 void ShenandoahHeap::entry_uncommit(double shrink_before) {
2968   static const char *msg = "Concurrent uncommit";
2969   ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_uncommit, true /* log_heap_usage */);
2970   EventMark em("%s", msg);
2971 
2972   op_uncommit(shrink_before);
2973 }
2974 
2975 void ShenandoahHeap::try_inject_alloc_failure() {
2976   if (ShenandoahAllocFailureALot && !cancelled_gc() && ((os::random() % 1000) > 950)) {
2977     _inject_alloc_failure.set();
2978     os::naked_short_sleep(1);
2979     if (cancelled_gc()) {
2980       log_info(gc)("Allocation failure was successfully injected");
2981     }
2982   }
2983 }
2984 
2985 bool ShenandoahHeap::should_inject_alloc_failure() {
2986   return _inject_alloc_failure.is_set() && _inject_alloc_failure.try_unset();
2987 }
2988 
2989 void ShenandoahHeap::initialize_serviceability() {
2990   _memory_pool = new ShenandoahMemoryPool(this);
2991   _cycle_memory_manager.add_pool(_memory_pool);
2992   _stw_memory_manager.add_pool(_memory_pool);
2993 }
2994 
2995 GrowableArray<GCMemoryManager*> ShenandoahHeap::memory_managers() {
2996   GrowableArray<GCMemoryManager*> memory_managers(2);
2997   memory_managers.append(&_cycle_memory_manager);
2998   memory_managers.append(&_stw_memory_manager);
2999   return memory_managers;
3000 }
3001 
3002 GrowableArray<MemoryPool*> ShenandoahHeap::memory_pools() {
3003   GrowableArray<MemoryPool*> memory_pools(1);
3004   memory_pools.append(_memory_pool);
3005   return memory_pools;
3006 }
3007 
3008 MemoryUsage ShenandoahHeap::memory_usage() {
3009   return _memory_pool->get_memory_usage();
3010 }
3011 
3012 ShenandoahRegionIterator::ShenandoahRegionIterator() :
3013   _heap(ShenandoahHeap::heap()),
3014   _index(0) {}
3015 
3016 ShenandoahRegionIterator::ShenandoahRegionIterator(ShenandoahHeap* heap) :
3017   _heap(heap),
3018   _index(0) {}
3019 
3020 void ShenandoahRegionIterator::reset() {
3021   _index = 0;
3022 }
3023 
3024 bool ShenandoahRegionIterator::has_next() const {
3025   return _index < _heap->num_regions();
3026 }
3027 
3028 char ShenandoahHeap::gc_state() const {
3029   return _gc_state.raw_value();
3030 }
3031 
3032 void ShenandoahHeap::deduplicate_string(oop str) {
3033   assert(java_lang_String::is_instance(str), "invariant");
3034 
3035   if (ShenandoahStringDedup::is_enabled()) {
3036     ShenandoahStringDedup::deduplicate(str);
3037   }
3038 }
3039 
3040 const char* ShenandoahHeap::init_mark_event_message() const {
3041   assert(!has_forwarded_objects(), "Should not have forwarded objects here");
3042 
3043   bool proc_refs = process_references();
3044   bool unload_cls = unload_classes();
3045 
3046   if (proc_refs && unload_cls) {
3047     return "Pause Init Mark (process weakrefs) (unload classes)";
3048   } else if (proc_refs) {
3049     return "Pause Init Mark (process weakrefs)";
3050   } else if (unload_cls) {
3051     return "Pause Init Mark (unload classes)";
3052   } else {
3053     return "Pause Init Mark";
3054   }
3055 }
3056 
3057 const char* ShenandoahHeap::final_mark_event_message() const {
3058   assert(!has_forwarded_objects(), "Should not have forwarded objects here");
3059 
3060   bool proc_refs = process_references();
3061   bool unload_cls = unload_classes();
3062 
3063   if (proc_refs && unload_cls) {
3064     return "Pause Final Mark (process weakrefs) (unload classes)";
3065   } else if (proc_refs) {
3066     return "Pause Final Mark (process weakrefs)";
3067   } else if (unload_cls) {
3068     return "Pause Final Mark (unload classes)";
3069   } else {
3070     return "Pause Final Mark";
3071   }
3072 }
3073 
3074 const char* ShenandoahHeap::conc_mark_event_message() const {
3075   assert(!has_forwarded_objects(), "Should not have forwarded objects here");
3076 
3077   bool proc_refs = process_references();
3078   bool unload_cls = unload_classes();
3079 
3080   if (proc_refs && unload_cls) {
3081     return "Concurrent marking (process weakrefs) (unload classes)";
3082   } else if (proc_refs) {
3083     return "Concurrent marking (process weakrefs)";
3084   } else if (unload_cls) {
3085     return "Concurrent marking (unload classes)";
3086   } else {
3087     return "Concurrent marking";
3088   }
3089 }
3090 
3091 const char* ShenandoahHeap::degen_event_message(ShenandoahDegenPoint point) const {
3092   switch (point) {
3093     case _degenerated_unset:
3094       return "Pause Degenerated GC (<UNSET>)";
3095     case _degenerated_outside_cycle:
3096       return "Pause Degenerated GC (Outside of Cycle)";
3097     case _degenerated_mark:
3098       return "Pause Degenerated GC (Mark)";
3099     case _degenerated_evac:
3100       return "Pause Degenerated GC (Evacuation)";
3101     case _degenerated_updaterefs:
3102       return "Pause Degenerated GC (Update Refs)";
3103     default:
3104       ShouldNotReachHere();
3105       return "ERROR";
3106   }
3107 }
3108 
3109 ShenandoahLiveData* ShenandoahHeap::get_liveness_cache(uint worker_id) {
3110 #ifdef ASSERT
3111   assert(_liveness_cache != NULL, "sanity");
3112   assert(worker_id < _max_workers, "sanity");
3113   for (uint i = 0; i < num_regions(); i++) {
3114     assert(_liveness_cache[worker_id][i] == 0, "liveness cache should be empty");
3115   }
3116 #endif
3117   return _liveness_cache[worker_id];
3118 }
3119 
3120 void ShenandoahHeap::flush_liveness_cache(uint worker_id) {
3121   assert(worker_id < _max_workers, "sanity");
3122   assert(_liveness_cache != NULL, "sanity");
3123   ShenandoahLiveData* ld = _liveness_cache[worker_id];
3124   for (uint i = 0; i < num_regions(); i++) {
3125     ShenandoahLiveData live = ld[i];
3126     if (live > 0) {
3127       ShenandoahHeapRegion* r = get_region(i);
3128       r->increase_live_data_gc_words(live);
3129       ld[i] = 0;
3130     }
3131   }
3132 }