1 /*
   2  * Copyright (c) 2013, 2015, Red Hat, Inc. and/or its affiliates.
   3  *
   4  * This code is free software; you can redistribute it and/or modify it
   5  * under the terms of the GNU General Public License version 2 only, as
   6  * published by the Free Software Foundation.
   7  *
   8  * This code is distributed in the hope that it will be useful, but WITHOUT
   9  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  10  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  11  * version 2 for more details (a copy is included in the LICENSE file that
  12  * accompanied this code).
  13  *
  14  * You should have received a copy of the GNU General Public License version
  15  * 2 along with this work; if not, write to the Free Software Foundation,
  16  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  17  *
  18  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  19  * or visit www.oracle.com if you need additional information or have any
  20  * questions.
  21  *
  22  */
  23 
  24 #include "precompiled.hpp"
  25 #include "memory/allocation.hpp"
  26 
  27 #include "gc/shared/gcTimer.hpp"
  28 #include "gc/shared/gcTraceTime.inline.hpp"
  29 #include "gc/shared/parallelCleaning.hpp"
  30 
  31 #include "gc/shenandoah/brooksPointer.hpp"
  32 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
  33 #include "gc/shenandoah/shenandoahCollectionSet.hpp"
  34 #include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
  35 #include "gc/shenandoah/shenandoahConcurrentMark.hpp"
  36 #include "gc/shenandoah/shenandoahConcurrentMark.inline.hpp"
  37 #include "gc/shenandoah/shenandoahConcurrentThread.hpp"
  38 #include "gc/shenandoah/shenandoahFreeSet.hpp"
  39 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
  40 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
  41 #include "gc/shenandoah/shenandoahHeapRegionSet.hpp"
  42 #include "gc/shenandoah/shenandoahHumongous.hpp"
  43 #include "gc/shenandoah/shenandoahMarkCompact.hpp"
  44 #include "gc/shenandoah/shenandoahMonitoringSupport.hpp"
  45 #include "gc/shenandoah/shenandoahPartialGC.hpp"
  46 #include "gc/shenandoah/shenandoahRootProcessor.hpp"
  47 #include "gc/shenandoah/vm_operations_shenandoah.hpp"
  48 
  49 #include "runtime/vmThread.hpp"
  50 #include "services/mallocTracker.hpp"
  51 
  52 SCMUpdateRefsClosure::SCMUpdateRefsClosure() : _heap(ShenandoahHeap::heap()) {}
  53 
  54 #ifdef ASSERT
  55 template <class T>
  56 void AssertToSpaceClosure::do_oop_nv(T* p) {
  57   T o = oopDesc::load_heap_oop(p);
  58   if (! oopDesc::is_null(o)) {
  59     oop obj = oopDesc::decode_heap_oop_not_null(o);
  60     assert(oopDesc::unsafe_equals(obj, ShenandoahBarrierSet::resolve_oop_static_not_null(obj)),
  61            "need to-space object here obj: "PTR_FORMAT" , rb(obj): "PTR_FORMAT", p: "PTR_FORMAT,
  62            p2i(obj), p2i(ShenandoahBarrierSet::resolve_oop_static_not_null(obj)), p2i(p));
  63   }
  64 }
  65 
  66 void AssertToSpaceClosure::do_oop(narrowOop* p) { do_oop_nv(p); }
  67 void AssertToSpaceClosure::do_oop(oop* p)       { do_oop_nv(p); }
  68 #endif
  69 
  70 const char* ShenandoahHeap::name() const {
  71   return "Shenandoah";
  72 }
  73 
  74 void ShenandoahHeap::print_heap_locations(HeapWord* start, HeapWord* end) {
  75   HeapWord* cur = NULL;
  76   for (cur = start; cur < end; cur++) {
  77     tty->print_cr(PTR_FORMAT" : "PTR_FORMAT, p2i(cur), p2i(*((HeapWord**) cur)));
  78   }
  79 }
  80 
  81 class PrintHeapRegionsClosure : public
  82    ShenandoahHeapRegionClosure {
  83 private:
  84   outputStream* _st;
  85 public:
  86   PrintHeapRegionsClosure() : _st(tty) {}
  87   PrintHeapRegionsClosure(outputStream* st) : _st(st) {}
  88 
  89   bool doHeapRegion(ShenandoahHeapRegion* r) {
  90     r->print_on(_st);
  91     return false;
  92   }
  93 };
  94 
  95 class ShenandoahPretouchTask : public AbstractGangTask {
  96 private:
  97   ShenandoahHeapRegionSet* _regions;
  98   const size_t _bitmap_size;
  99   const size_t _page_size;
 100   char* _bitmap0_base;
 101   char* _bitmap1_base;
 102 public:
 103   ShenandoahPretouchTask(ShenandoahHeapRegionSet* regions,
 104                          char* bitmap0_base, char* bitmap1_base, size_t bitmap_size,
 105                          size_t page_size) :
 106     AbstractGangTask("Shenandoah PreTouch",
 107                      Universe::is_fully_initialized() ? GCId::current_raw() :
 108                                                         // During VM initialization there is
 109                                                         // no GC cycle that this task can be
 110                                                         // associated with.
 111                                                         GCId::undefined()),
 112     _bitmap0_base(bitmap0_base),
 113     _bitmap1_base(bitmap1_base),
 114     _regions(regions),
 115     _bitmap_size(bitmap_size),
 116     _page_size(page_size) {
 117     _regions->clear_current_index();
 118   };
 119 
 120   virtual void work(uint worker_id) {
 121     ShenandoahHeapRegion* r = _regions->claim_next();
 122     while (r != NULL) {
 123       log_trace(gc, heap)("Pretouch region " SIZE_FORMAT ": " PTR_FORMAT " -> " PTR_FORMAT,
 124                           r->region_number(), p2i(r->bottom()), p2i(r->end()));
 125       os::pretouch_memory(r->bottom(), r->end(), _page_size);
 126 
 127       size_t start = r->region_number()       * ShenandoahHeapRegion::RegionSizeBytes / CMBitMap::heap_map_factor();
 128       size_t end   = (r->region_number() + 1) * ShenandoahHeapRegion::RegionSizeBytes / CMBitMap::heap_map_factor();
 129       assert (end <= _bitmap_size, "end is sane: " SIZE_FORMAT " < " SIZE_FORMAT, end, _bitmap_size);
 130 
 131       log_trace(gc, heap)("Pretouch bitmap under region " SIZE_FORMAT ": " PTR_FORMAT " -> " PTR_FORMAT,
 132                           r->region_number(), p2i(_bitmap0_base + start), p2i(_bitmap0_base + end));
 133       os::pretouch_memory(_bitmap0_base + start, _bitmap0_base + end, _page_size);
 134 
 135       log_trace(gc, heap)("Pretouch bitmap under region " SIZE_FORMAT ": " PTR_FORMAT " -> " PTR_FORMAT,
 136                           r->region_number(), p2i(_bitmap1_base + start), p2i(_bitmap1_base + end));
 137       os::pretouch_memory(_bitmap1_base + start, _bitmap1_base + end, _page_size);
 138 
 139       r = _regions->claim_next();
 140     }
 141   }
 142 };
 143 
 144 jint ShenandoahHeap::initialize() {
 145   CollectedHeap::pre_initialize();
 146 
 147   BrooksPointer::initial_checks();
 148 
 149   size_t init_byte_size = collector_policy()->initial_heap_byte_size();
 150   size_t max_byte_size = collector_policy()->max_heap_byte_size();
 151 
 152   Universe::check_alignment(max_byte_size,
 153                             ShenandoahHeapRegion::RegionSizeBytes,
 154                             "shenandoah heap");
 155   Universe::check_alignment(init_byte_size,
 156                             ShenandoahHeapRegion::RegionSizeBytes,
 157                             "shenandoah heap");
 158 
 159   ReservedSpace heap_rs = Universe::reserve_heap(max_byte_size,
 160                                                  Arguments::conservative_max_heap_alignment());
 161   initialize_reserved_region((HeapWord*)heap_rs.base(), (HeapWord*) (heap_rs.base() + heap_rs.size()));
 162 
 163   set_barrier_set(new ShenandoahBarrierSet(this));
 164   ReservedSpace pgc_rs = heap_rs.first_part(max_byte_size);
 165   _storage.initialize(pgc_rs, init_byte_size);
 166 
 167   _num_regions = init_byte_size / ShenandoahHeapRegion::RegionSizeBytes;
 168   _max_regions = max_byte_size / ShenandoahHeapRegion::RegionSizeBytes;
 169   _initialSize = _num_regions * ShenandoahHeapRegion::RegionSizeBytes;
 170   size_t regionSizeWords = ShenandoahHeapRegion::RegionSizeBytes / HeapWordSize;
 171   assert(init_byte_size == _initialSize, "tautology");
 172   _ordered_regions = new ShenandoahHeapRegionSet(_max_regions);
 173   _collection_set = new ShenandoahCollectionSet(_max_regions);
 174   _free_regions = new ShenandoahFreeSet(_max_regions);
 175 
 176   // Initialize fast collection set test structure.
 177   _in_cset_fast_test_length = _max_regions;
 178   _in_cset_fast_test_base =
 179                    NEW_C_HEAP_ARRAY(bool, _in_cset_fast_test_length, mtGC);
 180   _in_cset_fast_test = _in_cset_fast_test_base -
 181                ((uintx) pgc_rs.base() >> ShenandoahHeapRegion::RegionSizeShift);
 182 
 183   _next_top_at_mark_starts_base =
 184                    NEW_C_HEAP_ARRAY(HeapWord*, _max_regions, mtGC);
 185   _next_top_at_mark_starts = _next_top_at_mark_starts_base -
 186                ((uintx) pgc_rs.base() >> ShenandoahHeapRegion::RegionSizeShift);
 187 
 188   _complete_top_at_mark_starts_base =
 189                    NEW_C_HEAP_ARRAY(HeapWord*, _max_regions, mtGC);
 190   _complete_top_at_mark_starts = _complete_top_at_mark_starts_base -
 191                ((uintx) pgc_rs.base() >> ShenandoahHeapRegion::RegionSizeShift);
 192 
 193   size_t i = 0;
 194   for (i = 0; i < _num_regions; i++) {
 195     _in_cset_fast_test_base[i] = false; // Not in cset
 196     HeapWord* bottom = (HeapWord*) pgc_rs.base() + regionSizeWords * i;
 197     _complete_top_at_mark_starts_base[i] = bottom;
 198     _next_top_at_mark_starts_base[i] = bottom;
 199   }
 200 
 201   {
 202     ShenandoahHeapLock lock(this);
 203     for (i = 0; i < _num_regions; i++) {
 204       ShenandoahHeapRegion* current = new ShenandoahHeapRegion(this, (HeapWord*) pgc_rs.base() +
 205                                                                regionSizeWords * i, regionSizeWords, i);
 206       _free_regions->add_region(current);
 207       _ordered_regions->add_region(current);
 208     }
 209   }
 210   assert(((size_t) _ordered_regions->active_regions()) == _num_regions, "");
 211   _first_region = _ordered_regions->get(0);
 212   _first_region_bottom = _first_region->bottom();
 213   assert((((size_t) _first_region_bottom) &
 214           (ShenandoahHeapRegion::RegionSizeBytes - 1)) == 0,
 215          "misaligned heap: "PTR_FORMAT, p2i(_first_region_bottom));
 216 
 217   _numAllocs = 0;
 218 
 219   if (log_is_enabled(Trace, gc, region)) {
 220     ResourceMark rm;
 221     outputStream* out = Log(gc, region)::trace_stream();
 222     log_trace(gc, region)("All Regions");
 223     _ordered_regions->print(out);
 224     log_trace(gc, region)("Free Regions");
 225     _free_regions->print(out);
 226   }
 227 
 228   // The call below uses stuff (the SATB* things) that are in G1, but probably
 229   // belong into a shared location.
 230   JavaThread::satb_mark_queue_set().initialize(SATB_Q_CBL_mon,
 231                                                SATB_Q_FL_lock,
 232                                                20 /*G1SATBProcessCompletedThreshold */,
 233                                                Shared_SATB_Q_lock);
 234 
 235   // Reserve space for prev and next bitmap.
 236   _bitmap_size = CMBitMap::compute_size(heap_rs.size());
 237   _heap_region = MemRegion((HeapWord*) heap_rs.base(), heap_rs.size() / HeapWordSize);
 238 
 239   size_t page_size = UseLargePages ? (size_t)os::large_page_size() : (size_t)os::vm_page_size();
 240 
 241   ReservedSpace bitmap0(_bitmap_size, page_size);
 242   os::commit_memory_or_exit(bitmap0.base(), bitmap0.size(), false, "couldn't allocate mark bitmap");
 243   MemTracker::record_virtual_memory_type(bitmap0.base(), mtGC);
 244   MemRegion bitmap_region0 = MemRegion((HeapWord*) bitmap0.base(), bitmap0.size() / HeapWordSize);
 245 
 246   ReservedSpace bitmap1(_bitmap_size, page_size);
 247   os::commit_memory_or_exit(bitmap1.base(), bitmap1.size(), false, "couldn't allocate mark bitmap");
 248   MemTracker::record_virtual_memory_type(bitmap1.base(), mtGC);
 249   MemRegion bitmap_region1 = MemRegion((HeapWord*) bitmap1.base(), bitmap1.size() / HeapWordSize);
 250 
 251   if (ShenandoahVerify || (UseShenandoahMatrix && VerifyShenandoahMatrix)) {
 252     ReservedSpace verify_bitmap(_bitmap_size, page_size);
 253     os::commit_memory_or_exit(verify_bitmap.base(), verify_bitmap.size(), false,
 254                               "couldn't allocate verification bitmap");
 255     MemTracker::record_virtual_memory_type(verify_bitmap.base(), mtGC);
 256     MemRegion verify_bitmap_region = MemRegion((HeapWord *) verify_bitmap.base(), verify_bitmap.size() / HeapWordSize);
 257     _verification_bit_map.initialize(_heap_region, verify_bitmap_region);
 258   }
 259 
 260   if (ShenandoahAlwaysPreTouch) {
 261     assert (!AlwaysPreTouch, "Should have been overridden");
 262 
 263     // For NUMA, it is important to pre-touch the storage under bitmaps with worker threads,
 264     // before initialize() below zeroes it with initializing thread. For any given region,
 265     // we touch the region and the corresponding bitmaps from the same thread.
 266 
 267     log_info(gc, heap)("Parallel pretouch " SIZE_FORMAT " regions with " SIZE_FORMAT " byte pages",
 268                        _ordered_regions->count(), page_size);
 269     ShenandoahPretouchTask cl(_ordered_regions, bitmap0.base(), bitmap1.base(), _bitmap_size, page_size);
 270     _workers->run_task(&cl);
 271   }
 272 
 273   _mark_bit_map0.initialize(_heap_region, bitmap_region0);
 274   _complete_mark_bit_map = &_mark_bit_map0;
 275 
 276   _mark_bit_map1.initialize(_heap_region, bitmap_region1);
 277   _next_mark_bit_map = &_mark_bit_map1;
 278 
 279   _connection_matrix = new ShenandoahConnectionMatrix(_max_regions);
 280   _partial_gc = new ShenandoahPartialGC(this, _max_regions);
 281 
 282   _monitoring_support = new ShenandoahMonitoringSupport(this);
 283 
 284   _concurrent_gc_thread = new ShenandoahConcurrentThread();
 285 
 286   ShenandoahMarkCompact::initialize();
 287 
 288   return JNI_OK;
 289 }
 290 
 291 ShenandoahHeap::ShenandoahHeap(ShenandoahCollectorPolicy* policy) :
 292   CollectedHeap(),
 293   _shenandoah_policy(policy),
 294   _concurrent_mark_in_progress(0),
 295   _evacuation_in_progress(0),
 296   _full_gc_in_progress(false),
 297   _free_regions(NULL),
 298   _collection_set(NULL),
 299   _bytes_allocated_since_cm(0),
 300   _bytes_allocated_during_cm(0),
 301   _max_allocated_gc(0),
 302   _allocated_last_gc(0),
 303   _used_start_gc(0),
 304   _max_workers(MAX2(ConcGCThreads, ParallelGCThreads)),
 305   _ref_processor(NULL),
 306   _in_cset_fast_test(NULL),
 307   _in_cset_fast_test_base(NULL),
 308   _next_top_at_mark_starts(NULL),
 309   _next_top_at_mark_starts_base(NULL),
 310   _complete_top_at_mark_starts(NULL),
 311   _complete_top_at_mark_starts_base(NULL),
 312   _mark_bit_map0(),
 313   _mark_bit_map1(),
 314   _connection_matrix(NULL),
 315   _cancelled_concgc(false),
 316   _need_update_refs(false),
 317   _need_reset_bitmaps(false),
 318   _heap_lock(0),
 319 #ifdef ASSERT
 320   _heap_lock_owner(NULL),
 321 #endif
 322   _gc_timer(new (ResourceObj::C_HEAP, mtGC) ConcurrentGCTimer())
 323 
 324 {
 325   log_info(gc, init)("Parallel GC threads: "UINT32_FORMAT, ParallelGCThreads);
 326   log_info(gc, init)("Concurrent GC threads: "UINT32_FORMAT, ConcGCThreads);
 327   log_info(gc, init)("Parallel reference processing enabled: %s", BOOL_TO_STR(ParallelRefProcEnabled));
 328 
 329   _scm = new ShenandoahConcurrentMark();
 330   _used = 0;
 331 
 332   _max_workers = MAX2(_max_workers, 1U);
 333   _workers = new ShenandoahWorkGang("Shenandoah GC Threads", _max_workers,
 334                             /* are_GC_task_threads */true,
 335                             /* are_ConcurrentGC_threads */false);
 336   if (_workers == NULL) {
 337     vm_exit_during_initialization("Failed necessary allocation.");
 338   } else {
 339     _workers->initialize_workers();
 340   }
 341 }
 342 
 343 class ResetNextBitmapTask : public AbstractGangTask {
 344 private:
 345   ShenandoahHeapRegionSet* _regions;
 346 
 347 public:
 348   ResetNextBitmapTask(ShenandoahHeapRegionSet* regions) :
 349     AbstractGangTask("Parallel Reset Bitmap Task"),
 350     _regions(regions) {
 351     _regions->clear_current_index();
 352   }
 353 
 354   void work(uint worker_id) {
 355     ShenandoahHeapRegion* region = _regions->claim_next();
 356     ShenandoahHeap* heap = ShenandoahHeap::heap();
 357     while (region != NULL) {
 358       HeapWord* bottom = region->bottom();
 359       HeapWord* top = heap->next_top_at_mark_start(region->bottom());
 360       if (top > bottom) {
 361         heap->next_mark_bit_map()->clear_range_large(MemRegion(bottom, top));
 362       }
 363       region = _regions->claim_next();
 364     }
 365   }
 366 };
 367 
 368 void ShenandoahHeap::reset_next_mark_bitmap(WorkGang* workers) {
 369   ResetNextBitmapTask task = ResetNextBitmapTask(_ordered_regions);
 370   workers->run_task(&task);
 371 }
 372 
 373 class ResetCompleteBitmapTask : public AbstractGangTask {
 374 private:
 375   ShenandoahHeapRegionSet* _regions;
 376 
 377 public:
 378   ResetCompleteBitmapTask(ShenandoahHeapRegionSet* regions) :
 379     AbstractGangTask("Parallel Reset Bitmap Task"),
 380     _regions(regions) {
 381     _regions->clear_current_index();
 382   }
 383 
 384   void work(uint worker_id) {
 385     ShenandoahHeapRegion* region = _regions->claim_next();
 386     ShenandoahHeap* heap = ShenandoahHeap::heap();
 387     while (region != NULL) {
 388       HeapWord* bottom = region->bottom();
 389       HeapWord* top = heap->complete_top_at_mark_start(region->bottom());
 390       if (top > bottom) {
 391         heap->complete_mark_bit_map()->clear_range_large(MemRegion(bottom, top));
 392       }
 393       region = _regions->claim_next();
 394     }
 395   }
 396 };
 397 
 398 void ShenandoahHeap::reset_complete_mark_bitmap(WorkGang* workers) {
 399   ResetCompleteBitmapTask task = ResetCompleteBitmapTask(_ordered_regions);
 400   workers->run_task(&task);
 401 }
 402 
 403 bool ShenandoahHeap::is_next_bitmap_clear() {
 404   HeapWord* start = _ordered_regions->bottom();
 405   HeapWord* end = _ordered_regions->end();
 406   return _next_mark_bit_map->getNextMarkedWordAddress(start, end) == end;
 407 }
 408 
 409 bool ShenandoahHeap::is_complete_bitmap_clear_range(HeapWord* start, HeapWord* end) {
 410   return _complete_mark_bit_map->getNextMarkedWordAddress(start, end) == end;
 411 }
 412 
 413 void ShenandoahHeap::print_on(outputStream* st) const {
 414   st->print("Shenandoah Heap");
 415   st->print(" total = " SIZE_FORMAT " K, used " SIZE_FORMAT " K ", capacity()/ K, used() /K);
 416   st->print(" [" PTR_FORMAT ", " PTR_FORMAT ") ",
 417             p2i(reserved_region().start()),
 418             p2i(reserved_region().end()));
 419   st->print("Region size = " SIZE_FORMAT "K ", ShenandoahHeapRegion::RegionSizeBytes / K);
 420   if (_concurrent_mark_in_progress) {
 421     st->print("marking ");
 422   }
 423   if (_evacuation_in_progress) {
 424     st->print("evacuating ");
 425   }
 426   if (cancelled_concgc()) {
 427     st->print("cancelled ");
 428   }
 429   st->print("\n");
 430 
 431   // Adapted from VirtualSpace::print_on(), which is non-PRODUCT only
 432   st->print   ("Virtual space:");
 433   if (_storage.special()) st->print(" (pinned in memory)");
 434   st->cr();
 435   st->print_cr(" - committed: " SIZE_FORMAT, _storage.committed_size());
 436   st->print_cr(" - reserved:  " SIZE_FORMAT, _storage.reserved_size());
 437   st->print_cr(" - [low, high]:     [" INTPTR_FORMAT ", " INTPTR_FORMAT "]",  p2i(_storage.low()), p2i(_storage.high()));
 438   st->print_cr(" - [low_b, high_b]: [" INTPTR_FORMAT ", " INTPTR_FORMAT "]",  p2i(_storage.low_boundary()), p2i(_storage.high_boundary()));
 439 
 440   if (Verbose) {
 441     print_heap_regions(st);
 442   }
 443 }
 444 
 445 class InitGCLABClosure : public ThreadClosure {
 446 public:
 447   void do_thread(Thread* thread) {
 448     thread->gclab().initialize(true);
 449   }
 450 };
 451 
 452 void ShenandoahHeap::post_initialize() {
 453   if (UseTLAB) {
 454     // This is a very tricky point in VM lifetime. We cannot easily call Threads::threads_do
 455     // here, because some system threads (VMThread, WatcherThread, etc) are not yet available.
 456     // Their initialization should be handled separately. Is we miss some threads here,
 457     // then any other TLAB-related activity would fail with asserts.
 458 
 459     InitGCLABClosure init_gclabs;
 460     {
 461       MutexLocker ml(Threads_lock);
 462       for (JavaThread *thread = Threads::first(); thread != NULL; thread = thread->next()) {
 463         init_gclabs.do_thread(thread);
 464       }
 465     }
 466     gc_threads_do(&init_gclabs);
 467 
 468     // gclab can not be initialized early during VM startup, as it can not determinate its max_size.
 469     // Now, we will let WorkGang to initialize gclab when new worker is created.
 470     _workers->set_initialize_gclab();
 471   }
 472 
 473   _scm->initialize(_max_workers);
 474 
 475   ref_processing_init();
 476 }
 477 
 478 class CalculateUsedRegionClosure : public ShenandoahHeapRegionClosure {
 479   size_t sum;
 480 public:
 481 
 482   CalculateUsedRegionClosure() {
 483     sum = 0;
 484   }
 485 
 486   bool doHeapRegion(ShenandoahHeapRegion* r) {
 487     sum = sum + r->used();
 488     return false;
 489   }
 490 
 491   size_t getResult() { return sum;}
 492 };
 493 
 494 size_t ShenandoahHeap::calculateUsed() {
 495   CalculateUsedRegionClosure cl;
 496   heap_region_iterate(&cl);
 497   return cl.getResult();
 498 }
 499 
 500 void ShenandoahHeap::verify_heap_size_consistency() {
 501 
 502   assert(calculateUsed() == used(),
 503          "heap used size must be consistent heap-used: "SIZE_FORMAT" regions-used: "SIZE_FORMAT, used(), calculateUsed());
 504 }
 505 
 506 size_t ShenandoahHeap::used() const {
 507   OrderAccess::acquire();
 508   return _used;
 509 }
 510 
 511 void ShenandoahHeap::increase_used(size_t bytes) {
 512   assert_heaplock_or_safepoint();
 513   _used += bytes;
 514 }
 515 
 516 void ShenandoahHeap::set_used(size_t bytes) {
 517   assert_heaplock_or_safepoint();
 518   _used = bytes;
 519 }
 520 
 521 void ShenandoahHeap::decrease_used(size_t bytes) {
 522   assert_heaplock_or_safepoint();
 523   assert(_used >= bytes, "never decrease heap size by more than we've left");
 524   _used -= bytes;
 525 }
 526 
 527 size_t ShenandoahHeap::capacity() const {
 528   return _num_regions * ShenandoahHeapRegion::RegionSizeBytes;
 529 }
 530 
 531 bool ShenandoahHeap::is_maximal_no_gc() const {
 532   Unimplemented();
 533   return true;
 534 }
 535 
 536 size_t ShenandoahHeap::max_capacity() const {
 537   return _max_regions * ShenandoahHeapRegion::RegionSizeBytes;
 538 }
 539 
 540 size_t ShenandoahHeap::min_capacity() const {
 541   return _initialSize;
 542 }
 543 
 544 VirtualSpace* ShenandoahHeap::storage() const {
 545   return (VirtualSpace*) &_storage;
 546 }
 547 
 548 bool ShenandoahHeap::is_in(const void* p) const {
 549   HeapWord* first_region_bottom = _first_region->bottom();
 550   HeapWord* last_region_end = first_region_bottom + (ShenandoahHeapRegion::RegionSizeBytes / HeapWordSize) * _num_regions;
 551   return p >= _first_region_bottom && p < last_region_end;
 552 }
 553 
 554 bool ShenandoahHeap::is_scavengable(const void* p) {
 555   return true;
 556 }
 557 
 558 HeapWord* ShenandoahHeap::allocate_from_gclab_slow(Thread* thread, size_t size) {
 559   // Retain tlab and allocate object in shared space if
 560   // the amount free in the tlab is too large to discard.
 561   if (thread->gclab().free() > thread->gclab().refill_waste_limit()) {
 562     thread->gclab().record_slow_allocation(size);
 563     return NULL;
 564   }
 565 
 566   // Discard gclab and allocate a new one.
 567   // To minimize fragmentation, the last GCLAB may be smaller than the rest.
 568   size_t new_gclab_size = thread->gclab().compute_size(size);
 569 
 570   thread->gclab().clear_before_allocation();
 571 
 572   if (new_gclab_size == 0) {
 573     return NULL;
 574   }
 575 
 576   // Allocate a new GCLAB...
 577   HeapWord* obj = allocate_new_gclab(new_gclab_size);
 578   if (obj == NULL) {
 579     return NULL;
 580   }
 581 
 582   if (ZeroTLAB) {
 583     // ..and clear it.
 584     Copy::zero_to_words(obj, new_gclab_size);
 585   } else {
 586     // ...and zap just allocated object.
 587 #ifdef ASSERT
 588     // Skip mangling the space corresponding to the object header to
 589     // ensure that the returned space is not considered parsable by
 590     // any concurrent GC thread.
 591     size_t hdr_size = oopDesc::header_size();
 592     Copy::fill_to_words(obj + hdr_size, new_gclab_size - hdr_size, badHeapWordVal);
 593 #endif // ASSERT
 594   }
 595   thread->gclab().fill(obj, obj + size, new_gclab_size);
 596   return obj;
 597 }
 598 
 599 HeapWord* ShenandoahHeap::allocate_new_tlab(size_t word_size) {
 600   return allocate_new_tlab(word_size, false);
 601 }
 602 
 603 HeapWord* ShenandoahHeap::allocate_new_gclab(size_t word_size) {
 604   return allocate_new_tlab(word_size, true);
 605 }
 606 
 607 HeapWord* ShenandoahHeap::allocate_new_tlab(size_t word_size, bool evacuating) {
 608   HeapWord* result = allocate_memory(word_size, evacuating);
 609 
 610   if (result != NULL) {
 611     assert(! in_collection_set(result), "Never allocate in dirty region");
 612     _bytes_allocated_since_cm += word_size * HeapWordSize;
 613 
 614     log_develop_trace(gc, tlab)("allocating new tlab of size "SIZE_FORMAT" at addr "PTR_FORMAT, word_size, p2i(result));
 615 
 616   }
 617   return result;
 618 }
 619 
 620 ShenandoahHeap* ShenandoahHeap::heap() {
 621   CollectedHeap* heap = Universe::heap();
 622   assert(heap != NULL, "Unitialized access to ShenandoahHeap::heap()");
 623   assert(heap->kind() == CollectedHeap::ShenandoahHeap, "not a shenandoah heap");
 624   return (ShenandoahHeap*) heap;
 625 }
 626 
 627 ShenandoahHeap* ShenandoahHeap::heap_no_check() {
 628   CollectedHeap* heap = Universe::heap();
 629   return (ShenandoahHeap*) heap;
 630 }
 631 
 632 HeapWord* ShenandoahHeap::allocate_memory_work(size_t word_size) {
 633 
 634   ShenandoahHeapLock heap_lock(this);
 635 
 636   HeapWord* result = allocate_memory_under_lock(word_size);
 637   int grow_by = (word_size * HeapWordSize + ShenandoahHeapRegion::RegionSizeBytes - 1) / ShenandoahHeapRegion::RegionSizeBytes;
 638 
 639   while (result == NULL && _num_regions + grow_by <= _max_regions) {
 640     grow_heap_by(grow_by);
 641     result = allocate_memory_under_lock(word_size);
 642   }
 643 
 644   return result;
 645 }
 646 
 647 HeapWord* ShenandoahHeap::allocate_memory(size_t word_size, bool evacuating) {
 648   HeapWord* result = NULL;
 649   result = allocate_memory_work(word_size);
 650 
 651   if (!evacuating) {
 652     // Allocation failed, try full-GC, then retry allocation.
 653     //
 654     // It might happen that one of the threads requesting allocation would unblock
 655     // way later after full-GC happened, only to fail the second allocation, because
 656     // other threads have already depleted the free storage. In this case, a better
 657     // strategy would be to try full-GC again.
 658     //
 659     // Lacking the way to detect progress from "collect" call, we are left with blindly
 660     // retrying for some bounded number of times.
 661     // TODO: Poll if Full GC made enough progress to warrant retry.
 662     int tries = 0;
 663     while ((result == NULL) && (tries++ < ShenandoahFullGCTries)) {
 664       log_debug(gc)("[" PTR_FORMAT " Failed to allocate " SIZE_FORMAT " bytes, doing full GC, try %d",
 665                     p2i(Thread::current()), word_size * HeapWordSize, tries);
 666       collect(GCCause::_allocation_failure);
 667       result = allocate_memory_work(word_size);
 668     }
 669   }
 670 
 671   // Only update monitoring counters when not calling from a write-barrier.
 672   // Otherwise we might attempt to grab the Service_lock, which we must
 673   // not do when coming from a write-barrier (because the thread might
 674   // already hold the Compile_lock).
 675   if (! evacuating) {
 676     monitoring_support()->update_counters();
 677   }
 678 
 679   log_develop_trace(gc, alloc)("allocate memory chunk of size "SIZE_FORMAT" at addr "PTR_FORMAT " by thread %d ",
 680                                word_size, p2i(result), Thread::current()->osthread()->thread_id());
 681 
 682   return result;
 683 }
 684 
 685 bool ShenandoahHeap::call_from_write_barrier(bool evacuating) {
 686   return evacuating && Thread::current()->is_Java_thread();
 687 }
 688 
 689 HeapWord* ShenandoahHeap::allocate_memory_under_lock(size_t word_size) {
 690   assert_heaplock_owned_by_current_thread();
 691 
 692   if (word_size * HeapWordSize > ShenandoahHeapRegion::RegionSizeBytes) {
 693     return allocate_large_memory(word_size);
 694   }
 695 
 696   // Not enough memory in free region set.
 697   // Coming out of full GC, it is possible that there is not
 698   // free region available, so current_index may not be valid.
 699   if (word_size * HeapWordSize > _free_regions->capacity()) return NULL;
 700 
 701   ShenandoahHeapRegion* my_current_region = _free_regions->current_no_humongous();
 702 
 703   if (my_current_region == NULL) {
 704     return NULL; // No more room to make a new region. OOM.
 705   }
 706   assert(my_current_region != NULL, "should have a region at this point");
 707 
 708 #ifdef ASSERT
 709   if (in_collection_set(my_current_region)) {
 710     print_heap_regions();
 711   }
 712 #endif
 713   assert(! in_collection_set(my_current_region), "never get targetted regions in free-lists");
 714   assert(! my_current_region->is_humongous(), "never attempt to allocate from humongous object regions");
 715 
 716   HeapWord* result = my_current_region->allocate(word_size);
 717 
 718   while (result == NULL) {
 719     // 2nd attempt. Try next region.
 720     _free_regions->increase_used(my_current_region->free());
 721     ShenandoahHeapRegion* next_region = _free_regions->next_no_humongous();
 722     assert(next_region != my_current_region, "must not get current again");
 723     my_current_region = next_region;
 724 
 725     if (my_current_region == NULL) {
 726       return NULL; // No more room to make a new region. OOM.
 727     }
 728     assert(my_current_region != NULL, "should have a region at this point");
 729     assert(! in_collection_set(my_current_region), "never get targetted regions in free-lists");
 730     assert(! my_current_region->is_humongous(), "never attempt to allocate from humongous object regions");
 731     result = my_current_region->allocate(word_size);
 732   }
 733 
 734   my_current_region->increase_live_data_words(word_size);
 735   increase_used(word_size * HeapWordSize);
 736   _free_regions->increase_used(word_size * HeapWordSize);
 737   return result;
 738 }
 739 
 740 HeapWord* ShenandoahHeap::allocate_large_memory(size_t words) {
 741   assert_heaplock_owned_by_current_thread();
 742 
 743   uint required_regions = ShenandoahHumongous::required_regions(words * HeapWordSize);
 744   if (required_regions > _max_regions) return NULL;
 745 
 746   ShenandoahHeapRegion* r = _free_regions->allocate_contiguous(required_regions);
 747 
 748   HeapWord* result = NULL;
 749 
 750   if (r != NULL)  {
 751     result = r->bottom();
 752 
 753     log_debug(gc, humongous)("allocating humongous object of size: "SIZE_FORMAT" KB at location "PTR_FORMAT" in start region "SIZE_FORMAT,
 754                              (words * HeapWordSize) / K, p2i(result), r->region_number());
 755   } else {
 756     log_debug(gc, humongous)("allocating humongous object of size: "SIZE_FORMAT" KB at location "PTR_FORMAT" failed",
 757                              (words * HeapWordSize) / K, p2i(result));
 758   }
 759 
 760 
 761   return result;
 762 
 763 }
 764 
 765 HeapWord*  ShenandoahHeap::mem_allocate(size_t size,
 766                                         bool*  gc_overhead_limit_was_exceeded) {
 767 
 768 #ifdef ASSERT
 769   if (ShenandoahVerify && _numAllocs > 1000000) {
 770     _numAllocs = 0;
 771   }
 772   _numAllocs++;
 773 #endif
 774   HeapWord* filler = allocate_memory(size + BrooksPointer::word_size(), false);
 775   HeapWord* result = filler + BrooksPointer::word_size();
 776   if (filler != NULL) {
 777     BrooksPointer::initialize(oop(result));
 778     _bytes_allocated_since_cm += size * HeapWordSize;
 779 
 780     assert(! in_collection_set(result), "never allocate in targetted region");
 781     return result;
 782   } else {
 783     /*
 784     tty->print_cr("Out of memory. Requested number of words: "SIZE_FORMAT" used heap: "INT64_FORMAT", bytes allocated since last CM: "INT64_FORMAT,
 785                   size, used(), _bytes_allocated_since_cm);
 786     {
 787       print_heap_regions();
 788       tty->print("Printing "SIZE_FORMAT" free regions:\n", _free_regions->count());
 789       _free_regions->print();
 790     }
 791     */
 792     return NULL;
 793   }
 794 }
 795 
 796 class ParallelEvacuateRegionObjectClosure : public ObjectClosure {
 797 private:
 798   ShenandoahHeap* _heap;
 799   Thread* _thread;
 800   public:
 801   ParallelEvacuateRegionObjectClosure(ShenandoahHeap* heap) :
 802     _heap(heap), _thread(Thread::current()) {
 803   }
 804 
 805   void do_object(oop p) {
 806 
 807     log_develop_trace(gc, compaction)("Calling ParallelEvacuateRegionObjectClosure on "PTR_FORMAT" of size %d\n", p2i((HeapWord*) p), p->size());
 808 
 809     assert(_heap->is_marked_complete(p), "expect only marked objects");
 810     if (oopDesc::unsafe_equals(p, ShenandoahBarrierSet::resolve_oop_static_not_null(p))) {
 811       bool evac;
 812       _heap->evacuate_object(p, _thread, evac);
 813     }
 814   }
 815 };
 816 
 817 #ifdef ASSERT
 818 class VerifyEvacuatedObjectClosure : public ObjectClosure {
 819 
 820 public:
 821 
 822   void do_object(oop p) {
 823     if (ShenandoahHeap::heap()->is_marked_complete(p)) {
 824       oop p_prime = oopDesc::bs()->read_barrier(p);
 825       assert(! oopDesc::unsafe_equals(p, p_prime), "Should point to evacuated copy");
 826       if (p->klass() != p_prime->klass()) {
 827         tty->print_cr("copy has different class than original:");
 828         p->klass()->print_on(tty);
 829         p_prime->klass()->print_on(tty);
 830       }
 831       assert(p->klass() == p_prime->klass(), "Should have the same class p: "PTR_FORMAT", p_prime: "PTR_FORMAT, p2i(p), p2i(p_prime));
 832       //      assert(p->mark() == p_prime->mark(), "Should have the same mark");
 833       assert(p->size() == p_prime->size(), "Should be the same size");
 834       assert(oopDesc::unsafe_equals(p_prime, oopDesc::bs()->read_barrier(p_prime)), "One forward once");
 835     }
 836   }
 837 };
 838 
 839 void ShenandoahHeap::verify_evacuated_region(ShenandoahHeapRegion* from_region) {
 840   VerifyEvacuatedObjectClosure verify_evacuation;
 841   marked_object_iterate(from_region, &verify_evacuation);
 842 }
 843 #endif
 844 
 845 void ShenandoahHeap::parallel_evacuate_region(ShenandoahHeapRegion* from_region) {
 846 
 847   assert(from_region->has_live(), "all-garbage regions are reclaimed earlier");
 848 
 849   ParallelEvacuateRegionObjectClosure evacuate_region(this);
 850 
 851   marked_object_iterate(from_region, &evacuate_region);
 852 
 853 #ifdef ASSERT
 854   if (ShenandoahVerify && ! cancelled_concgc()) {
 855     verify_evacuated_region(from_region);
 856   }
 857 #endif
 858 }
 859 
 860 class ParallelEvacuationTask : public AbstractGangTask {
 861 private:
 862   ShenandoahHeap* _sh;
 863   ShenandoahCollectionSet* _cs;
 864 
 865 public:
 866   ParallelEvacuationTask(ShenandoahHeap* sh,
 867                          ShenandoahCollectionSet* cs) :
 868     AbstractGangTask("Parallel Evacuation Task"),
 869     _cs(cs),
 870     _sh(sh) {}
 871 
 872   void work(uint worker_id) {
 873 
 874     ShenandoahHeapRegion* from_hr = _cs->claim_next();
 875 
 876     while (from_hr != NULL) {
 877       log_develop_trace(gc, region)("Thread "INT32_FORMAT" claimed Heap Region "SIZE_FORMAT,
 878                                     worker_id,
 879                                     from_hr->region_number());
 880 
 881       assert(from_hr->has_live(), "all-garbage regions are reclaimed early");
 882       _sh->parallel_evacuate_region(from_hr);
 883 
 884       if (_sh->cancelled_concgc()) {
 885         log_develop_trace(gc, region)("Cancelled concgc while evacuating region " SIZE_FORMAT "\n", from_hr->region_number());
 886         break;
 887       }
 888       from_hr = _cs->claim_next();
 889     }
 890   }
 891 };
 892 
 893 class RecycleDirtyRegionsClosure: public ShenandoahHeapRegionClosure {
 894 private:
 895   ShenandoahHeap* _heap;
 896   size_t _bytes_reclaimed;
 897 public:
 898   RecycleDirtyRegionsClosure() : _heap(ShenandoahHeap::heap()) {}
 899 
 900   bool doHeapRegion(ShenandoahHeapRegion* r) {
 901 
 902     assert (! _heap->cancelled_concgc(), "no recycling after cancelled marking");
 903 
 904     if (_heap->in_collection_set(r)) {
 905       log_develop_trace(gc, region)("Recycling region " SIZE_FORMAT ":", r->region_number());
 906       _heap->decrease_used(r->used());
 907       _bytes_reclaimed += r->used();
 908       r->recycle();
 909     }
 910 
 911     return false;
 912   }
 913   size_t bytes_reclaimed() { return _bytes_reclaimed;}
 914   void clear_bytes_reclaimed() {_bytes_reclaimed = 0;}
 915 };
 916 
 917 void ShenandoahHeap::recycle_dirty_regions() {
 918   RecycleDirtyRegionsClosure cl;
 919   cl.clear_bytes_reclaimed();
 920 
 921   heap_region_iterate(&cl);
 922 
 923   _shenandoah_policy->record_bytes_reclaimed(cl.bytes_reclaimed());
 924   if (! cancelled_concgc()) {
 925     clear_cset_fast_test();
 926   }
 927 }
 928 
 929 ShenandoahFreeSet* ShenandoahHeap::free_regions() {
 930   return _free_regions;
 931 }
 932 
 933 void ShenandoahHeap::print_heap_regions(outputStream* st) const {
 934   _ordered_regions->print(st);
 935 }
 936 
 937 class PrintAllRefsOopClosure: public ExtendedOopClosure {
 938 private:
 939   int _index;
 940   const char* _prefix;
 941 
 942 public:
 943   PrintAllRefsOopClosure(const char* prefix) : _index(0), _prefix(prefix) {}
 944 
 945 private:
 946   template <class T>
 947   inline void do_oop_work(T* p) {
 948     oop o = oopDesc::load_decode_heap_oop(p);
 949     if (o != NULL) {
 950       if (ShenandoahHeap::heap()->is_in(o) && o->is_oop()) {
 951         tty->print_cr("%s "INT32_FORMAT" ("PTR_FORMAT")-> "PTR_FORMAT" (marked: %s) (%s "PTR_FORMAT")",
 952                       _prefix, _index,
 953                       p2i(p), p2i(o),
 954                       BOOL_TO_STR(ShenandoahHeap::heap()->is_marked_complete(o)),
 955                       o->klass()->internal_name(), p2i(o->klass()));
 956       } else {
 957         tty->print_cr("%s "INT32_FORMAT" ("PTR_FORMAT" dirty -> "PTR_FORMAT" (not in heap, possibly corrupted or dirty)",
 958                       _prefix, _index,
 959                       p2i(p), p2i(o));
 960       }
 961     } else {
 962       tty->print_cr("%s "INT32_FORMAT" ("PTR_FORMAT") -> "PTR_FORMAT, _prefix, _index, p2i(p), p2i((HeapWord*) o));
 963     }
 964     _index++;
 965   }
 966 
 967 public:
 968   void do_oop(oop* p) {
 969     do_oop_work(p);
 970   }
 971 
 972   void do_oop(narrowOop* p) {
 973     do_oop_work(p);
 974   }
 975 
 976 };
 977 
 978 class PrintAllRefsObjectClosure : public ObjectClosure {
 979   const char* _prefix;
 980 
 981 public:
 982   PrintAllRefsObjectClosure(const char* prefix) : _prefix(prefix) {}
 983 
 984   void do_object(oop p) {
 985     if (ShenandoahHeap::heap()->is_in(p)) {
 986         tty->print_cr("%s object "PTR_FORMAT" (marked: %s) (%s "PTR_FORMAT") refers to:",
 987                       _prefix, p2i(p),
 988                       BOOL_TO_STR(ShenandoahHeap::heap()->is_marked_complete(p)),
 989                       p->klass()->internal_name(), p2i(p->klass()));
 990         PrintAllRefsOopClosure cl(_prefix);
 991         p->oop_iterate(&cl);
 992       }
 993   }
 994 };
 995 
 996 void ShenandoahHeap::print_all_refs(const char* prefix) {
 997   tty->print_cr("printing all references in the heap");
 998   tty->print_cr("root references:");
 999 
1000   ensure_parsability(false);
1001 
1002   PrintAllRefsOopClosure cl(prefix);
1003   roots_iterate(&cl);
1004 
1005   tty->print_cr("heap references:");
1006   PrintAllRefsObjectClosure cl2(prefix);
1007   object_iterate(&cl2);
1008 }
1009 
1010 class VerifyAfterMarkingOopClosure: public ExtendedOopClosure {
1011 private:
1012   ShenandoahHeap*  _heap;
1013 
1014 public:
1015   VerifyAfterMarkingOopClosure() :
1016     _heap(ShenandoahHeap::heap()) { }
1017 
1018 private:
1019   template <class T>
1020   inline void do_oop_work(T* p) {
1021     oop o = oopDesc::load_decode_heap_oop(p);
1022     if (o != NULL) {
1023       if (! _heap->is_marked_complete(o)) {
1024         _heap->print_heap_regions();
1025         _heap->print_all_refs("post-mark");
1026         tty->print_cr("oop not marked, although referrer is marked: "PTR_FORMAT": in_heap: %s, is_marked: %s",
1027                       p2i((HeapWord*) o), BOOL_TO_STR(_heap->is_in(o)), BOOL_TO_STR(_heap->is_marked_complete(o)));
1028         _heap->print_heap_locations((HeapWord*) o, (HeapWord*) o + o->size());
1029 
1030         tty->print_cr("oop class: %s", o->klass()->internal_name());
1031         if (_heap->is_in(p)) {
1032           oop referrer = oop(_heap->heap_region_containing(p)->block_start_const(p));
1033           tty->print_cr("Referrer starts at addr "PTR_FORMAT, p2i((HeapWord*) referrer));
1034           referrer->print();
1035           _heap->print_heap_locations((HeapWord*) referrer, (HeapWord*) referrer + referrer->size());
1036         }
1037         tty->print_cr("heap region containing object:");
1038         _heap->heap_region_containing(o)->print();
1039         tty->print_cr("heap region containing referrer:");
1040         _heap->heap_region_containing(p)->print();
1041         tty->print_cr("heap region containing forwardee:");
1042         _heap->heap_region_containing(oopDesc::bs()->read_barrier(o))->print();
1043       }
1044       assert(o->is_oop(), "oop must be an oop");
1045       assert(Metaspace::contains(o->klass()), "klass pointer must go to metaspace");
1046       if (! oopDesc::unsafe_equals(o, oopDesc::bs()->read_barrier(o))) {
1047         tty->print_cr("oops has forwardee: p: "PTR_FORMAT" (%s), o = "PTR_FORMAT" (%s), new-o: "PTR_FORMAT" (%s)",
1048                       p2i(p),
1049                       BOOL_TO_STR(_heap->in_collection_set(p)),
1050                       p2i(o),
1051                       BOOL_TO_STR(_heap->in_collection_set(o)),
1052                       p2i((HeapWord*) oopDesc::bs()->read_barrier(o)),
1053                       BOOL_TO_STR(_heap->in_collection_set(oopDesc::bs()->read_barrier(o))));
1054         tty->print_cr("oop class: %s", o->klass()->internal_name());
1055       }
1056       assert(oopDesc::unsafe_equals(o, oopDesc::bs()->read_barrier(o)), "oops must not be forwarded");
1057       assert(! _heap->in_collection_set(o), "references must not point to dirty heap regions");
1058       assert(_heap->is_marked_complete(o), "live oops must be marked current");
1059     }
1060   }
1061 
1062 public:
1063   void do_oop(oop* p) {
1064     do_oop_work(p);
1065   }
1066 
1067   void do_oop(narrowOop* p) {
1068     do_oop_work(p);
1069   }
1070 
1071 };
1072 
1073 void ShenandoahHeap::verify_heap_after_marking() {
1074 
1075   verify_heap_size_consistency();
1076 
1077   log_trace(gc)("verifying heap after marking");
1078 
1079   VerifyAfterMarkingOopClosure cl;
1080   roots_iterate(&cl);
1081   ObjectToOopClosure objs(&cl);
1082   object_iterate(&objs);
1083 }
1084 
1085 
1086 void ShenandoahHeap::reclaim_humongous_region_at(ShenandoahHeapRegion* r) {
1087   assert(r->is_humongous_start(), "reclaim regions starting with the first one");
1088 
1089   oop humongous_obj = oop(r->bottom() + BrooksPointer::word_size());
1090   size_t size = humongous_obj->size() + BrooksPointer::word_size();
1091   uint required_regions = ShenandoahHumongous::required_regions(size * HeapWordSize);
1092   size_t index = r->region_number();
1093 
1094 
1095   assert(!r->has_live(), "liveness must be zero");
1096 
1097   for(size_t i = 0; i < required_regions; i++) {
1098 
1099     ShenandoahHeapRegion* region = _ordered_regions->get(index++);
1100 
1101     assert((region->is_humongous_start() || region->is_humongous_continuation()),
1102            "expect correct humongous start or continuation");
1103 
1104     if (log_is_enabled(Debug, gc, humongous)) {
1105       log_debug(gc, humongous)("reclaiming "UINT32_FORMAT" humongous regions for object of size: "SIZE_FORMAT" words", required_regions, size);
1106       ResourceMark rm;
1107       outputStream* out = Log(gc, humongous)::debug_stream();
1108       region->print_on(out);
1109     }
1110 
1111     region->recycle();
1112     ShenandoahHeap::heap()->decrease_used(ShenandoahHeapRegion::RegionSizeBytes);
1113   }
1114 }
1115 
1116 class ShenandoahReclaimHumongousRegionsClosure : public ShenandoahHeapRegionClosure {
1117 
1118   bool doHeapRegion(ShenandoahHeapRegion* r) {
1119     ShenandoahHeap* heap = ShenandoahHeap::heap();
1120 
1121     if (r->is_humongous_start()) {
1122       oop humongous_obj = oop(r->bottom() + BrooksPointer::word_size());
1123       if (! heap->is_marked_complete(humongous_obj)) {
1124 
1125         heap->reclaim_humongous_region_at(r);
1126       }
1127     }
1128     return false;
1129   }
1130 };
1131 
1132 #ifdef ASSERT
1133 class CheckCollectionSetClosure: public ShenandoahHeapRegionClosure {
1134   bool doHeapRegion(ShenandoahHeapRegion* r) {
1135     assert(! ShenandoahHeap::heap()->in_collection_set(r), "Should have been cleared by now");
1136     return false;
1137   }
1138 };
1139 #endif
1140 
1141 void ShenandoahHeap::prepare_for_concurrent_evacuation() {
1142   assert(_ordered_regions->get(0)->region_number() == 0, "FIXME CHF. FIXME CHF!");
1143 
1144   log_develop_trace(gc)("Thread %d started prepare_for_concurrent_evacuation", Thread::current()->osthread()->thread_id());
1145 
1146   if (!cancelled_concgc()) {
1147 
1148     recycle_dirty_regions();
1149 
1150     ensure_parsability(true);
1151 
1152     if (UseShenandoahMatrix) {
1153       if (PrintShenandoahMatrix) {
1154         outputStream* log = Log(gc)::info_stream();
1155         connection_matrix()->print_on(log);
1156       }
1157     }
1158 
1159     if (ShenandoahVerify || (UseShenandoahMatrix && VerifyShenandoahMatrix)) {
1160       verify_heap_reachable_at_safepoint();
1161     }
1162 
1163 #ifdef ASSERT
1164     if (ShenandoahVerify) {
1165       verify_heap_after_marking();
1166     }
1167 #endif
1168 
1169     // NOTE: This needs to be done during a stop the world pause, because
1170     // putting regions into the collection set concurrently with Java threads
1171     // will create a race. In particular, acmp could fail because when we
1172     // resolve the first operand, the containing region might not yet be in
1173     // the collection set, and thus return the original oop. When the 2nd
1174     // operand gets resolved, the region could be in the collection set
1175     // and the oop gets evacuated. If both operands have originally been
1176     // the same, we get false negatives.
1177 
1178     {
1179       ShenandoahHeapLock lock(this);
1180       _collection_set->clear();
1181       _free_regions->clear();
1182 
1183       ShenandoahReclaimHumongousRegionsClosure reclaim;
1184       heap_region_iterate(&reclaim);
1185 
1186 #ifdef ASSERT
1187       CheckCollectionSetClosure ccsc;
1188       _ordered_regions->heap_region_iterate(&ccsc);
1189 #endif
1190 
1191       _shenandoah_policy->choose_collection_set(_collection_set);
1192 
1193       _shenandoah_policy->choose_free_set(_free_regions);
1194     }
1195 
1196     _bytes_allocated_since_cm = 0;
1197 
1198     Universe::update_heap_info_at_gc();
1199   }
1200 }
1201 
1202 
1203 class RetireTLABClosure : public ThreadClosure {
1204 private:
1205   bool _retire;
1206 
1207 public:
1208   RetireTLABClosure(bool retire) : _retire(retire) {
1209   }
1210 
1211   void do_thread(Thread* thread) {
1212     thread->gclab().make_parsable(_retire);
1213   }
1214 };
1215 
1216 void ShenandoahHeap::ensure_parsability(bool retire_tlabs) {
1217   if (UseTLAB) {
1218     CollectedHeap::ensure_parsability(retire_tlabs);
1219     RetireTLABClosure cl(retire_tlabs);
1220     Threads::threads_do(&cl);
1221   }
1222 }
1223 
1224 class ShenandoahEvacuateUpdateRootsClosure: public ExtendedOopClosure {
1225 private:
1226   ShenandoahHeap* _heap;
1227   Thread* _thread;
1228 public:
1229   ShenandoahEvacuateUpdateRootsClosure() :
1230     _heap(ShenandoahHeap::heap()), _thread(Thread::current()) {
1231   }
1232 
1233 private:
1234   template <class T>
1235   void do_oop_work(T* p) {
1236     assert(_heap->is_evacuation_in_progress(), "Only do this when evacuation is in progress");
1237 
1238     T o = oopDesc::load_heap_oop(p);
1239     if (! oopDesc::is_null(o)) {
1240       oop obj = oopDesc::decode_heap_oop_not_null(o);
1241       if (_heap->in_collection_set(obj)) {
1242         assert(_heap->is_marked_complete(obj), "only evacuate marked objects %d %d",
1243                _heap->is_marked_complete(obj), _heap->is_marked_complete(ShenandoahBarrierSet::resolve_oop_static_not_null(obj)));
1244         oop resolved = ShenandoahBarrierSet::resolve_oop_static_not_null(obj);
1245         if (oopDesc::unsafe_equals(resolved, obj)) {
1246           bool evac;
1247           resolved = _heap->evacuate_object(obj, _thread, evac);
1248         }
1249         oopDesc::encode_store_heap_oop(p, resolved);
1250       }
1251     }
1252 #ifdef ASSERT
1253     else {
1254       // tty->print_cr("not updating root at: "PTR_FORMAT" with object: "PTR_FORMAT", is_in_heap: %s, is_in_cset: %s, is_marked: %s",
1255       //               p2i(p),
1256       //               p2i((HeapWord*) obj),
1257       //               BOOL_TO_STR(_heap->is_in(obj)),
1258       //               BOOL_TO_STR(_heap->in_cset_fast_test(obj)),
1259       //               BOOL_TO_STR(_heap->is_marked_complete(obj)));
1260     }
1261 #endif
1262   }
1263 
1264 public:
1265   void do_oop(oop* p) {
1266     do_oop_work(p);
1267   }
1268   void do_oop(narrowOop* p) {
1269     do_oop_work(p);
1270   }
1271 };
1272 
1273 class ShenandoahEvacuateUpdateRootsTask : public AbstractGangTask {
1274   ShenandoahRootEvacuator* _rp;
1275 public:
1276 
1277   ShenandoahEvacuateUpdateRootsTask(ShenandoahRootEvacuator* rp) :
1278     AbstractGangTask("Shenandoah evacuate and update roots"),
1279     _rp(rp)
1280   {
1281     // Nothing else to do.
1282   }
1283 
1284   void work(uint worker_id) {
1285     ShenandoahEvacuateUpdateRootsClosure cl;
1286     MarkingCodeBlobClosure blobsCl(&cl, CodeBlobToOopClosure::FixRelocations);
1287 
1288     _rp->process_evacuate_roots(&cl, &blobsCl, worker_id);
1289   }
1290 };
1291 
1292 class ShenandoahFixRootsTask : public AbstractGangTask {
1293   ShenandoahRootEvacuator* _rp;
1294 public:
1295 
1296   ShenandoahFixRootsTask(ShenandoahRootEvacuator* rp) :
1297     AbstractGangTask("Shenandoah update roots"),
1298     _rp(rp)
1299   {
1300     // Nothing else to do.
1301   }
1302 
1303   void work(uint worker_id) {
1304     SCMUpdateRefsClosure cl;
1305     MarkingCodeBlobClosure blobsCl(&cl, CodeBlobToOopClosure::FixRelocations);
1306 
1307     _rp->process_evacuate_roots(&cl, &blobsCl, worker_id);
1308   }
1309 };
1310 void ShenandoahHeap::evacuate_and_update_roots() {
1311 
1312   COMPILER2_PRESENT(DerivedPointerTable::clear());
1313 
1314 #ifdef ASSERT
1315   if (ShenandoahVerifyReadsToFromSpace) {
1316     set_from_region_protection(false);
1317   }
1318 #endif
1319 
1320   assert(SafepointSynchronize::is_at_safepoint(), "Only iterate roots while world is stopped");
1321   ClassLoaderDataGraph::clear_claimed_marks();
1322 
1323   {
1324     ShenandoahRootEvacuator rp(this, workers()->active_workers(), ShenandoahCollectorPolicy::evac_thread_roots);
1325     ShenandoahEvacuateUpdateRootsTask roots_task(&rp);
1326     workers()->run_task(&roots_task);
1327   }
1328 
1329   COMPILER2_PRESENT(DerivedPointerTable::update_pointers());
1330 
1331   if (cancelled_concgc()) {
1332     // If initial evacuation has been cancelled, we need to update all references
1333     // after all workers have finished. Otherwise we might run into the following problem:
1334     // GC thread 1 cannot allocate anymore, thus evacuation fails, leaves from-space ptr of object X.
1335     // GC thread 2 evacuates the same object X to to-space
1336     // which leaves a truly dangling from-space reference in the first root oop*. This must not happen.
1337     // clear() and update_pointers() must always be called in pairs,
1338     // cannot nest with above clear()/update_pointers().
1339     COMPILER2_PRESENT(DerivedPointerTable::clear());
1340     ShenandoahRootEvacuator rp(this, workers()->active_workers(), ShenandoahCollectorPolicy::evac_thread_roots);
1341     ShenandoahFixRootsTask update_roots_task(&rp);
1342     workers()->run_task(&update_roots_task);
1343     COMPILER2_PRESENT(DerivedPointerTable::update_pointers());
1344   }
1345 
1346 #ifdef ASSERT
1347   if (ShenandoahVerifyReadsToFromSpace) {
1348     set_from_region_protection(true);
1349   }
1350 #endif
1351 
1352 #ifdef ASSERT
1353   {
1354     AssertToSpaceClosure cl;
1355     CodeBlobToOopClosure code_cl(&cl, !CodeBlobToOopClosure::FixRelocations);
1356     ShenandoahRootEvacuator rp(this, 1);
1357     rp.process_evacuate_roots(&cl, &code_cl, 0);
1358   }
1359 #endif
1360 }
1361 
1362 
1363 void ShenandoahHeap::do_evacuation() {
1364 
1365   parallel_evacuate();
1366 
1367   if (ShenandoahVerify && ! cancelled_concgc()) {
1368     VM_ShenandoahVerifyHeapAfterEvacuation verify_after_evacuation;
1369     if (Thread::current()->is_VM_thread()) {
1370       verify_after_evacuation.doit();
1371     } else {
1372       VMThread::execute(&verify_after_evacuation);
1373     }
1374   }
1375 
1376 }
1377 
1378 void ShenandoahHeap::parallel_evacuate() {
1379   log_develop_trace(gc)("starting parallel_evacuate");
1380 
1381   _shenandoah_policy->record_phase_start(ShenandoahCollectorPolicy::conc_evac);
1382 
1383   if (log_is_enabled(Trace, gc, region)) {
1384     ResourceMark rm;
1385     outputStream *out = Log(gc, region)::trace_stream();
1386     out->print("Printing all available regions");
1387     print_heap_regions(out);
1388   }
1389 
1390   if (log_is_enabled(Trace, gc, cset)) {
1391     ResourceMark rm;
1392     outputStream *out = Log(gc, cset)::trace_stream();
1393     out->print("Printing collection set which contains "SIZE_FORMAT" regions:\n", _collection_set->count());
1394     _collection_set->print(out);
1395 
1396     out->print("Printing free set which contains "SIZE_FORMAT" regions:\n", _free_regions->count());
1397     _free_regions->print(out);
1398   }
1399 
1400   ParallelEvacuationTask evacuationTask = ParallelEvacuationTask(this, _collection_set);
1401 
1402 
1403   workers()->run_task(&evacuationTask);
1404 
1405   if (log_is_enabled(Trace, gc, cset)) {
1406     ResourceMark rm;
1407     outputStream *out = Log(gc, cset)::trace_stream();
1408     out->print("Printing postgc collection set which contains "SIZE_FORMAT" regions:\n",
1409                _collection_set->count());
1410 
1411     _collection_set->print(out);
1412 
1413     out->print("Printing postgc free regions which contain "SIZE_FORMAT" free regions:\n",
1414                _free_regions->count());
1415     _free_regions->print(out);
1416 
1417   }
1418 
1419   if (log_is_enabled(Trace, gc, region)) {
1420     ResourceMark rm;
1421     outputStream *out = Log(gc, region)::trace_stream();
1422     out->print_cr("all regions after evacuation:");
1423     print_heap_regions(out);
1424   }
1425 
1426   _shenandoah_policy->record_phase_end(ShenandoahCollectorPolicy::conc_evac);
1427 }
1428 
1429 class VerifyEvacuationClosure: public ExtendedOopClosure {
1430 private:
1431   ShenandoahHeap*  _heap;
1432   ShenandoahHeapRegion* _from_region;
1433 
1434 public:
1435   VerifyEvacuationClosure(ShenandoahHeapRegion* from_region) :
1436     _heap(ShenandoahHeap::heap()), _from_region(from_region) { }
1437 private:
1438   template <class T>
1439   inline void do_oop_work(T* p) {
1440     oop heap_oop = oopDesc::load_decode_heap_oop(p);
1441     if (! oopDesc::is_null(heap_oop)) {
1442       guarantee(! _from_region->is_in(heap_oop), "no references to from-region allowed after evacuation: "PTR_FORMAT, p2i((HeapWord*) heap_oop));
1443     }
1444   }
1445 
1446 public:
1447   void do_oop(oop* p)       {
1448     do_oop_work(p);
1449   }
1450 
1451   void do_oop(narrowOop* p) {
1452     do_oop_work(p);
1453   }
1454 
1455 };
1456 
1457 void ShenandoahHeap::roots_iterate(OopClosure* cl) {
1458 
1459   assert(SafepointSynchronize::is_at_safepoint(), "Only iterate roots while world is stopped");
1460 
1461   CodeBlobToOopClosure blobsCl(cl, false);
1462   CLDToOopClosure cldCl(cl);
1463 
1464   ClassLoaderDataGraph::clear_claimed_marks();
1465 
1466   ShenandoahRootProcessor rp(this, 1);
1467   rp.process_all_roots(cl, NULL, &cldCl, &blobsCl, 0);
1468 }
1469 
1470 void ShenandoahHeap::verify_evacuation(ShenandoahHeapRegion* from_region) {
1471 
1472   VerifyEvacuationClosure rootsCl(from_region);
1473   roots_iterate(&rootsCl);
1474 
1475 }
1476 
1477 bool ShenandoahHeap::supports_tlab_allocation() const {
1478   return true;
1479 }
1480 
1481 
1482 size_t  ShenandoahHeap::unsafe_max_tlab_alloc(Thread *thread) const {
1483   size_t idx = _free_regions->current_index();
1484   ShenandoahHeapRegion* current = _free_regions->get(idx);
1485   if (current == NULL) {
1486     return 0;
1487   } else if (current->free() > MinTLABSize) {
1488     // Current region has enough space left, can use it.
1489     return current->free();
1490   } else {
1491     // No more space in current region, we will take next free region
1492     // on the next TLAB allocation.
1493     return ShenandoahHeapRegion::RegionSizeBytes;
1494   }
1495 }
1496 
1497 size_t ShenandoahHeap::max_tlab_size() const {
1498   return ShenandoahHeapRegion::RegionSizeBytes;
1499 }
1500 
1501 class ResizeGCLABClosure : public ThreadClosure {
1502 public:
1503   void do_thread(Thread* thread) {
1504     thread->gclab().resize();
1505   }
1506 };
1507 
1508 void ShenandoahHeap::resize_all_tlabs() {
1509   CollectedHeap::resize_all_tlabs();
1510 
1511   ResizeGCLABClosure cl;
1512   Threads::threads_do(&cl);
1513 }
1514 
1515 class AccumulateStatisticsGCLABClosure : public ThreadClosure {
1516 public:
1517   void do_thread(Thread* thread) {
1518     thread->gclab().accumulate_statistics();
1519     thread->gclab().initialize_statistics();
1520   }
1521 };
1522 
1523 void ShenandoahHeap::accumulate_statistics_all_gclabs() {
1524   AccumulateStatisticsGCLABClosure cl;
1525   Threads::threads_do(&cl);
1526 }
1527 
1528 bool  ShenandoahHeap::can_elide_tlab_store_barriers() const {
1529   return true;
1530 }
1531 
1532 oop ShenandoahHeap::new_store_pre_barrier(JavaThread* thread, oop new_obj) {
1533   // Overridden to do nothing.
1534   return new_obj;
1535 }
1536 
1537 bool  ShenandoahHeap::can_elide_initializing_store_barrier(oop new_obj) {
1538   return true;
1539 }
1540 
1541 bool ShenandoahHeap::card_mark_must_follow_store() const {
1542   return false;
1543 }
1544 
1545 void ShenandoahHeap::collect(GCCause::Cause cause) {
1546   assert(cause != GCCause::_gc_locker, "no JNI critical callback");
1547   if (GCCause::is_user_requested_gc(cause)) {
1548     if (! DisableExplicitGC) {
1549       _concurrent_gc_thread->do_full_gc(cause);
1550     }
1551   } else if (cause == GCCause::_allocation_failure) {
1552     collector_policy()->set_should_clear_all_soft_refs(true);
1553     _concurrent_gc_thread->do_full_gc(cause);
1554   }
1555 }
1556 
1557 void ShenandoahHeap::do_full_collection(bool clear_all_soft_refs) {
1558   //assert(false, "Shouldn't need to do full collections");
1559 }
1560 
1561 AdaptiveSizePolicy* ShenandoahHeap::size_policy() {
1562   Unimplemented();
1563   return NULL;
1564 
1565 }
1566 
1567 CollectorPolicy* ShenandoahHeap::collector_policy() const {
1568   return _shenandoah_policy;
1569 }
1570 
1571 
1572 HeapWord* ShenandoahHeap::block_start(const void* addr) const {
1573   Space* sp = heap_region_containing(addr);
1574   if (sp != NULL) {
1575     return sp->block_start(addr);
1576   }
1577   return NULL;
1578 }
1579 
1580 size_t ShenandoahHeap::block_size(const HeapWord* addr) const {
1581   Space* sp = heap_region_containing(addr);
1582   assert(sp != NULL, "block_size of address outside of heap");
1583   return sp->block_size(addr);
1584 }
1585 
1586 bool ShenandoahHeap::block_is_obj(const HeapWord* addr) const {
1587   Space* sp = heap_region_containing(addr);
1588   return sp->block_is_obj(addr);
1589 }
1590 
1591 jlong ShenandoahHeap::millis_since_last_gc() {
1592   return 0;
1593 }
1594 
1595 void ShenandoahHeap::prepare_for_verify() {
1596   if (SafepointSynchronize::is_at_safepoint() || ! UseTLAB) {
1597     ensure_parsability(false);
1598   }
1599 }
1600 
1601 void ShenandoahHeap::print_gc_threads_on(outputStream* st) const {
1602   workers()->print_worker_threads_on(st);
1603 }
1604 
1605 void ShenandoahHeap::gc_threads_do(ThreadClosure* tcl) const {
1606   workers()->threads_do(tcl);
1607 }
1608 
1609 void ShenandoahHeap::print_tracing_info() const {
1610   if (log_is_enabled(Info, gc, stats)) {
1611     ResourceMark rm;
1612     outputStream* out = Log(gc, stats)::info_stream();
1613     _shenandoah_policy->print_tracing_info(out);
1614   }
1615 }
1616 
1617 class ShenandoahVerifyRootsClosure: public ExtendedOopClosure {
1618 private:
1619   ShenandoahHeap*  _heap;
1620   VerifyOption     _vo;
1621   bool             _failures;
1622 public:
1623   // _vo == UsePrevMarking -> use "prev" marking information,
1624   // _vo == UseNextMarking -> use "next" marking information,
1625   // _vo == UseMarkWord    -> use mark word from object header.
1626   ShenandoahVerifyRootsClosure(VerifyOption vo) :
1627     _heap(ShenandoahHeap::heap()),
1628     _vo(vo),
1629     _failures(false) { }
1630 
1631   bool failures() { return _failures; }
1632 
1633 private:
1634   template <class T>
1635   inline void do_oop_work(T* p) {
1636     oop obj = oopDesc::load_decode_heap_oop(p);
1637     if (! oopDesc::is_null(obj) && ! obj->is_oop()) {
1638       { // Just for debugging.
1639         tty->print_cr("Root location "PTR_FORMAT
1640                       "verified "PTR_FORMAT, p2i(p), p2i((void*) obj));
1641         //      obj->print_on(tty);
1642       }
1643     }
1644     guarantee(obj->is_oop_or_null(), "is oop or null");
1645   }
1646 
1647 public:
1648   void do_oop(oop* p)       {
1649     do_oop_work(p);
1650   }
1651 
1652   void do_oop(narrowOop* p) {
1653     do_oop_work(p);
1654   }
1655 
1656 };
1657 
1658 class ShenandoahVerifyHeapClosure: public ObjectClosure {
1659 private:
1660   ShenandoahVerifyRootsClosure _rootsCl;
1661 public:
1662   ShenandoahVerifyHeapClosure(ShenandoahVerifyRootsClosure rc) :
1663     _rootsCl(rc) {};
1664 
1665   void do_object(oop p) {
1666     _rootsCl.do_oop(&p);
1667   }
1668 };
1669 
1670 class ShenandoahVerifyKlassClosure: public KlassClosure {
1671   OopClosure *_oop_closure;
1672  public:
1673   ShenandoahVerifyKlassClosure(OopClosure* cl) : _oop_closure(cl) {}
1674   void do_klass(Klass* k) {
1675     k->oops_do(_oop_closure);
1676   }
1677 };
1678 
1679 void ShenandoahHeap::verify(VerifyOption vo) {
1680   if (SafepointSynchronize::is_at_safepoint() || ! UseTLAB) {
1681 
1682     ShenandoahVerifyRootsClosure rootsCl(vo);
1683 
1684     assert(Thread::current()->is_VM_thread(),
1685            "Expected to be executed serially by the VM thread at this point");
1686 
1687     roots_iterate(&rootsCl);
1688 
1689     bool failures = rootsCl.failures();
1690     log_trace(gc)("verify failures: %s", BOOL_TO_STR(failures));
1691 
1692     ShenandoahVerifyHeapClosure heapCl(rootsCl);
1693 
1694     object_iterate(&heapCl);
1695     // TODO: Implement rest of it.
1696   } else {
1697     tty->print("(SKIPPING roots, heapRegions, remset) ");
1698   }
1699 }
1700 size_t ShenandoahHeap::tlab_capacity(Thread *thr) const {
1701   return _free_regions->capacity();
1702 }
1703 
1704 class ShenandoahIterateObjectClosureRegionClosure: public ShenandoahHeapRegionClosure {
1705   ObjectClosure* _cl;
1706 public:
1707   ShenandoahIterateObjectClosureRegionClosure(ObjectClosure* cl) : _cl(cl) {}
1708   bool doHeapRegion(ShenandoahHeapRegion* r) {
1709     ShenandoahHeap::heap()->marked_object_iterate(r, _cl);
1710     return false;
1711   }
1712 };
1713 
1714 void ShenandoahHeap::object_iterate(ObjectClosure* cl) {
1715   ShenandoahIterateObjectClosureRegionClosure blk(cl);
1716   heap_region_iterate(&blk, false, true);
1717 }
1718 
1719 class ShenandoahSafeObjectIterateAdjustPtrsClosure : public MetadataAwareOopClosure {
1720 private:
1721   ShenandoahHeap* _heap;
1722 
1723 public:
1724   ShenandoahSafeObjectIterateAdjustPtrsClosure() : _heap(ShenandoahHeap::heap()) {}
1725 
1726 private:
1727   template <class T>
1728   inline void do_oop_work(T* p) {
1729     T o = oopDesc::load_heap_oop(p);
1730     if (!oopDesc::is_null(o)) {
1731       oop obj = oopDesc::decode_heap_oop_not_null(o);
1732       oopDesc::encode_store_heap_oop(p, BrooksPointer::forwardee(obj));
1733     }
1734   }
1735 public:
1736   void do_oop(oop* p) {
1737     do_oop_work(p);
1738   }
1739   void do_oop(narrowOop* p) {
1740     do_oop_work(p);
1741   }
1742 };
1743 
1744 class ShenandoahSafeObjectIterateAndUpdate : public ObjectClosure {
1745 private:
1746   ObjectClosure* _cl;
1747 public:
1748   ShenandoahSafeObjectIterateAndUpdate(ObjectClosure *cl) : _cl(cl) {}
1749 
1750   virtual void do_object(oop obj) {
1751     assert (oopDesc::unsafe_equals(obj, BrooksPointer::forwardee(obj)),
1752             "avoid double-counting: only non-forwarded objects here");
1753 
1754     // Fix up the ptrs.
1755     ShenandoahSafeObjectIterateAdjustPtrsClosure adjust_ptrs;
1756     obj->oop_iterate(&adjust_ptrs);
1757 
1758     // Can reply the object now:
1759     _cl->do_object(obj);
1760   }
1761 };
1762 
1763 void ShenandoahHeap::safe_object_iterate(ObjectClosure* cl) {
1764   assert(SafepointSynchronize::is_at_safepoint(), "safe iteration is only available during safepoints");
1765 
1766   // Safe iteration does objects only with correct references.
1767   // This is why we skip dirty regions that have stale copies of objects,
1768   // and fix up the pointers in the returned objects.
1769 
1770   ShenandoahSafeObjectIterateAndUpdate safe_cl(cl);
1771   ShenandoahIterateObjectClosureRegionClosure blk(&safe_cl);
1772   heap_region_iterate(&blk,
1773                       /* skip_dirty_regions = */ true,
1774                       /* skip_humongous_continuations = */ true);
1775 
1776   _need_update_refs = false; // already updated the references
1777 }
1778 
1779 // Apply blk->doHeapRegion() on all committed regions in address order,
1780 // terminating the iteration early if doHeapRegion() returns true.
1781 void ShenandoahHeap::heap_region_iterate(ShenandoahHeapRegionClosure* blk, bool skip_dirty_regions, bool skip_humongous_continuation) const {
1782   for (size_t i = 0; i < _num_regions; i++) {
1783     ShenandoahHeapRegion* current  = _ordered_regions->get(i);
1784     if (skip_humongous_continuation && current->is_humongous_continuation()) {
1785       continue;
1786     }
1787     if (skip_dirty_regions && in_collection_set(current)) {
1788       continue;
1789     }
1790     if (blk->doHeapRegion(current)) {
1791       return;
1792     }
1793   }
1794 }
1795 
1796 class ClearLivenessClosure : public ShenandoahHeapRegionClosure {
1797   ShenandoahHeap* sh;
1798 public:
1799   ClearLivenessClosure(ShenandoahHeap* heap) : sh(heap) { }
1800 
1801   bool doHeapRegion(ShenandoahHeapRegion* r) {
1802     r->clear_live_data();
1803     sh->set_next_top_at_mark_start(r->bottom(), r->top());
1804     return false;
1805   }
1806 };
1807 
1808 void ShenandoahHeap::start_concurrent_marking() {
1809 
1810   shenandoahPolicy()->record_phase_start(ShenandoahCollectorPolicy::accumulate_stats);
1811   accumulate_statistics_all_tlabs();
1812   shenandoahPolicy()->record_phase_end(ShenandoahCollectorPolicy::accumulate_stats);
1813 
1814   set_concurrent_mark_in_progress(true);
1815   // We need to reset all TLABs because we'd lose marks on all objects allocated in them.
1816   if (UseTLAB) {
1817     shenandoahPolicy()->record_phase_start(ShenandoahCollectorPolicy::make_parsable);
1818     ensure_parsability(true);
1819     shenandoahPolicy()->record_phase_end(ShenandoahCollectorPolicy::make_parsable);
1820   }
1821 
1822   _shenandoah_policy->record_bytes_allocated(_bytes_allocated_since_cm);
1823   _used_start_gc = used();
1824 
1825 #ifdef ASSERT
1826   if (ShenandoahDumpHeapBeforeConcurrentMark) {
1827     ensure_parsability(false);
1828     print_all_refs("pre-mark");
1829   }
1830 #endif
1831 
1832   shenandoahPolicy()->record_phase_start(ShenandoahCollectorPolicy::clear_liveness);
1833   ClearLivenessClosure clc(this);
1834   heap_region_iterate(&clc);
1835   shenandoahPolicy()->record_phase_end(ShenandoahCollectorPolicy::clear_liveness);
1836 
1837   if (UseShenandoahMatrix) {
1838     connection_matrix()->clear_all();
1839   }
1840   // print_all_refs("pre -mark");
1841 
1842   // oopDesc::_debug = true;
1843 
1844   // Make above changes visible to worker threads
1845   OrderAccess::fence();
1846 
1847   shenandoahPolicy()->record_phase_start(ShenandoahCollectorPolicy::scan_roots);
1848   concurrentMark()->init_mark_roots();
1849   shenandoahPolicy()->record_phase_end(ShenandoahCollectorPolicy::scan_roots);
1850 
1851   //  print_all_refs("pre-mark2");
1852 }
1853 
1854 class VerifyAfterEvacuationClosure : public ExtendedOopClosure {
1855 
1856   ShenandoahHeap* _sh;
1857 
1858 public:
1859   VerifyAfterEvacuationClosure() : _sh ( ShenandoahHeap::heap() ) {}
1860 
1861   template<class T> void do_oop_nv(T* p) {
1862     T heap_oop = oopDesc::load_heap_oop(p);
1863     if (!oopDesc::is_null(heap_oop)) {
1864       oop obj = oopDesc::decode_heap_oop_not_null(heap_oop);
1865       guarantee(_sh->in_collection_set(obj) == (! oopDesc::unsafe_equals(obj, oopDesc::bs()->read_barrier(obj))),
1866                 "forwarded objects can only exist in dirty (from-space) regions is_dirty: %s, is_forwarded: %s obj-klass: %s, marked: %s",
1867                 BOOL_TO_STR(_sh->in_collection_set(obj)),
1868                 BOOL_TO_STR(! oopDesc::unsafe_equals(obj, oopDesc::bs()->read_barrier(obj))),
1869                 obj->klass()->external_name(),
1870                 BOOL_TO_STR(_sh->is_marked_complete(obj))
1871                 );
1872       obj = oopDesc::bs()->read_barrier(obj);
1873       guarantee(! _sh->in_collection_set(obj), "forwarded oops must not point to dirty regions");
1874       guarantee(obj->is_oop(), "is_oop");
1875       guarantee(Metaspace::contains(obj->klass()), "klass pointer must go to metaspace");
1876     }
1877   }
1878 
1879   void do_oop(oop* p)       { do_oop_nv(p); }
1880   void do_oop(narrowOop* p) { do_oop_nv(p); }
1881 
1882 };
1883 
1884 void ShenandoahHeap::verify_heap_after_evacuation() {
1885 
1886   verify_heap_size_consistency();
1887 
1888   ensure_parsability(false);
1889 
1890   VerifyAfterEvacuationClosure cl;
1891   roots_iterate(&cl);
1892 
1893   ObjectToOopClosure objs(&cl);
1894   object_iterate(&objs);
1895 
1896 }
1897 
1898 class VerifyRegionsAfterUpdateRefsClosure : public ShenandoahHeapRegionClosure {
1899 public:
1900   bool doHeapRegion(ShenandoahHeapRegion* r) {
1901     assert(! ShenandoahHeap::heap()->in_collection_set(r), "no region must be in collection set");
1902     return false;
1903   }
1904 };
1905 
1906 void ShenandoahHeap::swap_mark_bitmaps() {
1907   // Swap bitmaps.
1908   CMBitMap* tmp1 = _complete_mark_bit_map;
1909   _complete_mark_bit_map = _next_mark_bit_map;
1910   _next_mark_bit_map = tmp1;
1911 
1912   // Swap top-at-mark-start pointers
1913   HeapWord** tmp2 = _complete_top_at_mark_starts;
1914   _complete_top_at_mark_starts = _next_top_at_mark_starts;
1915   _next_top_at_mark_starts = tmp2;
1916 
1917   HeapWord** tmp3 = _complete_top_at_mark_starts_base;
1918   _complete_top_at_mark_starts_base = _next_top_at_mark_starts_base;
1919   _next_top_at_mark_starts_base = tmp3;
1920 }
1921 
1922 class VerifyReachableHeapClosure : public ExtendedOopClosure {
1923 private:
1924   SCMObjToScanQueue* _queue;
1925   ShenandoahHeap* _heap;
1926   CMBitMap* _map;
1927   bool _check_matrix;
1928   oop _obj;
1929 public:
1930   VerifyReachableHeapClosure(SCMObjToScanQueue* queue, CMBitMap* map, bool check_matrix) :
1931           _queue(queue), _heap(ShenandoahHeap::heap()), _map(map), _check_matrix(check_matrix) {};
1932   template <class T>
1933   void do_oop_work(T* p) {
1934     T o = oopDesc::load_heap_oop(p);
1935     if (!oopDesc::is_null(o)) {
1936       oop obj = oopDesc::decode_heap_oop_not_null(o);
1937       guarantee(check_obj_alignment(obj), "sanity");
1938 
1939       guarantee(!oopDesc::is_null(obj), "sanity");
1940       guarantee(_heap->is_in(obj), "sanity");
1941 
1942       oop forw = BrooksPointer::forwardee(obj);
1943       guarantee(!oopDesc::is_null(forw), "sanity");
1944       guarantee(_heap->is_in(forw), "sanity");
1945 
1946       guarantee(oopDesc::unsafe_equals(obj, forw), "should not be forwarded");
1947 
1948       if (_check_matrix) {
1949         size_t from_idx = _heap->heap_region_index_containing(p);
1950         size_t to_idx = _heap->heap_region_index_containing(obj);
1951         if (!_heap->connection_matrix()->is_connected(from_idx, to_idx)) {
1952           tty->print_cr("from-obj: ");
1953           _obj->print_on(tty);
1954           tty->print_cr("to-obj:");
1955           obj->print_on(tty);
1956           tty->print_cr("from-obj allocated after mark: %s", BOOL_TO_STR(_heap->allocated_after_complete_mark_start((HeapWord*) _obj)));
1957           tty->print_cr("to-obj allocated after mark: %s", BOOL_TO_STR(_heap->allocated_after_complete_mark_start((HeapWord*) obj)));
1958           tty->print_cr("from-obj marked: %s", BOOL_TO_STR(_heap->is_marked_complete(_obj)));
1959           tty->print_cr("to-obj marked: %s", BOOL_TO_STR(_heap->is_marked_complete(obj)));
1960           tty->print_cr("from-idx: " SIZE_FORMAT ", to-idx: " SIZE_FORMAT, from_idx, to_idx);
1961 
1962           oop fwd_from = BrooksPointer::forwardee(_obj);
1963           oop fwd_to = BrooksPointer::forwardee(obj);
1964           tty->print_cr("from-obj forwardee: " PTR_FORMAT, p2i(fwd_from));
1965           tty->print_cr("to-obj forwardee: " PTR_FORMAT, p2i(fwd_to));
1966           tty->print_cr("forward(from-obj) marked: %s", BOOL_TO_STR(_heap->is_marked_complete(fwd_from)));
1967           tty->print_cr("forward(to-obj) marked: %s", BOOL_TO_STR(_heap->is_marked_complete(fwd_to)));
1968           size_t fwd_from_idx = _heap->heap_region_index_containing(fwd_from);
1969           size_t fwd_to_idx = _heap->heap_region_index_containing(fwd_to);
1970           tty->print_cr("forward(from-idx): " SIZE_FORMAT ", forward(to-idx): " SIZE_FORMAT, fwd_from_idx, fwd_to_idx);
1971           tty->print_cr("forward(from) connected with forward(to)? %s", BOOL_TO_STR(_heap->connection_matrix()->is_connected(fwd_from_idx, fwd_to_idx)));
1972         }
1973         guarantee(oopDesc::unsafe_equals(ShenandoahBarrierSet::resolve_oop_static_not_null(obj), obj), "polizeilich verboten");
1974         guarantee(_heap->connection_matrix()->is_connected(from_idx, to_idx), "must be connected");
1975       }
1976 
1977       if (_map->parMark((HeapWord*) obj)) {
1978         _queue->push(SCMTask(obj));
1979       }
1980     }
1981   }
1982 
1983   void do_oop(oop* p) { do_oop_work(p); }
1984   void do_oop(narrowOop* p) { do_oop_work(p); }
1985   void set_obj(oop o) { _obj = o; }
1986 };
1987 
1988 void ShenandoahHeap::verify_heap_reachable_at_safepoint() {
1989   guarantee(SafepointSynchronize::is_at_safepoint(), "only when nothing else happens");
1990   guarantee(ShenandoahVerify || (UseShenandoahMatrix && VerifyShenandoahMatrix),
1991             "only when these are enabled, and bitmap is initialized in ShenandoahHeap::initialize");
1992 
1993   OrderAccess::fence();
1994   ensure_parsability(false);
1995 
1996   // Allocate temporary bitmap for storing marking wavefront:
1997   MemRegion mr = MemRegion(_verification_bit_map.startWord(), _verification_bit_map.endWord());
1998   _verification_bit_map.clear_range_large(mr);
1999 
2000   // Initialize a single queue
2001   SCMObjToScanQueue* q = new SCMObjToScanQueue();
2002   q->initialize();
2003 
2004   // Scan root set
2005   ClassLoaderDataGraph::clear_claimed_marks();
2006   ShenandoahRootProcessor rp(this, 1);
2007 
2008   {
2009     VerifyReachableHeapClosure cl(q, &_verification_bit_map, false);
2010     CLDToOopClosure cld_cl(&cl);
2011     CodeBlobToOopClosure code_cl(&cl, ! CodeBlobToOopClosure::FixRelocations);
2012     rp.process_all_roots(&cl, &cl, &cld_cl, &code_cl, 0);
2013   }
2014 
2015   // Finish the scan
2016   {
2017     VerifyReachableHeapClosure cl(q, &_verification_bit_map, UseShenandoahMatrix && VerifyShenandoahMatrix);
2018     SCMTask task;
2019     while ((q->pop_buffer(task) ||
2020             q->pop_local(task) ||
2021             q->pop_overflow(task))) {
2022       oop obj = task.obj();
2023       assert(!oopDesc::is_null(obj), "must not be null");
2024       cl.set_obj(obj);
2025       obj->oop_iterate(&cl);
2026     }
2027   }
2028 
2029   // Clean up!
2030   delete(q);
2031 }
2032 
2033 void ShenandoahHeap::stop_concurrent_marking() {
2034   assert(concurrent_mark_in_progress(), "How else could we get here?");
2035   if (! cancelled_concgc()) {
2036     // If we needed to update refs, and concurrent marking has been cancelled,
2037     // we need to finish updating references.
2038     set_need_update_refs(false);
2039     swap_mark_bitmaps();
2040   }
2041   set_concurrent_mark_in_progress(false);
2042 
2043   if (log_is_enabled(Trace, gc, region)) {
2044     ResourceMark rm;
2045     outputStream* out = Log(gc, region)::trace_stream();
2046     print_heap_regions(out);
2047   }
2048 
2049 }
2050 
2051 void ShenandoahHeap::set_concurrent_mark_in_progress(bool in_progress) {
2052   _concurrent_mark_in_progress = in_progress ? 1 : 0;
2053   JavaThread::satb_mark_queue_set().set_active_all_threads(in_progress, !in_progress);
2054 }
2055 
2056 void ShenandoahHeap::set_evacuation_in_progress_concurrently(bool in_progress) {
2057   // Note: it is important to first release the _evacuation_in_progress flag here,
2058   // so that Java threads can get out of oom_during_evacuation() and reach a safepoint,
2059   // in case a VM task is pending.
2060   set_evacuation_in_progress(in_progress);
2061   MutexLocker mu(Threads_lock);
2062   JavaThread::set_evacuation_in_progress_all_threads(in_progress);
2063 }
2064 
2065 void ShenandoahHeap::set_evacuation_in_progress_at_safepoint(bool in_progress) {
2066   assert(SafepointSynchronize::is_at_safepoint(), "Only call this at safepoint");
2067   set_evacuation_in_progress(in_progress);
2068   JavaThread::set_evacuation_in_progress_all_threads(in_progress);
2069 }
2070 
2071 void ShenandoahHeap::set_evacuation_in_progress(bool in_progress) {
2072   _evacuation_in_progress = in_progress ? 1 : 0;
2073   OrderAccess::fence();
2074 }
2075 
2076 void ShenandoahHeap::verify_copy(oop p,oop c){
2077     assert(! oopDesc::unsafe_equals(p, oopDesc::bs()->read_barrier(p)), "forwarded correctly");
2078     assert(oopDesc::unsafe_equals(oopDesc::bs()->read_barrier(p), c), "verify pointer is correct");
2079     if (p->klass() != c->klass()) {
2080       print_heap_regions();
2081     }
2082     assert(p->klass() == c->klass(), "verify class p-size: "INT32_FORMAT" c-size: "INT32_FORMAT, p->size(), c->size());
2083     assert(p->size() == c->size(), "verify size");
2084     // Object may have been locked between copy and verification
2085     //    assert(p->mark() == c->mark(), "verify mark");
2086     assert(oopDesc::unsafe_equals(c, oopDesc::bs()->read_barrier(c)), "verify only forwarded once");
2087   }
2088 
2089 void ShenandoahHeap::oom_during_evacuation() {
2090   log_develop_trace(gc)("Out of memory during evacuation, cancel evacuation, schedule full GC by thread %d",
2091                         Thread::current()->osthread()->thread_id());
2092 
2093   // We ran out of memory during evacuation. Cancel evacuation, and schedule a full-GC.
2094   collector_policy()->set_should_clear_all_soft_refs(true);
2095   concurrent_thread()->try_set_full_gc();
2096   cancel_concgc(_oom_evacuation);
2097 
2098   if ((! Thread::current()->is_GC_task_thread()) && (! Thread::current()->is_ConcurrentGC_thread())) {
2099     assert(! Threads_lock->owned_by_self()
2100            || SafepointSynchronize::is_at_safepoint(), "must not hold Threads_lock here");
2101     log_warning(gc)("OOM during evacuation. Let Java thread wait until evacuation finishes.");
2102     while (_evacuation_in_progress) { // wait.
2103       Thread::current()->_ParkEvent->park(1);
2104     }
2105   }
2106 
2107 }
2108 
2109 HeapWord* ShenandoahHeap::tlab_post_allocation_setup(HeapWord* obj) {
2110   // Initialize Brooks pointer for the next object
2111   HeapWord* result = obj + BrooksPointer::word_size();
2112   BrooksPointer::initialize(oop(result));
2113   return result;
2114 }
2115 
2116 uint ShenandoahHeap::oop_extra_words() {
2117   return BrooksPointer::word_size();
2118 }
2119 
2120 void ShenandoahHeap::grow_heap_by(size_t num_regions) {
2121   size_t base = _num_regions;
2122   ensure_new_regions(num_regions);
2123   for (size_t i = 0; i < num_regions; i++) {
2124     size_t new_region_index = i + base;
2125     HeapWord* start = _first_region_bottom + (ShenandoahHeapRegion::RegionSizeBytes / HeapWordSize) * new_region_index;
2126     ShenandoahHeapRegion* new_region = new ShenandoahHeapRegion(this, start, ShenandoahHeapRegion::RegionSizeBytes / HeapWordSize, new_region_index);
2127 
2128     if (log_is_enabled(Trace, gc, region)) {
2129       ResourceMark rm;
2130       outputStream* out = Log(gc, region)::trace_stream();
2131       out->print_cr("allocating new region at index: "SIZE_FORMAT, new_region_index);
2132       new_region->print_on(out);
2133     }
2134 
2135     assert(_ordered_regions->active_regions() == new_region->region_number(), "must match");
2136     _ordered_regions->add_region(new_region);
2137     _in_cset_fast_test_base[new_region_index] = false; // Not in cset
2138     _next_top_at_mark_starts_base[new_region_index] = new_region->bottom();
2139     _complete_top_at_mark_starts_base[new_region_index] = new_region->bottom();
2140 
2141     _free_regions->add_region(new_region);
2142   }
2143 }
2144 
2145 void ShenandoahHeap::ensure_new_regions(size_t new_regions) {
2146 
2147   size_t num_regions = _num_regions;
2148   size_t new_num_regions = num_regions + new_regions;
2149   assert(new_num_regions <= _max_regions, "we checked this earlier");
2150 
2151   size_t expand_size = new_regions * ShenandoahHeapRegion::RegionSizeBytes;
2152   log_trace(gc, region)("expanding storage by "SIZE_FORMAT_HEX" bytes, for "SIZE_FORMAT" new regions", expand_size, new_regions);
2153   bool success = _storage.expand_by(expand_size, ShenandoahAlwaysPreTouch);
2154   assert(success, "should always be able to expand by requested size");
2155 
2156   _num_regions = new_num_regions;
2157 
2158 }
2159 
2160 ShenandoahForwardedIsAliveClosure::ShenandoahForwardedIsAliveClosure() :
2161   _heap(ShenandoahHeap::heap_no_check()) {
2162 }
2163 
2164 void ShenandoahForwardedIsAliveClosure::init(ShenandoahHeap* heap) {
2165   _heap = heap;
2166 }
2167 
2168 bool ShenandoahForwardedIsAliveClosure::do_object_b(oop obj) {
2169 
2170   assert(_heap != NULL, "sanity");
2171   obj = ShenandoahBarrierSet::resolve_oop_static_not_null(obj);
2172 #ifdef ASSERT
2173   if (_heap->concurrent_mark_in_progress()) {
2174     assert(oopDesc::unsafe_equals(obj, ShenandoahBarrierSet::resolve_oop_static_not_null(obj)), "only query to-space");
2175   }
2176 #endif
2177   assert(!oopDesc::is_null(obj), "null");
2178   return _heap->is_marked_next(obj);
2179 }
2180 
2181 void ShenandoahHeap::ref_processing_init() {
2182   MemRegion mr = reserved_region();
2183 
2184   isAlive.init(ShenandoahHeap::heap());
2185   assert(_max_workers > 0, "Sanity");
2186 
2187   _ref_processor =
2188     new ReferenceProcessor(mr,    // span
2189                            ParallelRefProcEnabled,
2190                            // mt processing
2191                            _max_workers,
2192                            // degree of mt processing
2193                            true,
2194                            // mt discovery
2195                            _max_workers,
2196                            // degree of mt discovery
2197                            false,
2198                            // Reference discovery is not atomic
2199                            &isAlive);
2200 }
2201 
2202 #ifdef ASSERT
2203 void ShenandoahHeap::set_from_region_protection(bool protect) {
2204   for (uint i = 0; i < _num_regions; i++) {
2205     ShenandoahHeapRegion* region = _ordered_regions->get(i);
2206     if (region != NULL && in_collection_set(region)) {
2207       if (protect) {
2208         region->memProtectionOn();
2209       } else {
2210         region->memProtectionOff();
2211       }
2212     }
2213   }
2214 }
2215 #endif
2216 
2217 size_t ShenandoahHeap::num_regions() {
2218   return _num_regions;
2219 }
2220 
2221 size_t ShenandoahHeap::max_regions() {
2222   return _max_regions;
2223 }
2224 
2225 GCTracer* ShenandoahHeap::tracer() {
2226   return shenandoahPolicy()->tracer();
2227 }
2228 
2229 size_t ShenandoahHeap::tlab_used(Thread* thread) const {
2230   return _free_regions->used();
2231 }
2232 
2233 void ShenandoahHeap::cancel_concgc(GCCause::Cause cause) {
2234   if (try_cancel_concgc()) {
2235     log_info(gc)("Cancelling concurrent GC: %s", GCCause::to_string(cause));
2236     _shenandoah_policy->report_concgc_cancelled();
2237   }
2238 }
2239 
2240 void ShenandoahHeap::cancel_concgc(ShenandoahCancelCause cause) {
2241   if (try_cancel_concgc()) {
2242     log_info(gc)("Cancelling concurrent GC: %s", cancel_cause_to_string(cause));
2243     _shenandoah_policy->report_concgc_cancelled();
2244   }
2245 }
2246 
2247 const char* ShenandoahHeap::cancel_cause_to_string(ShenandoahCancelCause cause) {
2248   switch (cause) {
2249     case _oom_evacuation:
2250       return "Out of memory for evacuation";
2251     case _vm_stop:
2252       return "Stopping VM";
2253     default:
2254       return "Unknown";
2255   }
2256 }
2257 
2258 uint ShenandoahHeap::max_workers() {
2259   return _max_workers;
2260 }
2261 
2262 void ShenandoahHeap::stop() {
2263   // The shutdown sequence should be able to terminate when GC is running.
2264 
2265   // Step 1. Notify control thread that we are in shutdown.
2266   // Note that we cannot do that with stop(), because stop() is blocking and waits for the actual shutdown.
2267   // Doing stop() here would wait for the normal GC cycle to complete, never falling through to cancel below.
2268   _concurrent_gc_thread->prepare_for_graceful_shutdown();
2269 
2270   // Step 2. Notify GC workers that we are cancelling GC.
2271   cancel_concgc(_vm_stop);
2272 
2273   // Step 3. Wait until GC worker exits normally.
2274   _concurrent_gc_thread->stop();
2275 }
2276 
2277 void ShenandoahHeap::unlink_string_and_symbol_table(BoolObjectClosure* is_alive, bool process_strings, bool process_symbols) {
2278 
2279   StringSymbolTableUnlinkTask shenandoah_unlink_task(is_alive, process_strings, process_symbols);
2280   workers()->run_task(&shenandoah_unlink_task);
2281 
2282   //  if (G1StringDedup::is_enabled()) {
2283   //    G1StringDedup::unlink(is_alive);
2284   //  }
2285 }
2286 
2287 void ShenandoahHeap::set_need_update_refs(bool need_update_refs) {
2288   _need_update_refs = need_update_refs;
2289 }
2290 
2291 //fixme this should be in heapregionset
2292 ShenandoahHeapRegion* ShenandoahHeap::next_compaction_region(const ShenandoahHeapRegion* r) {
2293   size_t region_idx = r->region_number() + 1;
2294   ShenandoahHeapRegion* next = _ordered_regions->get(region_idx);
2295   guarantee(next->region_number() == region_idx, "region number must match");
2296   while (next->is_humongous()) {
2297     region_idx = next->region_number() + 1;
2298     next = _ordered_regions->get(region_idx);
2299     guarantee(next->region_number() == region_idx, "region number must match");
2300   }
2301   return next;
2302 }
2303 
2304 void ShenandoahHeap::set_region_in_collection_set(size_t region_index, bool b) {
2305   _in_cset_fast_test_base[region_index] = b;
2306 }
2307 
2308 ShenandoahMonitoringSupport* ShenandoahHeap::monitoring_support() {
2309   return _monitoring_support;
2310 }
2311 
2312 CMBitMap* ShenandoahHeap::complete_mark_bit_map() {
2313   return _complete_mark_bit_map;
2314 }
2315 
2316 CMBitMap* ShenandoahHeap::next_mark_bit_map() {
2317   return _next_mark_bit_map;
2318 }
2319 
2320 void ShenandoahHeap::add_free_region(ShenandoahHeapRegion* r) {
2321   _free_regions->add_region(r);
2322 }
2323 
2324 void ShenandoahHeap::clear_free_regions() {
2325   _free_regions->clear();
2326 }
2327 
2328 address ShenandoahHeap::in_cset_fast_test_addr() {
2329   return (address) (ShenandoahHeap::heap()->_in_cset_fast_test);
2330 }
2331 
2332 address ShenandoahHeap::cancelled_concgc_addr() {
2333   return (address) &(ShenandoahHeap::heap()->_cancelled_concgc);
2334 }
2335 
2336 void ShenandoahHeap::clear_cset_fast_test() {
2337   assert(_in_cset_fast_test_base != NULL, "sanity");
2338   memset(_in_cset_fast_test_base, false,
2339          _in_cset_fast_test_length * sizeof(bool));
2340 }
2341 
2342 size_t ShenandoahHeap::conservative_max_heap_alignment() {
2343   return ShenandoahMaxRegionSize;
2344 }
2345 
2346 size_t ShenandoahHeap::bytes_allocated_since_cm() {
2347   return _bytes_allocated_since_cm;
2348 }
2349 
2350 void ShenandoahHeap::set_bytes_allocated_since_cm(size_t bytes) {
2351   _bytes_allocated_since_cm = bytes;
2352 }
2353 
2354 size_t ShenandoahHeap::max_allocated_gc() {
2355   return _max_allocated_gc;
2356 }
2357 
2358 void ShenandoahHeap::set_next_top_at_mark_start(HeapWord* region_base, HeapWord* addr) {
2359   uintx index = ((uintx) region_base) >> ShenandoahHeapRegion::RegionSizeShift;
2360   _next_top_at_mark_starts[index] = addr;
2361 }
2362 
2363 HeapWord* ShenandoahHeap::next_top_at_mark_start(HeapWord* region_base) {
2364   uintx index = ((uintx) region_base) >> ShenandoahHeapRegion::RegionSizeShift;
2365   return _next_top_at_mark_starts[index];
2366 }
2367 
2368 void ShenandoahHeap::set_complete_top_at_mark_start(HeapWord* region_base, HeapWord* addr) {
2369   uintx index = ((uintx) region_base) >> ShenandoahHeapRegion::RegionSizeShift;
2370   _complete_top_at_mark_starts[index] = addr;
2371 }
2372 
2373 HeapWord* ShenandoahHeap::complete_top_at_mark_start(HeapWord* region_base) {
2374   uintx index = ((uintx) region_base) >> ShenandoahHeapRegion::RegionSizeShift;
2375   return _complete_top_at_mark_starts[index];
2376 }
2377 
2378 void ShenandoahHeap::set_full_gc_in_progress(bool in_progress) {
2379   _full_gc_in_progress = in_progress;
2380 }
2381 
2382 bool ShenandoahHeap::is_full_gc_in_progress() const {
2383   return _full_gc_in_progress;
2384 }
2385 
2386 class NMethodOopInitializer : public OopClosure {
2387 private:
2388   ShenandoahHeap* _heap;
2389 public:
2390   NMethodOopInitializer() : _heap(ShenandoahHeap::heap()) {
2391   }
2392 
2393 private:
2394   template <class T>
2395   inline void do_oop_work(T* p) {
2396     T o = oopDesc::load_heap_oop(p);
2397     if (! oopDesc::is_null(o)) {
2398       oop obj1 = oopDesc::decode_heap_oop_not_null(o);
2399       oop obj2 = oopDesc::bs()->write_barrier(obj1);
2400       if (! oopDesc::unsafe_equals(obj1, obj2)) {
2401         oopDesc::encode_store_heap_oop(p, obj2);
2402       }
2403     }
2404   }
2405 
2406 public:
2407   void do_oop(oop* o) {
2408     do_oop_work(o);
2409   }
2410   void do_oop(narrowOop* o) {
2411     do_oop_work(o);
2412   }
2413 };
2414 
2415 void ShenandoahHeap::register_nmethod(nmethod* nm) {
2416   NMethodOopInitializer init;
2417   nm->oops_do(&init);
2418   nm->fix_oop_relocations();
2419 }
2420 
2421 void ShenandoahHeap::unregister_nmethod(nmethod* nm) {
2422 }
2423 
2424 void ShenandoahHeap::pin_object(oop o) {
2425   heap_region_containing(o)->pin();
2426 }
2427 
2428 void ShenandoahHeap::unpin_object(oop o) {
2429   heap_region_containing(o)->unpin();
2430 }
2431 
2432 
2433 GCTimer* ShenandoahHeap::gc_timer() const {
2434   return _gc_timer;
2435 }
2436 
2437 class RecordAllRefsOopClosure: public ExtendedOopClosure {
2438 private:
2439   size_t _x;
2440   int *_matrix;
2441   size_t _num_regions;
2442   oop _p;
2443 
2444 public:
2445   RecordAllRefsOopClosure(int *matrix, size_t x, size_t num_regions, oop p) :
2446     _matrix(matrix), _x(x), _num_regions(num_regions), _p(p) {}
2447 
2448   template <class T>
2449   void do_oop_work(T* p) {
2450     oop o = oopDesc::load_decode_heap_oop(p);
2451     if (o != NULL) {
2452       if (ShenandoahHeap::heap()->is_in(o) && o->is_oop() ) {
2453         size_t y = ShenandoahHeap::heap()->heap_region_containing(o)->region_number();
2454         _matrix[_x * _num_regions + y]++;
2455       }
2456     }
2457   }
2458   void do_oop(oop* p) {
2459     do_oop_work(p);
2460   }
2461 
2462   void do_oop(narrowOop* p) {
2463     do_oop_work(p);
2464   }
2465 
2466 };
2467 
2468 class RecordAllRefsObjectClosure : public ObjectClosure {
2469   int *_matrix;
2470   size_t _num_regions;
2471 
2472 public:
2473   RecordAllRefsObjectClosure(int *matrix, size_t num_regions) :
2474     _matrix(matrix), _num_regions(num_regions) {}
2475 
2476   void do_object(oop p) {
2477     if (ShenandoahHeap::heap()->is_in(p) && ShenandoahHeap::heap()->is_marked_next(p)  && p->is_oop()) {
2478       size_t x = ShenandoahHeap::heap()->heap_region_containing(p)->region_number();
2479       RecordAllRefsOopClosure cl(_matrix, x, _num_regions, p);
2480       p->oop_iterate(&cl);
2481     }
2482   }
2483 };
2484 void ShenandoahHeap::calculate_matrix(int* connections) {
2485   log_develop_trace(gc)("calculating matrix");
2486   ensure_parsability(false);
2487   size_t num = num_regions();
2488 
2489   for (size_t i = 0; i < num; i++) {
2490     for (size_t j = 0; j < num; j++) {
2491       connections[i * num + j] = 0;
2492     }
2493   }
2494 
2495   RecordAllRefsOopClosure cl(connections, 0, num, NULL);
2496   roots_iterate(&cl);
2497 
2498   RecordAllRefsObjectClosure cl2(connections, num);
2499   object_iterate(&cl2);
2500 
2501 }
2502 
2503 void ShenandoahHeap::print_matrix(int* connections) {
2504   size_t num = num_regions();
2505   int cs_regions = 0;
2506   int referenced = 0;
2507 
2508   for (size_t i = 0; i < num; i++) {
2509     size_t liveData = ShenandoahHeap::heap()->regions()->get(i)->get_live_data_bytes();
2510 
2511     int numReferencedRegions = 0;
2512     int numReferencedByRegions = 0;
2513 
2514     for (size_t j = 0; j < num; j++) {
2515       if (connections[i * num + j] > 0)
2516         numReferencedRegions++;
2517 
2518       if (connections [j * num + i] > 0)
2519         numReferencedByRegions++;
2520 
2521       cs_regions++;
2522       referenced += numReferencedByRegions;
2523     }
2524 
2525     if (ShenandoahHeap::heap()->regions()->get(i)->has_live()) {
2526       tty->print("Region " SIZE_FORMAT " is referenced by %d regions {", i, numReferencedByRegions);
2527       int col_count = 0;
2528       for (size_t j = 0; j < num; j++) {
2529         int foo = connections[j * num + i];
2530         if (foo > 0) {
2531           col_count++;
2532           if ((col_count % 10) == 0)
2533             tty->print("\n");
2534           tty->print("" SIZE_FORMAT "(%d), ", j, foo);
2535         }
2536       }
2537       tty->print("} \n");
2538     }
2539   }
2540 
2541   double avg = (double)referenced / (double) cs_regions;
2542   tty->print("Average Number of regions scanned / region = %lf\n", avg);
2543 }
2544 
2545 class ShenandoahCountGarbageClosure : public ShenandoahHeapRegionClosure {
2546 private:
2547   size_t _garbage;
2548 public:
2549   ShenandoahCountGarbageClosure() : _garbage(0) {
2550   }
2551 
2552   bool doHeapRegion(ShenandoahHeapRegion* r) {
2553     if (! r->is_humongous() && ! r->is_pinned() && ! r->in_collection_set()) {
2554       _garbage += r->garbage();
2555     }
2556     return false;
2557   }
2558 
2559   size_t garbage() {
2560     return _garbage;
2561   }
2562 };
2563 
2564 size_t ShenandoahHeap::garbage() {
2565   ShenandoahCountGarbageClosure cl;
2566   heap_region_iterate(&cl);
2567   return cl.garbage();
2568 }
2569 
2570 ShenandoahConnectionMatrix* ShenandoahHeap::connection_matrix() {
2571   return _connection_matrix;
2572 }
2573 
2574 ShenandoahPartialGC* ShenandoahHeap::partial_gc() {
2575   return _partial_gc;
2576 }
2577 
2578 void ShenandoahHeap::do_partial_collection() {
2579   {
2580     ShenandoahHeapLock lock(this);
2581     partial_gc()->prepare();
2582   }
2583   partial_gc()->do_partial_collection();
2584 }
2585 
2586 #ifdef ASSERT
2587 void ShenandoahHeap::assert_heaplock_owned_by_current_thread() {
2588   assert(_heap_lock == locked, "must be locked");
2589   assert(_heap_lock_owner == Thread::current(), "must be owned by current thread");
2590 }
2591 
2592 void ShenandoahHeap::assert_heaplock_or_safepoint() {
2593   Thread* thr = Thread::current();
2594   assert((_heap_lock == locked && _heap_lock_owner == thr) ||
2595          (SafepointSynchronize::is_at_safepoint() && thr->is_VM_thread()),
2596   "must own heap lock or by VM thread at safepoint");
2597 }
2598 
2599 #endif