1 /*
   2  * Copyright (c) 2013, 2019, Red Hat, Inc. All rights reserved.
   3  *
   4  * This code is free software; you can redistribute it and/or modify it
   5  * under the terms of the GNU General Public License version 2 only, as
   6  * published by the Free Software Foundation.
   7  *
   8  * This code is distributed in the hope that it will be useful, but WITHOUT
   9  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  10  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  11  * version 2 for more details (a copy is included in the LICENSE file that
  12  * accompanied this code).
  13  *
  14  * You should have received a copy of the GNU General Public License version
  15  * 2 along with this work; if not, write to the Free Software Foundation,
  16  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  17  *
  18  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  19  * or visit www.oracle.com if you need additional information or have any
  20  * questions.
  21  *
  22  */
  23 
  24 #include "precompiled.hpp"
  25 #include "memory/allocation.hpp"
  26 #include "gc/shenandoah/shenandoahHeapRegionSet.inline.hpp"
  27 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
  28 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
  29 #include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"
  30 #include "gc/shenandoah/shenandoahTraversalGC.hpp"
  31 #include "gc/shared/space.inline.hpp"
  32 #include "jfr/jfrEvents.hpp"
  33 #include "memory/iterator.inline.hpp"
  34 #include "memory/resourceArea.hpp"
  35 #include "memory/universe.hpp"
  36 #include "oops/oop.inline.hpp"
  37 #include "runtime/java.hpp"
  38 #include "runtime/mutexLocker.hpp"
  39 #include "runtime/os.hpp"
  40 #include "runtime/safepoint.hpp"
  41 
  42 size_t ShenandoahHeapRegion::RegionCount = 0;
  43 size_t ShenandoahHeapRegion::RegionSizeBytes = 0;
  44 size_t ShenandoahHeapRegion::RegionSizeWords = 0;
  45 size_t ShenandoahHeapRegion::RegionSizeBytesShift = 0;
  46 size_t ShenandoahHeapRegion::RegionSizeWordsShift = 0;
  47 size_t ShenandoahHeapRegion::RegionSizeBytesMask = 0;
  48 size_t ShenandoahHeapRegion::RegionSizeWordsMask = 0;
  49 size_t ShenandoahHeapRegion::HumongousThresholdBytes = 0;
  50 size_t ShenandoahHeapRegion::HumongousThresholdWords = 0;
  51 size_t ShenandoahHeapRegion::MaxTLABSizeBytes = 0;
  52 size_t ShenandoahHeapRegion::MaxTLABSizeWords = 0;
  53 
  54 ShenandoahHeapRegion::PaddedAllocSeqNum ShenandoahHeapRegion::_alloc_seq_num;
  55 
  56 ShenandoahHeapRegion::ShenandoahHeapRegion(ShenandoahHeap* heap, HeapWord* start,
  57                                            size_t size_words, size_t index, bool committed) :
  58   _heap(heap),
  59   _reserved(MemRegion(start, size_words)),
  60   _region_number(index),
  61   _new_top(NULL),
  62   _empty_time(os::elapsedTime()),
  63   _state(committed ? _empty_committed : _empty_uncommitted),
  64   _tlab_allocs(0),
  65   _gclab_allocs(0),
  66   _shared_allocs(0),
  67   _seqnum_first_alloc_mutator(0),
  68   _seqnum_first_alloc_gc(0),
  69   _seqnum_last_alloc_mutator(0),
  70   _seqnum_last_alloc_gc(0),
  71   _live_data(0),
  72   _critical_pins(0) {
  73 
  74   ContiguousSpace::initialize(_reserved, true, committed);
  75 }
  76 
  77 size_t ShenandoahHeapRegion::region_number() const {
  78   return _region_number;
  79 }
  80 
  81 void ShenandoahHeapRegion::report_illegal_transition(const char *method) {
  82   ResourceMark rm;
  83   stringStream ss;
  84   ss.print("Illegal region state transition from \"%s\", at %s\n  ", region_state_to_string(_state), method);
  85   print_on(&ss);
  86   fatal("%s", ss.as_string());
  87 }
  88 
  89 void ShenandoahHeapRegion::make_regular_allocation() {
  90   _heap->assert_heaplock_owned_by_current_thread();
  91 
  92   switch (_state) {
  93     case _empty_uncommitted:
  94       do_commit();
  95     case _empty_committed:
  96       set_state(_regular);
  97     case _regular:
  98     case _pinned:
  99       return;
 100     default:
 101       report_illegal_transition("regular allocation");
 102   }
 103 }
 104 
 105 void ShenandoahHeapRegion::make_regular_bypass() {
 106   _heap->assert_heaplock_owned_by_current_thread();
 107   assert (_heap->is_full_gc_in_progress() || _heap->is_degenerated_gc_in_progress(),
 108           "only for full or degen GC");
 109 
 110   switch (_state) {
 111     case _empty_uncommitted:
 112       do_commit();
 113     case _empty_committed:
 114     case _cset:
 115     case _humongous_start:
 116     case _humongous_cont:
 117       set_state(_regular);
 118       return;
 119     case _pinned_cset:
 120       set_state(_pinned);
 121       return;
 122     case _regular:
 123     case _pinned:
 124       return;
 125     default:
 126       report_illegal_transition("regular bypass");
 127   }
 128 }
 129 
 130 void ShenandoahHeapRegion::make_humongous_start() {
 131   _heap->assert_heaplock_owned_by_current_thread();
 132   switch (_state) {
 133     case _empty_uncommitted:
 134       do_commit();
 135     case _empty_committed:
 136       set_state(_humongous_start);
 137       return;
 138     default:
 139       report_illegal_transition("humongous start allocation");
 140   }
 141 }
 142 
 143 void ShenandoahHeapRegion::make_humongous_start_bypass() {
 144   _heap->assert_heaplock_owned_by_current_thread();
 145   assert (_heap->is_full_gc_in_progress(), "only for full GC");
 146 
 147   switch (_state) {
 148     case _empty_committed:
 149     case _regular:
 150     case _humongous_start:
 151     case _humongous_cont:
 152       set_state(_humongous_start);
 153       return;
 154     default:
 155       report_illegal_transition("humongous start bypass");
 156   }
 157 }
 158 
 159 void ShenandoahHeapRegion::make_humongous_cont() {
 160   _heap->assert_heaplock_owned_by_current_thread();
 161   switch (_state) {
 162     case _empty_uncommitted:
 163       do_commit();
 164     case _empty_committed:
 165      set_state(_humongous_cont);
 166       return;
 167     default:
 168       report_illegal_transition("humongous continuation allocation");
 169   }
 170 }
 171 
 172 void ShenandoahHeapRegion::make_humongous_cont_bypass() {
 173   _heap->assert_heaplock_owned_by_current_thread();
 174   assert (_heap->is_full_gc_in_progress(), "only for full GC");
 175 
 176   switch (_state) {
 177     case _empty_committed:
 178     case _regular:
 179     case _humongous_start:
 180     case _humongous_cont:
 181       set_state(_humongous_cont);
 182       return;
 183     default:
 184       report_illegal_transition("humongous continuation bypass");
 185   }
 186 }
 187 
 188 void ShenandoahHeapRegion::make_pinned() {
 189   _heap->assert_heaplock_owned_by_current_thread();
 190   assert(pin_count() > 0, "Should have pins: " SIZE_FORMAT, pin_count());
 191 
 192   switch (_state) {
 193     case _regular:
 194       set_state(_pinned);
 195     case _pinned_cset:
 196     case _pinned:
 197       return;
 198     case _humongous_start:
 199       set_state(_pinned_humongous_start);
 200     case _pinned_humongous_start:
 201       return;
 202     case _cset:
 203       _state = _pinned_cset;
 204       return;
 205     default:
 206       report_illegal_transition("pinning");
 207   }
 208 }
 209 
 210 void ShenandoahHeapRegion::make_unpinned() {
 211   _heap->assert_heaplock_owned_by_current_thread();
 212   assert(pin_count() == 0, "Should not have pins: " SIZE_FORMAT, pin_count());
 213 
 214   switch (_state) {
 215     case _pinned:
 216       set_state(_regular);
 217       return;
 218     case _regular:
 219     case _humongous_start:
 220       return;
 221     case _pinned_cset:
 222       set_state(_cset);
 223       return;
 224     case _pinned_humongous_start:
 225       set_state(_humongous_start);
 226       return;
 227     default:
 228       report_illegal_transition("unpinning");
 229   }
 230 }
 231 
 232 void ShenandoahHeapRegion::make_cset() {
 233   _heap->assert_heaplock_owned_by_current_thread();
 234   switch (_state) {
 235     case _regular:
 236       set_state(_cset);
 237     case _cset:
 238       return;
 239     default:
 240       report_illegal_transition("cset");
 241   }
 242 }
 243 
 244 void ShenandoahHeapRegion::make_trash() {
 245   _heap->assert_heaplock_owned_by_current_thread();
 246   switch (_state) {
 247     case _cset:
 248       // Reclaiming cset regions
 249     case _humongous_start:
 250     case _humongous_cont:
 251       // Reclaiming humongous regions
 252     case _regular:
 253       // Immediate region reclaim
 254       set_state(_trash);
 255       return;
 256     default:
 257       report_illegal_transition("trashing");
 258   }
 259 }
 260 
 261 void ShenandoahHeapRegion::make_trash_immediate() {
 262   make_trash();
 263 
 264   // On this path, we know there are no marked objects in the region,
 265   // tell marking context about it to bypass bitmap resets.
 266   _heap->complete_marking_context()->reset_top_bitmap(this);
 267 }
 268 
 269 void ShenandoahHeapRegion::make_empty() {
 270   _heap->assert_heaplock_owned_by_current_thread();
 271   switch (_state) {
 272     case _trash:
 273       set_state(_empty_committed);
 274       _empty_time = os::elapsedTime();
 275       return;
 276     default:
 277       report_illegal_transition("emptying");
 278   }
 279 }
 280 
 281 void ShenandoahHeapRegion::make_uncommitted() {
 282   _heap->assert_heaplock_owned_by_current_thread();
 283   switch (_state) {
 284     case _empty_committed:
 285       do_uncommit();
 286       set_state(_empty_uncommitted);
 287       return;
 288     default:
 289       report_illegal_transition("uncommiting");
 290   }
 291 }
 292 
 293 void ShenandoahHeapRegion::make_committed_bypass() {
 294   _heap->assert_heaplock_owned_by_current_thread();
 295   assert (_heap->is_full_gc_in_progress(), "only for full GC");
 296 
 297   switch (_state) {
 298     case _empty_uncommitted:
 299       do_commit();
 300       set_state(_empty_committed);
 301       return;
 302     default:
 303       report_illegal_transition("commit bypass");
 304   }
 305 }
 306 
 307 void ShenandoahHeapRegion::clear_live_data() {
 308   Atomic::release_store_fence(&_live_data, (size_t)0);
 309 }
 310 
 311 void ShenandoahHeapRegion::reset_alloc_metadata() {
 312   _tlab_allocs = 0;
 313   _gclab_allocs = 0;
 314   _shared_allocs = 0;
 315   _seqnum_first_alloc_mutator = 0;
 316   _seqnum_last_alloc_mutator = 0;
 317   _seqnum_first_alloc_gc = 0;
 318   _seqnum_last_alloc_gc = 0;
 319 }
 320 
 321 void ShenandoahHeapRegion::reset_alloc_metadata_to_shared() {
 322   if (used() > 0) {
 323     _tlab_allocs = 0;
 324     _gclab_allocs = 0;
 325     _shared_allocs = used() >> LogHeapWordSize;
 326     uint64_t next = _alloc_seq_num.value++;
 327     _seqnum_first_alloc_mutator = next;
 328     _seqnum_last_alloc_mutator = next;
 329     _seqnum_first_alloc_gc = 0;
 330     _seqnum_last_alloc_gc = 0;
 331   } else {
 332     reset_alloc_metadata();
 333   }
 334 }
 335 
 336 size_t ShenandoahHeapRegion::get_shared_allocs() const {
 337   return _shared_allocs * HeapWordSize;
 338 }
 339 
 340 size_t ShenandoahHeapRegion::get_tlab_allocs() const {
 341   return _tlab_allocs * HeapWordSize;
 342 }
 343 
 344 size_t ShenandoahHeapRegion::get_gclab_allocs() const {
 345   return _gclab_allocs * HeapWordSize;
 346 }
 347 
 348 void ShenandoahHeapRegion::set_live_data(size_t s) {
 349   assert(Thread::current()->is_VM_thread(), "by VM thread");
 350   _live_data = (s >> LogHeapWordSize);
 351 }
 352 
 353 size_t ShenandoahHeapRegion::get_live_data_words() const {
 354   return Atomic::load_acquire(&_live_data);
 355 }
 356 
 357 size_t ShenandoahHeapRegion::get_live_data_bytes() const {
 358   return get_live_data_words() * HeapWordSize;
 359 }
 360 
 361 bool ShenandoahHeapRegion::has_live() const {
 362   return get_live_data_words() != 0;
 363 }
 364 
 365 size_t ShenandoahHeapRegion::garbage() const {
 366   assert(used() >= get_live_data_bytes(), "Live Data must be a subset of used() live: " SIZE_FORMAT " used: " SIZE_FORMAT,
 367          get_live_data_bytes(), used());
 368 
 369   size_t result = used() - get_live_data_bytes();
 370   return result;
 371 }
 372 
 373 void ShenandoahHeapRegion::print_on(outputStream* st) const {
 374   st->print("|");
 375   st->print(SIZE_FORMAT_W(5), this->_region_number);
 376 
 377   switch (_state) {
 378     case _empty_uncommitted:
 379       st->print("|EU ");
 380       break;
 381     case _empty_committed:
 382       st->print("|EC ");
 383       break;
 384     case _regular:
 385       st->print("|R  ");
 386       break;
 387     case _humongous_start:
 388       st->print("|H  ");
 389       break;
 390     case _pinned_humongous_start:
 391       st->print("|HP ");
 392       break;
 393     case _humongous_cont:
 394       st->print("|HC ");
 395       break;
 396     case _cset:
 397       st->print("|CS ");
 398       break;
 399     case _trash:
 400       st->print("|T  ");
 401       break;
 402     case _pinned:
 403       st->print("|P  ");
 404       break;
 405     case _pinned_cset:
 406       st->print("|CSP");
 407       break;
 408     default:
 409       ShouldNotReachHere();
 410   }
 411   st->print("|BTE " INTPTR_FORMAT_W(12) ", " INTPTR_FORMAT_W(12) ", " INTPTR_FORMAT_W(12),
 412             p2i(bottom()), p2i(top()), p2i(end()));
 413   st->print("|TAMS " INTPTR_FORMAT_W(12),
 414             p2i(_heap->marking_context()->top_at_mark_start(const_cast<ShenandoahHeapRegion*>(this))));
 415   st->print("|U " SIZE_FORMAT_W(5) "%1s", byte_size_in_proper_unit(used()),                proper_unit_for_byte_size(used()));
 416   st->print("|T " SIZE_FORMAT_W(5) "%1s", byte_size_in_proper_unit(get_tlab_allocs()),     proper_unit_for_byte_size(get_tlab_allocs()));
 417   st->print("|G " SIZE_FORMAT_W(5) "%1s", byte_size_in_proper_unit(get_gclab_allocs()),    proper_unit_for_byte_size(get_gclab_allocs()));
 418   st->print("|S " SIZE_FORMAT_W(5) "%1s", byte_size_in_proper_unit(get_shared_allocs()),   proper_unit_for_byte_size(get_shared_allocs()));
 419   st->print("|L " SIZE_FORMAT_W(5) "%1s", byte_size_in_proper_unit(get_live_data_bytes()), proper_unit_for_byte_size(get_live_data_bytes()));
 420   st->print("|CP " SIZE_FORMAT_W(3), pin_count());
 421   st->print("|SN " UINT64_FORMAT_X_W(12) ", " UINT64_FORMAT_X_W(8) ", " UINT64_FORMAT_X_W(8) ", " UINT64_FORMAT_X_W(8),
 422             seqnum_first_alloc_mutator(), seqnum_last_alloc_mutator(),
 423             seqnum_first_alloc_gc(), seqnum_last_alloc_gc());
 424   st->cr();
 425 }
 426 
 427 void ShenandoahHeapRegion::oop_iterate(OopIterateClosure* blk) {
 428   if (!is_active()) return;
 429   if (is_humongous()) {
 430     oop_iterate_humongous(blk);
 431   } else {
 432     oop_iterate_objects(blk);
 433   }
 434 }
 435 
 436 void ShenandoahHeapRegion::oop_iterate_objects(OopIterateClosure* blk) {
 437   assert(! is_humongous(), "no humongous region here");
 438   HeapWord* obj_addr = bottom();
 439   HeapWord* t = top();
 440   // Could call objects iterate, but this is easier.
 441   while (obj_addr < t) {
 442     oop obj = oop(obj_addr);
 443     obj_addr += obj->oop_iterate_size(blk);
 444   }
 445 }
 446 
 447 void ShenandoahHeapRegion::oop_iterate_humongous(OopIterateClosure* blk) {
 448   assert(is_humongous(), "only humongous region here");
 449   // Find head.
 450   ShenandoahHeapRegion* r = humongous_start_region();
 451   assert(r->is_humongous_start(), "need humongous head here");
 452   oop obj = oop(r->bottom());
 453   obj->oop_iterate(blk, MemRegion(bottom(), top()));
 454 }
 455 
 456 ShenandoahHeapRegion* ShenandoahHeapRegion::humongous_start_region() const {
 457   assert(is_humongous(), "Must be a part of the humongous region");
 458   size_t reg_num = region_number();
 459   ShenandoahHeapRegion* r = const_cast<ShenandoahHeapRegion*>(this);
 460   while (!r->is_humongous_start()) {
 461     assert(reg_num > 0, "Sanity");
 462     reg_num --;
 463     r = _heap->get_region(reg_num);
 464     assert(r->is_humongous(), "Must be a part of the humongous region");
 465   }
 466   assert(r->is_humongous_start(), "Must be");
 467   return r;
 468 }
 469 
 470 void ShenandoahHeapRegion::recycle() {
 471   ContiguousSpace::clear(false);
 472   if (ZapUnusedHeapArea) {
 473     ContiguousSpace::mangle_unused_area_complete();
 474   }
 475   clear_live_data();
 476 
 477   reset_alloc_metadata();
 478 
 479   _heap->marking_context()->reset_top_at_mark_start(this);
 480 
 481   make_empty();
 482 }
 483 
 484 HeapWord* ShenandoahHeapRegion::block_start_const(const void* p) const {
 485   assert(MemRegion(bottom(), end()).contains(p),
 486          "p (" PTR_FORMAT ") not in space [" PTR_FORMAT ", " PTR_FORMAT ")",
 487          p2i(p), p2i(bottom()), p2i(end()));
 488   if (p >= top()) {
 489     return top();
 490   } else {
 491     HeapWord* last = bottom();
 492     HeapWord* cur = last;
 493     while (cur <= p) {
 494       last = cur;
 495       cur += oop(cur)->size();
 496     }
 497     shenandoah_assert_correct(NULL, oop(last));
 498     return last;
 499   }
 500 }
 501 
 502 void ShenandoahHeapRegion::setup_sizes(size_t max_heap_size) {
 503   // Absolute minimums we should not ever break.
 504   static const size_t MIN_REGION_SIZE = 256*K;
 505 
 506   if (FLAG_IS_DEFAULT(ShenandoahMinRegionSize)) {
 507     FLAG_SET_DEFAULT(ShenandoahMinRegionSize, MIN_REGION_SIZE);
 508   }
 509 
 510   size_t region_size;
 511   if (FLAG_IS_DEFAULT(ShenandoahHeapRegionSize)) {
 512     if (ShenandoahMinRegionSize > max_heap_size / MIN_NUM_REGIONS) {
 513       err_msg message("Max heap size (" SIZE_FORMAT "%s) is too low to afford the minimum number "
 514                       "of regions (" SIZE_FORMAT ") of minimum region size (" SIZE_FORMAT "%s).",
 515                       byte_size_in_proper_unit(max_heap_size), proper_unit_for_byte_size(max_heap_size),
 516                       MIN_NUM_REGIONS,
 517                       byte_size_in_proper_unit(ShenandoahMinRegionSize), proper_unit_for_byte_size(ShenandoahMinRegionSize));
 518       vm_exit_during_initialization("Invalid -XX:ShenandoahMinRegionSize option", message);
 519     }
 520     if (ShenandoahMinRegionSize < MIN_REGION_SIZE) {
 521       err_msg message("" SIZE_FORMAT "%s should not be lower than minimum region size (" SIZE_FORMAT "%s).",
 522                       byte_size_in_proper_unit(ShenandoahMinRegionSize), proper_unit_for_byte_size(ShenandoahMinRegionSize),
 523                       byte_size_in_proper_unit(MIN_REGION_SIZE),         proper_unit_for_byte_size(MIN_REGION_SIZE));
 524       vm_exit_during_initialization("Invalid -XX:ShenandoahMinRegionSize option", message);
 525     }
 526     if (ShenandoahMinRegionSize < MinTLABSize) {
 527       err_msg message("" SIZE_FORMAT "%s should not be lower than TLAB size size (" SIZE_FORMAT "%s).",
 528                       byte_size_in_proper_unit(ShenandoahMinRegionSize), proper_unit_for_byte_size(ShenandoahMinRegionSize),
 529                       byte_size_in_proper_unit(MinTLABSize),             proper_unit_for_byte_size(MinTLABSize));
 530       vm_exit_during_initialization("Invalid -XX:ShenandoahMinRegionSize option", message);
 531     }
 532     if (ShenandoahMaxRegionSize < MIN_REGION_SIZE) {
 533       err_msg message("" SIZE_FORMAT "%s should not be lower than min region size (" SIZE_FORMAT "%s).",
 534                       byte_size_in_proper_unit(ShenandoahMaxRegionSize), proper_unit_for_byte_size(ShenandoahMaxRegionSize),
 535                       byte_size_in_proper_unit(MIN_REGION_SIZE),         proper_unit_for_byte_size(MIN_REGION_SIZE));
 536       vm_exit_during_initialization("Invalid -XX:ShenandoahMaxRegionSize option", message);
 537     }
 538     if (ShenandoahMinRegionSize > ShenandoahMaxRegionSize) {
 539       err_msg message("Minimum (" SIZE_FORMAT "%s) should be larger than maximum (" SIZE_FORMAT "%s).",
 540                       byte_size_in_proper_unit(ShenandoahMinRegionSize), proper_unit_for_byte_size(ShenandoahMinRegionSize),
 541                       byte_size_in_proper_unit(ShenandoahMaxRegionSize), proper_unit_for_byte_size(ShenandoahMaxRegionSize));
 542       vm_exit_during_initialization("Invalid -XX:ShenandoahMinRegionSize or -XX:ShenandoahMaxRegionSize", message);
 543     }
 544 
 545     // We rapidly expand to max_heap_size in most scenarios, so that is the measure
 546     // for usual heap sizes. Do not depend on initial_heap_size here.
 547     region_size = max_heap_size / ShenandoahTargetNumRegions;
 548 
 549     // Now make sure that we don't go over or under our limits.
 550     region_size = MAX2(ShenandoahMinRegionSize, region_size);
 551     region_size = MIN2(ShenandoahMaxRegionSize, region_size);
 552 
 553   } else {
 554     if (ShenandoahHeapRegionSize > max_heap_size / MIN_NUM_REGIONS) {
 555       err_msg message("Max heap size (" SIZE_FORMAT "%s) is too low to afford the minimum number "
 556                               "of regions (" SIZE_FORMAT ") of requested size (" SIZE_FORMAT "%s).",
 557                       byte_size_in_proper_unit(max_heap_size), proper_unit_for_byte_size(max_heap_size),
 558                       MIN_NUM_REGIONS,
 559                       byte_size_in_proper_unit(ShenandoahHeapRegionSize), proper_unit_for_byte_size(ShenandoahHeapRegionSize));
 560       vm_exit_during_initialization("Invalid -XX:ShenandoahHeapRegionSize option", message);
 561     }
 562     if (ShenandoahHeapRegionSize < ShenandoahMinRegionSize) {
 563       err_msg message("Heap region size (" SIZE_FORMAT "%s) should be larger than min region size (" SIZE_FORMAT "%s).",
 564                       byte_size_in_proper_unit(ShenandoahHeapRegionSize), proper_unit_for_byte_size(ShenandoahHeapRegionSize),
 565                       byte_size_in_proper_unit(ShenandoahMinRegionSize),  proper_unit_for_byte_size(ShenandoahMinRegionSize));
 566       vm_exit_during_initialization("Invalid -XX:ShenandoahHeapRegionSize option", message);
 567     }
 568     if (ShenandoahHeapRegionSize > ShenandoahMaxRegionSize) {
 569       err_msg message("Heap region size (" SIZE_FORMAT "%s) should be lower than max region size (" SIZE_FORMAT "%s).",
 570                       byte_size_in_proper_unit(ShenandoahHeapRegionSize), proper_unit_for_byte_size(ShenandoahHeapRegionSize),
 571                       byte_size_in_proper_unit(ShenandoahMaxRegionSize),  proper_unit_for_byte_size(ShenandoahMaxRegionSize));
 572       vm_exit_during_initialization("Invalid -XX:ShenandoahHeapRegionSize option", message);
 573     }
 574     region_size = ShenandoahHeapRegionSize;
 575   }
 576 
 577   // Make sure region size is at least one large page, if enabled.
 578   // Otherwise, uncommitting one region may falsely uncommit the adjacent
 579   // regions too.
 580   // Also see shenandoahArguments.cpp, where it handles UseLargePages.
 581   if (UseLargePages && ShenandoahUncommit) {
 582     region_size = MAX2(region_size, os::large_page_size());
 583   }
 584 
 585   int region_size_log = log2_long((jlong) region_size);
 586   // Recalculate the region size to make sure it's a power of
 587   // 2. This means that region_size is the largest power of 2 that's
 588   // <= what we've calculated so far.
 589   region_size = size_t(1) << region_size_log;
 590 
 591   // Now, set up the globals.
 592   guarantee(RegionSizeBytesShift == 0, "we should only set it once");
 593   RegionSizeBytesShift = (size_t)region_size_log;
 594 
 595   guarantee(RegionSizeWordsShift == 0, "we should only set it once");
 596   RegionSizeWordsShift = RegionSizeBytesShift - LogHeapWordSize;
 597 
 598   guarantee(RegionSizeBytes == 0, "we should only set it once");
 599   RegionSizeBytes = region_size;
 600   RegionSizeWords = RegionSizeBytes >> LogHeapWordSize;
 601   assert (RegionSizeWords*HeapWordSize == RegionSizeBytes, "sanity");
 602 
 603   guarantee(RegionSizeWordsMask == 0, "we should only set it once");
 604   RegionSizeWordsMask = RegionSizeWords - 1;
 605 
 606   guarantee(RegionSizeBytesMask == 0, "we should only set it once");
 607   RegionSizeBytesMask = RegionSizeBytes - 1;
 608 
 609   guarantee(RegionCount == 0, "we should only set it once");
 610   RegionCount = max_heap_size / RegionSizeBytes;
 611   guarantee(RegionCount >= MIN_NUM_REGIONS, "Should have at least minimum regions");
 612 
 613   guarantee(HumongousThresholdWords == 0, "we should only set it once");
 614   HumongousThresholdWords = RegionSizeWords * ShenandoahHumongousThreshold / 100;
 615   HumongousThresholdWords = align_down(HumongousThresholdWords, MinObjAlignment);
 616   assert (HumongousThresholdWords <= RegionSizeWords, "sanity");
 617 
 618   guarantee(HumongousThresholdBytes == 0, "we should only set it once");
 619   HumongousThresholdBytes = HumongousThresholdWords * HeapWordSize;
 620   assert (HumongousThresholdBytes <= RegionSizeBytes, "sanity");
 621 
 622   // The rationale for trimming the TLAB sizes has to do with the raciness in
 623   // TLAB allocation machinery. It may happen that TLAB sizing policy polls Shenandoah
 624   // about next free size, gets the answer for region #N, goes away for a while, then
 625   // tries to allocate in region #N, and fail because some other thread have claimed part
 626   // of the region #N, and then the freeset allocation code has to retire the region #N,
 627   // before moving the allocation to region #N+1.
 628   //
 629   // The worst case realizes when "answer" is "region size", which means it could
 630   // prematurely retire an entire region. Having smaller TLABs does not fix that
 631   // completely, but reduces the probability of too wasteful region retirement.
 632   // With current divisor, we will waste no more than 1/8 of region size in the worst
 633   // case. This also has a secondary effect on collection set selection: even under
 634   // the race, the regions would be at least 7/8 used, which allows relying on
 635   // "used" - "live" for cset selection. Otherwise, we can get the fragmented region
 636   // below the garbage threshold that would never be considered for collection.
 637   //
 638   // The whole thing is mitigated if Elastic TLABs are enabled.
 639   //
 640   guarantee(MaxTLABSizeWords == 0, "we should only set it once");
 641   MaxTLABSizeWords = MIN2(ShenandoahElasticTLAB ? RegionSizeWords : (RegionSizeWords / 8), HumongousThresholdWords);
 642   MaxTLABSizeWords = align_down(MaxTLABSizeWords, MinObjAlignment);
 643 
 644   guarantee(MaxTLABSizeBytes == 0, "we should only set it once");
 645   MaxTLABSizeBytes = MaxTLABSizeWords * HeapWordSize;
 646   assert (MaxTLABSizeBytes > MinTLABSize, "should be larger");
 647 
 648   log_info(gc, init)("Regions: " SIZE_FORMAT " x " SIZE_FORMAT "%s",
 649                      RegionCount, byte_size_in_proper_unit(RegionSizeBytes), proper_unit_for_byte_size(RegionSizeBytes));
 650   log_info(gc, init)("Humongous object threshold: " SIZE_FORMAT "%s",
 651                      byte_size_in_proper_unit(HumongousThresholdBytes), proper_unit_for_byte_size(HumongousThresholdBytes));
 652   log_info(gc, init)("Max TLAB size: " SIZE_FORMAT "%s",
 653                      byte_size_in_proper_unit(MaxTLABSizeBytes), proper_unit_for_byte_size(MaxTLABSizeBytes));
 654 }
 655 
 656 void ShenandoahHeapRegion::do_commit() {
 657   if (!_heap->is_heap_region_special() && !os::commit_memory((char *) _reserved.start(), _reserved.byte_size(), false)) {
 658     report_java_out_of_memory("Unable to commit region");
 659   }
 660   if (!_heap->commit_bitmap_slice(this)) {
 661     report_java_out_of_memory("Unable to commit bitmaps for region");
 662   }
 663   _heap->increase_committed(ShenandoahHeapRegion::region_size_bytes());
 664 }
 665 
 666 void ShenandoahHeapRegion::do_uncommit() {
 667   if (!_heap->is_heap_region_special() && !os::uncommit_memory((char *) _reserved.start(), _reserved.byte_size())) {
 668     report_java_out_of_memory("Unable to uncommit region");
 669   }
 670   if (!_heap->uncommit_bitmap_slice(this)) {
 671     report_java_out_of_memory("Unable to uncommit bitmaps for region");
 672   }
 673   _heap->decrease_committed(ShenandoahHeapRegion::region_size_bytes());
 674 }
 675 
 676 void ShenandoahHeapRegion::set_state(RegionState to) {
 677   EventShenandoahHeapRegionStateChange evt;
 678   if (evt.should_commit()){
 679     evt.set_index((unsigned)region_number());
 680     evt.set_start((uintptr_t)bottom());
 681     evt.set_used(used());
 682     evt.set_from(_state);
 683     evt.set_to(to);
 684     evt.commit();
 685   }
 686   _state = to;
 687 }
 688 
 689 void ShenandoahHeapRegion::record_pin() {
 690   Atomic::add(&_critical_pins, (size_t)1);
 691 }
 692 
 693 void ShenandoahHeapRegion::record_unpin() {
 694   assert(pin_count() > 0, "Region " SIZE_FORMAT " should have non-zero pins", region_number());
 695   Atomic::sub(&_critical_pins, (size_t)1);
 696 }
 697 
 698 size_t ShenandoahHeapRegion::pin_count() const {
 699   return Atomic::load(&_critical_pins);
 700 }