1 /*
   2  * Copyright (c) 2013, 2018, Red Hat, Inc. All rights reserved.
   3  *
   4  * This code is free software; you can redistribute it and/or modify it
   5  * under the terms of the GNU General Public License version 2 only, as
   6  * published by the Free Software Foundation.
   7  *
   8  * This code is distributed in the hope that it will be useful, but WITHOUT
   9  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  10  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  11  * version 2 for more details (a copy is included in the LICENSE file that
  12  * accompanied this code).
  13  *
  14  * You should have received a copy of the GNU General Public License version
  15  * 2 along with this work; if not, write to the Free Software Foundation,
  16  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  17  *
  18  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  19  * or visit www.oracle.com if you need additional information or have any
  20  * questions.
  21  *
  22  */
  23 
  24 #include "precompiled.hpp"
  25 #include "memory/allocation.hpp"
  26 #include "gc/shenandoah/shenandoahBrooksPointer.hpp"
  27 #include "gc/shenandoah/shenandoahHeapRegionSet.inline.hpp"
  28 #include "gc/shenandoah/shenandoahHeap.hpp"
  29 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
  30 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
  31 #include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"
  32 #include "gc/shenandoah/shenandoahTraversalGC.hpp"
  33 #include "gc/shared/space.inline.hpp"
  34 #include "memory/iterator.inline.hpp"
  35 #include "memory/resourceArea.hpp"
  36 #include "memory/universe.hpp"
  37 #include "oops/oop.inline.hpp"
  38 #include "runtime/java.hpp"
  39 #include "runtime/mutexLocker.hpp"
  40 #include "runtime/os.hpp"
  41 #include "runtime/safepoint.hpp"
  42 
  43 size_t ShenandoahHeapRegion::RegionCount = 0;
  44 size_t ShenandoahHeapRegion::RegionSizeBytes = 0;
  45 size_t ShenandoahHeapRegion::RegionSizeWords = 0;
  46 size_t ShenandoahHeapRegion::RegionSizeBytesShift = 0;
  47 size_t ShenandoahHeapRegion::RegionSizeWordsShift = 0;
  48 size_t ShenandoahHeapRegion::RegionSizeBytesMask = 0;
  49 size_t ShenandoahHeapRegion::RegionSizeWordsMask = 0;
  50 size_t ShenandoahHeapRegion::HumongousThresholdBytes = 0;
  51 size_t ShenandoahHeapRegion::HumongousThresholdWords = 0;
  52 size_t ShenandoahHeapRegion::MaxTLABSizeBytes = 0;
  53 size_t ShenandoahHeapRegion::MaxTLABSizeWords = 0;
  54 
  55 ShenandoahHeapRegion::PaddedAllocSeqNum ShenandoahHeapRegion::_alloc_seq_num;
  56 
  57 ShenandoahHeapRegion::ShenandoahHeapRegion(ShenandoahHeap* heap, HeapWord* start,
  58                                            size_t size_words, size_t index, bool committed) :
  59   _heap(heap),
  60   _pacer(ShenandoahPacing ? heap->pacer() : NULL),
  61   _reserved(MemRegion(start, size_words)),
  62   _region_number(index),
  63   _new_top(NULL),
  64   _critical_pins(0),
  65   _empty_time(os::elapsedTime()),
  66   _state(committed ? _empty_committed : _empty_uncommitted),
  67   _tlab_allocs(0),
  68   _gclab_allocs(0),
  69   _shared_allocs(0),
  70   _seqnum_first_alloc_mutator(0),
  71   _seqnum_first_alloc_gc(0),
  72   _seqnum_last_alloc_mutator(0),
  73   _seqnum_last_alloc_gc(0),
  74   _live_data(0) {
  75 
  76   ContiguousSpace::initialize(_reserved, true, committed);
  77 }
  78 
  79 size_t ShenandoahHeapRegion::region_number() const {
  80   return _region_number;
  81 }
  82 
  83 void ShenandoahHeapRegion::report_illegal_transition(const char *method) {
  84   ResourceMark rm;
  85   stringStream ss;
  86   ss.print("Illegal region state transition from \"%s\", at %s\n  ", region_state_to_string(_state), method);
  87   print_on(&ss);
  88   fatal("%s", ss.as_string());
  89 }
  90 
  91 void ShenandoahHeapRegion::make_regular_allocation() {
  92   _heap->assert_heaplock_owned_by_current_thread();
  93 
  94   switch (_state) {
  95     case _empty_uncommitted:
  96       do_commit();
  97     case _empty_committed:
  98       _state = _regular;
  99     case _regular:
 100     case _pinned:
 101       return;
 102     default:
 103       report_illegal_transition("regular allocation");
 104   }
 105 }
 106 
 107 void ShenandoahHeapRegion::make_regular_bypass() {
 108   _heap->assert_heaplock_owned_by_current_thread();
 109   assert (_heap->is_full_gc_in_progress() || _heap->is_degenerated_gc_in_progress(),
 110           "only for full or degen GC");
 111 
 112   switch (_state) {
 113     case _empty_uncommitted:
 114       do_commit();
 115     case _empty_committed:
 116     case _cset:
 117     case _humongous_start:
 118     case _humongous_cont:
 119       _state = _regular;
 120       return;
 121     case _pinned_cset:
 122       _state = _pinned;
 123       return;
 124     case _regular:
 125     case _pinned:
 126       return;
 127     default:
 128       report_illegal_transition("regular bypass");
 129   }
 130 }
 131 
 132 void ShenandoahHeapRegion::make_humongous_start() {
 133   _heap->assert_heaplock_owned_by_current_thread();
 134   switch (_state) {
 135     case _empty_uncommitted:
 136       do_commit();
 137     case _empty_committed:
 138       _state = _humongous_start;
 139       return;
 140     default:
 141       report_illegal_transition("humongous start allocation");
 142   }
 143 }
 144 
 145 void ShenandoahHeapRegion::make_humongous_start_bypass() {
 146   _heap->assert_heaplock_owned_by_current_thread();
 147   assert (_heap->is_full_gc_in_progress(), "only for full GC");
 148 
 149   switch (_state) {
 150     case _empty_committed:
 151     case _regular:
 152     case _humongous_start:
 153     case _humongous_cont:
 154       _state = _humongous_start;
 155       return;
 156     default:
 157       report_illegal_transition("humongous start bypass");
 158   }
 159 }
 160 
 161 void ShenandoahHeapRegion::make_humongous_cont() {
 162   _heap->assert_heaplock_owned_by_current_thread();
 163   switch (_state) {
 164     case _empty_uncommitted:
 165       do_commit();
 166     case _empty_committed:
 167       _state = _humongous_cont;
 168       return;
 169     default:
 170       report_illegal_transition("humongous continuation allocation");
 171   }
 172 }
 173 
 174 void ShenandoahHeapRegion::make_humongous_cont_bypass() {
 175   _heap->assert_heaplock_owned_by_current_thread();
 176   assert (_heap->is_full_gc_in_progress(), "only for full GC");
 177 
 178   switch (_state) {
 179     case _empty_committed:
 180     case _regular:
 181     case _humongous_start:
 182     case _humongous_cont:
 183       _state = _humongous_cont;
 184       return;
 185     default:
 186       report_illegal_transition("humongous continuation bypass");
 187   }
 188 }
 189 
 190 void ShenandoahHeapRegion::make_pinned() {
 191   _heap->assert_heaplock_owned_by_current_thread();
 192   switch (_state) {
 193     case _regular:
 194       assert (_critical_pins == 0, "sanity");
 195       _state = _pinned;
 196     case _pinned_cset:
 197     case _pinned:
 198       _critical_pins++;
 199       return;
 200     case _humongous_start:
 201       assert (_critical_pins == 0, "sanity");
 202       _state = _pinned_humongous_start;
 203     case _pinned_humongous_start:
 204       _critical_pins++;
 205       return;
 206     case _cset:
 207       guarantee(_heap->cancelled_gc(), "only valid when evac has been cancelled");
 208       assert (_critical_pins == 0, "sanity");
 209       _state = _pinned_cset;
 210       _critical_pins++;
 211       return;
 212     default:
 213       report_illegal_transition("pinning");
 214   }
 215 }
 216 
 217 void ShenandoahHeapRegion::make_unpinned() {
 218   _heap->assert_heaplock_owned_by_current_thread();
 219   switch (_state) {
 220     case _pinned:
 221       assert (_critical_pins > 0, "sanity");
 222       _critical_pins--;
 223       if (_critical_pins == 0) {
 224         _state = _regular;
 225       }
 226       return;
 227     case _regular:
 228     case _humongous_start:
 229       assert (_critical_pins == 0, "sanity");
 230       return;
 231     case _pinned_cset:
 232       guarantee(_heap->cancelled_gc(), "only valid when evac has been cancelled");
 233       assert (_critical_pins > 0, "sanity");
 234       _critical_pins--;
 235       if (_critical_pins == 0) {
 236         _state = _cset;
 237       }
 238       return;
 239     case _pinned_humongous_start:
 240       assert (_critical_pins > 0, "sanity");
 241       _critical_pins--;
 242       if (_critical_pins == 0) {
 243         _state = _humongous_start;
 244       }
 245       return;
 246     default:
 247       report_illegal_transition("unpinning");
 248   }
 249 }
 250 
 251 void ShenandoahHeapRegion::make_cset() {
 252   _heap->assert_heaplock_owned_by_current_thread();
 253   switch (_state) {
 254     case _regular:
 255       _state = _cset;
 256     case _cset:
 257       return;
 258     default:
 259       report_illegal_transition("cset");
 260   }
 261 }
 262 
 263 void ShenandoahHeapRegion::make_trash() {
 264   _heap->assert_heaplock_owned_by_current_thread();
 265   switch (_state) {
 266     case _cset:
 267       // Reclaiming cset regions
 268     case _humongous_start:
 269     case _humongous_cont:
 270       // Reclaiming humongous regions
 271     case _regular:
 272       // Immediate region reclaim
 273       _state = _trash;
 274       return;
 275     default:
 276       report_illegal_transition("trashing");
 277   }
 278 }
 279 
 280 void ShenandoahHeapRegion::make_trash_immediate() {
 281   make_trash();
 282 
 283   // On this path, we know there are no marked objects in the region,
 284   // tell marking context about it to bypass bitmap resets.
 285   _heap->complete_marking_context()->reset_top_bitmap(this);
 286 }
 287 
 288 void ShenandoahHeapRegion::make_empty() {
 289   _heap->assert_heaplock_owned_by_current_thread();
 290   switch (_state) {
 291     case _trash:
 292       _state = _empty_committed;
 293       _empty_time = os::elapsedTime();
 294       return;
 295     default:
 296       report_illegal_transition("emptying");
 297   }
 298 }
 299 
 300 void ShenandoahHeapRegion::make_uncommitted() {
 301   _heap->assert_heaplock_owned_by_current_thread();
 302   switch (_state) {
 303     case _empty_committed:
 304       do_uncommit();
 305       _state = _empty_uncommitted;
 306       return;
 307     default:
 308       report_illegal_transition("uncommiting");
 309   }
 310 }
 311 
 312 void ShenandoahHeapRegion::make_committed_bypass() {
 313   _heap->assert_heaplock_owned_by_current_thread();
 314   assert (_heap->is_full_gc_in_progress(), "only for full GC");
 315 
 316   switch (_state) {
 317     case _empty_uncommitted:
 318       do_commit();
 319       _state = _empty_committed;
 320       return;
 321     default:
 322       report_illegal_transition("commit bypass");
 323   }
 324 }
 325 
 326 void ShenandoahHeapRegion::clear_live_data() {
 327   OrderAccess::release_store_fence<size_t>(&_live_data, 0);
 328 }
 329 
 330 void ShenandoahHeapRegion::reset_alloc_metadata() {
 331   _tlab_allocs = 0;
 332   _gclab_allocs = 0;
 333   _shared_allocs = 0;
 334   _seqnum_first_alloc_mutator = 0;
 335   _seqnum_last_alloc_mutator = 0;
 336   _seqnum_first_alloc_gc = 0;
 337   _seqnum_last_alloc_gc = 0;
 338 }
 339 
 340 void ShenandoahHeapRegion::reset_alloc_metadata_to_shared() {
 341   if (used() > 0) {
 342     _tlab_allocs = 0;
 343     _gclab_allocs = 0;
 344     _shared_allocs = used() >> LogHeapWordSize;
 345     uint64_t next = _alloc_seq_num.value++;
 346     _seqnum_first_alloc_mutator = next;
 347     _seqnum_last_alloc_mutator = next;
 348     _seqnum_first_alloc_gc = 0;
 349     _seqnum_last_alloc_gc = 0;
 350   } else {
 351     reset_alloc_metadata();
 352   }
 353 }
 354 
 355 size_t ShenandoahHeapRegion::get_shared_allocs() const {
 356   return _shared_allocs * HeapWordSize;
 357 }
 358 
 359 size_t ShenandoahHeapRegion::get_tlab_allocs() const {
 360   return _tlab_allocs * HeapWordSize;
 361 }
 362 
 363 size_t ShenandoahHeapRegion::get_gclab_allocs() const {
 364   return _gclab_allocs * HeapWordSize;
 365 }
 366 
 367 void ShenandoahHeapRegion::set_live_data(size_t s) {
 368   assert(Thread::current()->is_VM_thread(), "by VM thread");
 369   _live_data = (s >> LogHeapWordSize);
 370 }
 371 
 372 size_t ShenandoahHeapRegion::get_live_data_words() const {
 373   return OrderAccess::load_acquire(&_live_data);
 374 }
 375 
 376 size_t ShenandoahHeapRegion::get_live_data_bytes() const {
 377   return get_live_data_words() * HeapWordSize;
 378 }
 379 
 380 bool ShenandoahHeapRegion::has_live() const {
 381   return get_live_data_words() != 0;
 382 }
 383 
 384 size_t ShenandoahHeapRegion::garbage() const {
 385   assert(used() >= get_live_data_bytes(), "Live Data must be a subset of used() live: " SIZE_FORMAT " used: " SIZE_FORMAT,
 386          get_live_data_bytes(), used());
 387 
 388   size_t result = used() - get_live_data_bytes();
 389   return result;
 390 }
 391 
 392 void ShenandoahHeapRegion::print_on(outputStream* st) const {
 393   st->print("|");
 394   st->print(SIZE_FORMAT_W(5), this->_region_number);
 395 
 396   switch (_state) {
 397     case _empty_uncommitted:
 398       st->print("|EU ");
 399       break;
 400     case _empty_committed:
 401       st->print("|EC ");
 402       break;
 403     case _regular:
 404       st->print("|R  ");
 405       break;
 406     case _humongous_start:
 407       st->print("|H  ");
 408       break;
 409     case _pinned_humongous_start:
 410       st->print("|HP ");
 411       break;
 412     case _humongous_cont:
 413       st->print("|HC ");
 414       break;
 415     case _cset:
 416       st->print("|CS ");
 417       break;
 418     case _trash:
 419       st->print("|T  ");
 420       break;
 421     case _pinned:
 422       st->print("|P  ");
 423       break;
 424     case _pinned_cset:
 425       st->print("|CSP");
 426       break;
 427     default:
 428       ShouldNotReachHere();
 429   }
 430   st->print("|BTE " INTPTR_FORMAT_W(12) ", " INTPTR_FORMAT_W(12) ", " INTPTR_FORMAT_W(12),
 431             p2i(bottom()), p2i(top()), p2i(end()));
 432   st->print("|TAMS " INTPTR_FORMAT_W(12),
 433             p2i(_heap->marking_context()->top_at_mark_start(const_cast<ShenandoahHeapRegion*>(this))));
 434   st->print("|U " SIZE_FORMAT_W(5) "%1s", byte_size_in_proper_unit(used()),                proper_unit_for_byte_size(used()));
 435   st->print("|T " SIZE_FORMAT_W(5) "%1s", byte_size_in_proper_unit(get_tlab_allocs()),     proper_unit_for_byte_size(get_tlab_allocs()));
 436   st->print("|G " SIZE_FORMAT_W(5) "%1s", byte_size_in_proper_unit(get_gclab_allocs()),    proper_unit_for_byte_size(get_gclab_allocs()));
 437   st->print("|S " SIZE_FORMAT_W(5) "%1s", byte_size_in_proper_unit(get_shared_allocs()),   proper_unit_for_byte_size(get_shared_allocs()));
 438   st->print("|L " SIZE_FORMAT_W(5) "%1s", byte_size_in_proper_unit(get_live_data_bytes()), proper_unit_for_byte_size(get_live_data_bytes()));
 439   st->print("|CP " SIZE_FORMAT_W(3), _critical_pins);
 440   st->print("|SN " UINT64_FORMAT_X_W(12) ", " UINT64_FORMAT_X_W(8) ", " UINT64_FORMAT_X_W(8) ", " UINT64_FORMAT_X_W(8),
 441             seqnum_first_alloc_mutator(), seqnum_last_alloc_mutator(),
 442             seqnum_first_alloc_gc(), seqnum_last_alloc_gc());
 443   st->cr();
 444 }
 445 
 446 void ShenandoahHeapRegion::oop_iterate(OopIterateClosure* blk) {
 447   if (!is_active()) return;
 448   if (is_humongous()) {
 449     oop_iterate_humongous(blk);
 450   } else {
 451     oop_iterate_objects(blk);
 452   }
 453 }
 454 
 455 void ShenandoahHeapRegion::oop_iterate_objects(OopIterateClosure* blk) {
 456   assert(! is_humongous(), "no humongous region here");
 457   HeapWord* obj_addr = bottom() + ShenandoahBrooksPointer::word_size();
 458   HeapWord* t = top();
 459   // Could call objects iterate, but this is easier.
 460   while (obj_addr < t) {
 461     oop obj = oop(obj_addr);
 462     obj_addr += obj->oop_iterate_size(blk) + ShenandoahBrooksPointer::word_size();
 463   }
 464 }
 465 
 466 void ShenandoahHeapRegion::oop_iterate_humongous(OopIterateClosure* blk) {
 467   assert(is_humongous(), "only humongous region here");
 468   // Find head.
 469   ShenandoahHeapRegion* r = humongous_start_region();
 470   assert(r->is_humongous_start(), "need humongous head here");
 471   oop obj = oop(r->bottom() + ShenandoahBrooksPointer::word_size());
 472   obj->oop_iterate(blk, MemRegion(bottom(), top()));
 473 }
 474 
 475 ShenandoahHeapRegion* ShenandoahHeapRegion::humongous_start_region() const {
 476   assert(is_humongous(), "Must be a part of the humongous region");
 477   size_t reg_num = region_number();
 478   ShenandoahHeapRegion* r = const_cast<ShenandoahHeapRegion*>(this);
 479   while (!r->is_humongous_start()) {
 480     assert(reg_num > 0, "Sanity");
 481     reg_num --;
 482     r = _heap->get_region(reg_num);
 483     assert(r->is_humongous(), "Must be a part of the humongous region");
 484   }
 485   assert(r->is_humongous_start(), "Must be");
 486   return r;
 487 }
 488 
 489 void ShenandoahHeapRegion::recycle() {
 490   ContiguousSpace::clear(false);
 491   if (ZapUnusedHeapArea) {
 492     ContiguousSpace::mangle_unused_area_complete();
 493   }
 494   clear_live_data();
 495 
 496   reset_alloc_metadata();
 497 
 498   _heap->marking_context()->reset_top_at_mark_start(this);
 499 
 500   make_empty();
 501 }
 502 
 503 HeapWord* ShenandoahHeapRegion::block_start_const(const void* p) const {
 504   assert(MemRegion(bottom(), end()).contains(p),
 505          "p (" PTR_FORMAT ") not in space [" PTR_FORMAT ", " PTR_FORMAT ")",
 506          p2i(p), p2i(bottom()), p2i(end()));
 507   if (p >= top()) {
 508     return top();
 509   } else {
 510     HeapWord* last = bottom() + ShenandoahBrooksPointer::word_size();
 511     HeapWord* cur = last;
 512     while (cur <= p) {
 513       last = cur;
 514       cur += oop(cur)->size() + ShenandoahBrooksPointer::word_size();
 515     }
 516     shenandoah_assert_correct(NULL, oop(last));
 517     return last;
 518   }
 519 }
 520 
 521 void ShenandoahHeapRegion::setup_sizes(size_t initial_heap_size, size_t max_heap_size) {
 522   // Absolute minimums we should not ever break.
 523   static const size_t MIN_REGION_SIZE = 256*K;
 524 
 525   if (FLAG_IS_DEFAULT(ShenandoahMinRegionSize)) {
 526     FLAG_SET_DEFAULT(ShenandoahMinRegionSize, MIN_REGION_SIZE);
 527   }
 528 
 529   size_t region_size;
 530   if (FLAG_IS_DEFAULT(ShenandoahHeapRegionSize)) {
 531     if (ShenandoahMinRegionSize > initial_heap_size / MIN_NUM_REGIONS) {
 532       err_msg message("Initial heap size (" SIZE_FORMAT "K) is too low to afford the minimum number "
 533                       "of regions (" SIZE_FORMAT ") of minimum region size (" SIZE_FORMAT "K).",
 534                       initial_heap_size/K, MIN_NUM_REGIONS, ShenandoahMinRegionSize/K);
 535       vm_exit_during_initialization("Invalid -XX:ShenandoahMinRegionSize option", message);
 536     }
 537     if (ShenandoahMinRegionSize < MIN_REGION_SIZE) {
 538       err_msg message("" SIZE_FORMAT "K should not be lower than minimum region size (" SIZE_FORMAT "K).",
 539                       ShenandoahMinRegionSize/K,  MIN_REGION_SIZE/K);
 540       vm_exit_during_initialization("Invalid -XX:ShenandoahMinRegionSize option", message);
 541     }
 542     if (ShenandoahMinRegionSize < MinTLABSize) {
 543       err_msg message("" SIZE_FORMAT "K should not be lower than TLAB size size (" SIZE_FORMAT "K).",
 544                       ShenandoahMinRegionSize/K,  MinTLABSize/K);
 545       vm_exit_during_initialization("Invalid -XX:ShenandoahMinRegionSize option", message);
 546     }
 547     if (ShenandoahMaxRegionSize < MIN_REGION_SIZE) {
 548       err_msg message("" SIZE_FORMAT "K should not be lower than min region size (" SIZE_FORMAT "K).",
 549                       ShenandoahMaxRegionSize/K,  MIN_REGION_SIZE/K);
 550       vm_exit_during_initialization("Invalid -XX:ShenandoahMaxRegionSize option", message);
 551     }
 552     if (ShenandoahMinRegionSize > ShenandoahMaxRegionSize) {
 553       err_msg message("Minimum (" SIZE_FORMAT "K) should be larger than maximum (" SIZE_FORMAT "K).",
 554                       ShenandoahMinRegionSize/K, ShenandoahMaxRegionSize/K);
 555       vm_exit_during_initialization("Invalid -XX:ShenandoahMinRegionSize or -XX:ShenandoahMaxRegionSize", message);
 556     }
 557 
 558     // We rapidly expand to max_heap_size in most scenarios, so that is the measure
 559     // for usual heap sizes. Do not depend on initial_heap_size here.
 560     region_size = max_heap_size / ShenandoahTargetNumRegions;
 561 
 562     // Now make sure that we don't go over or under our limits.
 563     region_size = MAX2(ShenandoahMinRegionSize, region_size);
 564     region_size = MIN2(ShenandoahMaxRegionSize, region_size);
 565 
 566   } else {
 567     if (ShenandoahHeapRegionSize > initial_heap_size / MIN_NUM_REGIONS) {
 568       err_msg message("Initial heap size (" SIZE_FORMAT "K) is too low to afford the minimum number "
 569                               "of regions (" SIZE_FORMAT ") of requested size (" SIZE_FORMAT "K).",
 570                       initial_heap_size/K, MIN_NUM_REGIONS, ShenandoahHeapRegionSize/K);
 571       vm_exit_during_initialization("Invalid -XX:ShenandoahHeapRegionSize option", message);
 572     }
 573     if (ShenandoahHeapRegionSize < ShenandoahMinRegionSize) {
 574       err_msg message("Heap region size (" SIZE_FORMAT "K) should be larger than min region size (" SIZE_FORMAT "K).",
 575                       ShenandoahHeapRegionSize/K, ShenandoahMinRegionSize/K);
 576       vm_exit_during_initialization("Invalid -XX:ShenandoahHeapRegionSize option", message);
 577     }
 578     if (ShenandoahHeapRegionSize > ShenandoahMaxRegionSize) {
 579       err_msg message("Heap region size (" SIZE_FORMAT "K) should be lower than max region size (" SIZE_FORMAT "K).",
 580                       ShenandoahHeapRegionSize/K, ShenandoahMaxRegionSize/K);
 581       vm_exit_during_initialization("Invalid -XX:ShenandoahHeapRegionSize option", message);
 582     }
 583     region_size = ShenandoahHeapRegionSize;
 584   }
 585 
 586   // Make sure region size is at least one large page, if enabled.
 587   // Otherwise, uncommitting one region may falsely uncommit the adjacent
 588   // regions too.
 589   // Also see shenandoahArguments.cpp, where it handles UseLargePages.
 590   if (UseLargePages && ShenandoahUncommit) {
 591     region_size = MAX2(region_size, os::large_page_size());
 592   }
 593 
 594   int region_size_log = log2_long((jlong) region_size);
 595   // Recalculate the region size to make sure it's a power of
 596   // 2. This means that region_size is the largest power of 2 that's
 597   // <= what we've calculated so far.
 598   region_size = size_t(1) << region_size_log;
 599 
 600   // Now, set up the globals.
 601   guarantee(RegionSizeBytesShift == 0, "we should only set it once");
 602   RegionSizeBytesShift = (size_t)region_size_log;
 603 
 604   guarantee(RegionSizeWordsShift == 0, "we should only set it once");
 605   RegionSizeWordsShift = RegionSizeBytesShift - LogHeapWordSize;
 606 
 607   guarantee(RegionSizeBytes == 0, "we should only set it once");
 608   RegionSizeBytes = region_size;
 609   RegionSizeWords = RegionSizeBytes >> LogHeapWordSize;
 610   assert (RegionSizeWords*HeapWordSize == RegionSizeBytes, "sanity");
 611 
 612   guarantee(RegionSizeWordsMask == 0, "we should only set it once");
 613   RegionSizeWordsMask = RegionSizeWords - 1;
 614 
 615   guarantee(RegionSizeBytesMask == 0, "we should only set it once");
 616   RegionSizeBytesMask = RegionSizeBytes - 1;
 617 
 618   guarantee(RegionCount == 0, "we should only set it once");
 619   RegionCount = max_heap_size / RegionSizeBytes;
 620   guarantee(RegionCount >= MIN_NUM_REGIONS, "Should have at least minimum regions");
 621 
 622   guarantee(HumongousThresholdWords == 0, "we should only set it once");
 623   HumongousThresholdWords = RegionSizeWords * ShenandoahHumongousThreshold / 100;
 624   assert (HumongousThresholdWords <= RegionSizeWords, "sanity");
 625 
 626   guarantee(HumongousThresholdBytes == 0, "we should only set it once");
 627   HumongousThresholdBytes = HumongousThresholdWords * HeapWordSize;
 628   assert (HumongousThresholdBytes <= RegionSizeBytes, "sanity");
 629 
 630   // The rationale for trimming the TLAB sizes has to do with the raciness in
 631   // TLAB allocation machinery. It may happen that TLAB sizing policy polls Shenandoah
 632   // about next free size, gets the answer for region #N, goes away for a while, then
 633   // tries to allocate in region #N, and fail because some other thread have claimed part
 634   // of the region #N, and then the freeset allocation code has to retire the region #N,
 635   // before moving the allocation to region #N+1.
 636   //
 637   // The worst case realizes when "answer" is "region size", which means it could
 638   // prematurely retire an entire region. Having smaller TLABs does not fix that
 639   // completely, but reduces the probability of too wasteful region retirement.
 640   // With current divisor, we will waste no more than 1/8 of region size in the worst
 641   // case. This also has a secondary effect on collection set selection: even under
 642   // the race, the regions would be at least 7/8 used, which allows relying on
 643   // "used" - "live" for cset selection. Otherwise, we can get the fragmented region
 644   // below the garbage threshold that would never be considered for collection.
 645   //
 646   // The whole thing is mitigated if Elastic TLABs are enabled.
 647   //
 648   guarantee(MaxTLABSizeBytes == 0, "we should only set it once");
 649   MaxTLABSizeBytes = MIN2(ShenandoahElasticTLAB ? RegionSizeBytes : (RegionSizeBytes / 8), HumongousThresholdBytes);
 650   assert (MaxTLABSizeBytes > MinTLABSize, "should be larger");
 651 
 652   guarantee(MaxTLABSizeWords == 0, "we should only set it once");
 653   MaxTLABSizeWords = MaxTLABSizeBytes / HeapWordSize;
 654 
 655   log_info(gc, init)("Regions: " SIZE_FORMAT " x " SIZE_FORMAT "%s",
 656                      RegionCount, byte_size_in_proper_unit(RegionSizeBytes), proper_unit_for_byte_size(RegionSizeBytes));
 657   log_info(gc, init)("Humongous object threshold: " SIZE_FORMAT "%s",
 658                      byte_size_in_proper_unit(HumongousThresholdBytes), proper_unit_for_byte_size(HumongousThresholdBytes));
 659   log_info(gc, init)("Max TLAB size: " SIZE_FORMAT "%s",
 660                      byte_size_in_proper_unit(MaxTLABSizeBytes), proper_unit_for_byte_size(MaxTLABSizeBytes));
 661 }
 662 
 663 void ShenandoahHeapRegion::do_commit() {
 664   if (!os::commit_memory((char *) _reserved.start(), _reserved.byte_size(), false)) {
 665     report_java_out_of_memory("Unable to commit region");
 666   }
 667   if (!_heap->commit_bitmap_slice(this)) {
 668     report_java_out_of_memory("Unable to commit bitmaps for region");
 669   }
 670   _heap->increase_committed(ShenandoahHeapRegion::region_size_bytes());
 671 }
 672 
 673 void ShenandoahHeapRegion::do_uncommit() {
 674   if (!os::uncommit_memory((char *) _reserved.start(), _reserved.byte_size())) {
 675     report_java_out_of_memory("Unable to uncommit region");
 676   }
 677   if (!_heap->uncommit_bitmap_slice(this)) {
 678     report_java_out_of_memory("Unable to uncommit bitmaps for region");
 679   }
 680   _heap->decrease_committed(ShenandoahHeapRegion::region_size_bytes());
 681 }