1 /*
2 * Copyright (c) 2013, 2019, Red Hat, Inc. All rights reserved.
3 *
4 * This code is free software; you can redistribute it and/or modify it
5 * under the terms of the GNU General Public License version 2 only, as
6 * published by the Free Software Foundation.
7 *
8 * This code is distributed in the hope that it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
11 * version 2 for more details (a copy is included in the LICENSE file that
12 * accompanied this code).
13 *
14 * You should have received a copy of the GNU General Public License version
15 * 2 along with this work; if not, write to the Free Software Foundation,
16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
17 *
18 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
19 * or visit www.oracle.com if you need additional information or have any
20 * questions.
21 *
22 */
23
24 #include "precompiled.hpp"
25 #include "memory/allocation.hpp"
26 #include "gc/shenandoah/shenandoahForwarding.hpp"
27 #include "gc/shenandoah/shenandoahHeapRegionSet.inline.hpp"
28 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
29 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
30 #include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"
31 #include "gc/shenandoah/shenandoahTraversalGC.hpp"
32 #include "gc/shared/space.inline.hpp"
33 #include "jfr/jfrEvents.hpp"
34 #include "memory/iterator.inline.hpp"
35 #include "memory/resourceArea.hpp"
36 #include "memory/universe.hpp"
37 #include "oops/oop.inline.hpp"
38 #include "runtime/java.hpp"
39 #include "runtime/mutexLocker.hpp"
40 #include "runtime/os.hpp"
41 #include "runtime/safepoint.hpp"
42
43 size_t ShenandoahHeapRegion::RegionCount = 0;
44 size_t ShenandoahHeapRegion::RegionSizeBytes = 0;
45 size_t ShenandoahHeapRegion::RegionSizeWords = 0;
46 size_t ShenandoahHeapRegion::RegionSizeBytesShift = 0;
47 size_t ShenandoahHeapRegion::RegionSizeWordsShift = 0;
48 size_t ShenandoahHeapRegion::RegionSizeBytesMask = 0;
49 size_t ShenandoahHeapRegion::RegionSizeWordsMask = 0;
50 size_t ShenandoahHeapRegion::HumongousThresholdBytes = 0;
51 size_t ShenandoahHeapRegion::HumongousThresholdWords = 0;
52 size_t ShenandoahHeapRegion::MaxTLABSizeBytes = 0;
53 size_t ShenandoahHeapRegion::MaxTLABSizeWords = 0;
54
55 ShenandoahHeapRegion::PaddedAllocSeqNum ShenandoahHeapRegion::_alloc_seq_num;
56
57 ShenandoahHeapRegion::ShenandoahHeapRegion(ShenandoahHeap* heap, HeapWord* start,
58 size_t size_words, size_t index, bool committed) :
59 _heap(heap),
60 _reserved(MemRegion(start, size_words)),
61 _region_number(index),
62 _new_top(NULL),
63 _critical_pins(0),
64 _empty_time(os::elapsedTime()),
65 _state(committed ? _empty_committed : _empty_uncommitted),
66 _tlab_allocs(0),
67 _gclab_allocs(0),
68 _shared_allocs(0),
69 _seqnum_first_alloc_mutator(0),
70 _seqnum_first_alloc_gc(0),
71 _seqnum_last_alloc_mutator(0),
72 _seqnum_last_alloc_gc(0),
73 _live_data(0) {
74
75 ContiguousSpace::initialize(_reserved, true, committed);
76 }
77
78 size_t ShenandoahHeapRegion::region_number() const {
79 return _region_number;
80 }
81
82 void ShenandoahHeapRegion::report_illegal_transition(const char *method) {
83 ResourceMark rm;
84 stringStream ss;
85 ss.print("Illegal region state transition from \"%s\", at %s\n ", region_state_to_string(_state), method);
86 print_on(&ss);
87 fatal("%s", ss.as_string());
88 }
89
90 void ShenandoahHeapRegion::make_regular_allocation() {
91 _heap->assert_heaplock_owned_by_current_thread();
92
93 switch (_state) {
94 case _empty_uncommitted:
95 do_commit();
96 case _empty_committed:
97 set_state(_regular);
98 case _regular:
99 case _pinned:
100 return;
101 default:
102 report_illegal_transition("regular allocation");
103 }
104 }
105
106 void ShenandoahHeapRegion::make_regular_bypass() {
107 _heap->assert_heaplock_owned_by_current_thread();
108 assert (_heap->is_full_gc_in_progress() || _heap->is_degenerated_gc_in_progress(),
109 "only for full or degen GC");
110
111 switch (_state) {
112 case _empty_uncommitted:
113 do_commit();
114 case _empty_committed:
115 case _cset:
116 case _humongous_start:
117 case _humongous_cont:
118 set_state(_regular);
119 return;
120 case _pinned_cset:
121 set_state(_pinned);
122 return;
123 case _regular:
124 case _pinned:
125 return;
126 default:
127 report_illegal_transition("regular bypass");
128 }
129 }
130
131 void ShenandoahHeapRegion::make_humongous_start() {
132 _heap->assert_heaplock_owned_by_current_thread();
133 switch (_state) {
134 case _empty_uncommitted:
135 do_commit();
136 case _empty_committed:
137 set_state(_humongous_start);
138 return;
139 default:
140 report_illegal_transition("humongous start allocation");
141 }
142 }
143
144 void ShenandoahHeapRegion::make_humongous_start_bypass() {
145 _heap->assert_heaplock_owned_by_current_thread();
146 assert (_heap->is_full_gc_in_progress(), "only for full GC");
147
148 switch (_state) {
149 case _empty_committed:
150 case _regular:
151 case _humongous_start:
152 case _humongous_cont:
153 set_state(_humongous_start);
154 return;
155 default:
156 report_illegal_transition("humongous start bypass");
157 }
158 }
159
160 void ShenandoahHeapRegion::make_humongous_cont() {
161 _heap->assert_heaplock_owned_by_current_thread();
162 switch (_state) {
163 case _empty_uncommitted:
164 do_commit();
165 case _empty_committed:
166 set_state(_humongous_cont);
167 return;
168 default:
169 report_illegal_transition("humongous continuation allocation");
170 }
171 }
172
173 void ShenandoahHeapRegion::make_humongous_cont_bypass() {
174 _heap->assert_heaplock_owned_by_current_thread();
175 assert (_heap->is_full_gc_in_progress(), "only for full GC");
176
177 switch (_state) {
178 case _empty_committed:
179 case _regular:
180 case _humongous_start:
181 case _humongous_cont:
182 set_state(_humongous_cont);
183 return;
184 default:
185 report_illegal_transition("humongous continuation bypass");
186 }
187 }
188
189 void ShenandoahHeapRegion::make_pinned() {
190 _heap->assert_heaplock_owned_by_current_thread();
191 switch (_state) {
192 case _regular:
193 assert (_critical_pins == 0, "sanity");
194 set_state(_pinned);
195 case _pinned_cset:
196 case _pinned:
197 _critical_pins++;
198 return;
199 case _humongous_start:
200 assert (_critical_pins == 0, "sanity");
201 set_state(_pinned_humongous_start);
202 case _pinned_humongous_start:
203 _critical_pins++;
204 return;
205 case _cset:
206 guarantee(_heap->cancelled_gc(), "only valid when evac has been cancelled");
207 assert (_critical_pins == 0, "sanity");
208 _state = _pinned_cset;
209 _critical_pins++;
210 return;
211 default:
212 report_illegal_transition("pinning");
213 }
214 }
215
216 void ShenandoahHeapRegion::make_unpinned() {
217 _heap->assert_heaplock_owned_by_current_thread();
218 switch (_state) {
219 case _pinned:
220 assert (_critical_pins > 0, "sanity");
221 _critical_pins--;
222 if (_critical_pins == 0) {
223 set_state(_regular);
224 }
225 return;
226 case _regular:
227 case _humongous_start:
228 assert (_critical_pins == 0, "sanity");
229 return;
230 case _pinned_cset:
231 guarantee(_heap->cancelled_gc(), "only valid when evac has been cancelled");
232 assert (_critical_pins > 0, "sanity");
233 _critical_pins--;
234 if (_critical_pins == 0) {
235 set_state(_cset);
236 }
237 return;
238 case _pinned_humongous_start:
239 assert (_critical_pins > 0, "sanity");
240 _critical_pins--;
241 if (_critical_pins == 0) {
242 set_state(_humongous_start);
243 }
244 return;
245 default:
246 report_illegal_transition("unpinning");
247 }
248 }
249
250 void ShenandoahHeapRegion::make_cset() {
251 _heap->assert_heaplock_owned_by_current_thread();
252 switch (_state) {
253 case _regular:
254 set_state(_cset);
255 case _cset:
256 return;
257 default:
258 report_illegal_transition("cset");
259 }
260 }
261
262 void ShenandoahHeapRegion::make_trash() {
263 _heap->assert_heaplock_owned_by_current_thread();
264 switch (_state) {
265 case _cset:
266 // Reclaiming cset regions
267 case _humongous_start:
268 case _humongous_cont:
269 // Reclaiming humongous regions
270 case _regular:
271 // Immediate region reclaim
272 set_state(_trash);
273 return;
274 default:
275 report_illegal_transition("trashing");
276 }
277 }
278
279 void ShenandoahHeapRegion::make_trash_immediate() {
280 make_trash();
281
282 // On this path, we know there are no marked objects in the region,
283 // tell marking context about it to bypass bitmap resets.
284 _heap->complete_marking_context()->reset_top_bitmap(this);
285 }
286
287 void ShenandoahHeapRegion::make_empty() {
288 _heap->assert_heaplock_owned_by_current_thread();
289 switch (_state) {
290 case _trash:
291 set_state(_empty_committed);
292 _empty_time = os::elapsedTime();
293 return;
294 default:
295 report_illegal_transition("emptying");
296 }
297 }
298
299 void ShenandoahHeapRegion::make_uncommitted() {
300 _heap->assert_heaplock_owned_by_current_thread();
301 switch (_state) {
302 case _empty_committed:
303 do_uncommit();
304 set_state(_empty_uncommitted);
305 return;
306 default:
307 report_illegal_transition("uncommiting");
308 }
309 }
310
311 void ShenandoahHeapRegion::make_committed_bypass() {
312 _heap->assert_heaplock_owned_by_current_thread();
313 assert (_heap->is_full_gc_in_progress(), "only for full GC");
314
315 switch (_state) {
316 case _empty_uncommitted:
317 do_commit();
318 set_state(_empty_committed);
319 return;
320 default:
321 report_illegal_transition("commit bypass");
322 }
323 }
324
325 void ShenandoahHeapRegion::clear_live_data() {
326 OrderAccess::release_store_fence<size_t>(&_live_data, 0);
327 }
328
329 void ShenandoahHeapRegion::reset_alloc_metadata() {
330 _tlab_allocs = 0;
331 _gclab_allocs = 0;
332 _shared_allocs = 0;
333 _seqnum_first_alloc_mutator = 0;
334 _seqnum_last_alloc_mutator = 0;
335 _seqnum_first_alloc_gc = 0;
336 _seqnum_last_alloc_gc = 0;
337 }
338
339 void ShenandoahHeapRegion::reset_alloc_metadata_to_shared() {
340 if (used() > 0) {
341 _tlab_allocs = 0;
342 _gclab_allocs = 0;
343 _shared_allocs = used() >> LogHeapWordSize;
344 uint64_t next = _alloc_seq_num.value++;
345 _seqnum_first_alloc_mutator = next;
346 _seqnum_last_alloc_mutator = next;
347 _seqnum_first_alloc_gc = 0;
348 _seqnum_last_alloc_gc = 0;
349 } else {
350 reset_alloc_metadata();
351 }
352 }
353
354 size_t ShenandoahHeapRegion::get_shared_allocs() const {
355 return _shared_allocs * HeapWordSize;
356 }
357
358 size_t ShenandoahHeapRegion::get_tlab_allocs() const {
359 return _tlab_allocs * HeapWordSize;
360 }
361
362 size_t ShenandoahHeapRegion::get_gclab_allocs() const {
363 return _gclab_allocs * HeapWordSize;
364 }
365
366 void ShenandoahHeapRegion::set_live_data(size_t s) {
367 assert(Thread::current()->is_VM_thread(), "by VM thread");
368 _live_data = (s >> LogHeapWordSize);
369 }
370
371 size_t ShenandoahHeapRegion::get_live_data_words() const {
372 return OrderAccess::load_acquire(&_live_data);
373 }
374
375 size_t ShenandoahHeapRegion::get_live_data_bytes() const {
376 return get_live_data_words() * HeapWordSize;
377 }
378
379 bool ShenandoahHeapRegion::has_live() const {
380 return get_live_data_words() != 0;
381 }
382
383 size_t ShenandoahHeapRegion::garbage() const {
384 assert(used() >= get_live_data_bytes(), "Live Data must be a subset of used() live: " SIZE_FORMAT " used: " SIZE_FORMAT,
385 get_live_data_bytes(), used());
386
387 size_t result = used() - get_live_data_bytes();
388 return result;
389 }
390
391 void ShenandoahHeapRegion::print_on(outputStream* st) const {
392 st->print("|");
393 st->print(SIZE_FORMAT_W(5), this->_region_number);
394
395 switch (_state) {
396 case _empty_uncommitted:
397 st->print("|EU ");
398 break;
399 case _empty_committed:
400 st->print("|EC ");
401 break;
402 case _regular:
403 st->print("|R ");
404 break;
405 case _humongous_start:
406 st->print("|H ");
407 break;
408 case _pinned_humongous_start:
409 st->print("|HP ");
410 break;
411 case _humongous_cont:
412 st->print("|HC ");
413 break;
414 case _cset:
415 st->print("|CS ");
416 break;
417 case _trash:
418 st->print("|T ");
419 break;
420 case _pinned:
421 st->print("|P ");
422 break;
423 case _pinned_cset:
424 st->print("|CSP");
425 break;
426 default:
427 ShouldNotReachHere();
428 }
429 st->print("|BTE " INTPTR_FORMAT_W(12) ", " INTPTR_FORMAT_W(12) ", " INTPTR_FORMAT_W(12),
430 p2i(bottom()), p2i(top()), p2i(end()));
431 st->print("|TAMS " INTPTR_FORMAT_W(12),
432 p2i(_heap->marking_context()->top_at_mark_start(const_cast<ShenandoahHeapRegion*>(this))));
433 st->print("|U " SIZE_FORMAT_W(5) "%1s", byte_size_in_proper_unit(used()), proper_unit_for_byte_size(used()));
434 st->print("|T " SIZE_FORMAT_W(5) "%1s", byte_size_in_proper_unit(get_tlab_allocs()), proper_unit_for_byte_size(get_tlab_allocs()));
435 st->print("|G " SIZE_FORMAT_W(5) "%1s", byte_size_in_proper_unit(get_gclab_allocs()), proper_unit_for_byte_size(get_gclab_allocs()));
436 st->print("|S " SIZE_FORMAT_W(5) "%1s", byte_size_in_proper_unit(get_shared_allocs()), proper_unit_for_byte_size(get_shared_allocs()));
437 st->print("|L " SIZE_FORMAT_W(5) "%1s", byte_size_in_proper_unit(get_live_data_bytes()), proper_unit_for_byte_size(get_live_data_bytes()));
438 st->print("|CP " SIZE_FORMAT_W(3), _critical_pins);
439 st->print("|SN " UINT64_FORMAT_X_W(12) ", " UINT64_FORMAT_X_W(8) ", " UINT64_FORMAT_X_W(8) ", " UINT64_FORMAT_X_W(8),
440 seqnum_first_alloc_mutator(), seqnum_last_alloc_mutator(),
441 seqnum_first_alloc_gc(), seqnum_last_alloc_gc());
442 st->cr();
443 }
444
445 void ShenandoahHeapRegion::oop_iterate(OopIterateClosure* blk) {
446 if (!is_active()) return;
447 if (is_humongous()) {
448 oop_iterate_humongous(blk);
449 } else {
450 oop_iterate_objects(blk);
451 }
452 }
453
454 void ShenandoahHeapRegion::oop_iterate_objects(OopIterateClosure* blk) {
455 assert(! is_humongous(), "no humongous region here");
456 HeapWord* obj_addr = bottom() + ShenandoahForwarding::word_size();
457 HeapWord* t = top();
458 // Could call objects iterate, but this is easier.
459 while (obj_addr < t) {
460 oop obj = oop(obj_addr);
461 obj_addr += obj->oop_iterate_size(blk) + ShenandoahForwarding::word_size();
462 }
463 }
464
465 void ShenandoahHeapRegion::oop_iterate_humongous(OopIterateClosure* blk) {
466 assert(is_humongous(), "only humongous region here");
467 // Find head.
468 ShenandoahHeapRegion* r = humongous_start_region();
469 assert(r->is_humongous_start(), "need humongous head here");
470 oop obj = oop(r->bottom() + ShenandoahForwarding::word_size());
471 obj->oop_iterate(blk, MemRegion(bottom(), top()));
472 }
473
474 ShenandoahHeapRegion* ShenandoahHeapRegion::humongous_start_region() const {
475 assert(is_humongous(), "Must be a part of the humongous region");
476 size_t reg_num = region_number();
477 ShenandoahHeapRegion* r = const_cast<ShenandoahHeapRegion*>(this);
478 while (!r->is_humongous_start()) {
479 assert(reg_num > 0, "Sanity");
480 reg_num --;
481 r = _heap->get_region(reg_num);
482 assert(r->is_humongous(), "Must be a part of the humongous region");
483 }
484 assert(r->is_humongous_start(), "Must be");
485 return r;
486 }
487
488 void ShenandoahHeapRegion::recycle() {
489 ContiguousSpace::clear(false);
490 if (ZapUnusedHeapArea) {
491 ContiguousSpace::mangle_unused_area_complete();
492 }
493 clear_live_data();
494
495 reset_alloc_metadata();
496
497 _heap->marking_context()->reset_top_at_mark_start(this);
498
499 make_empty();
500 }
501
502 HeapWord* ShenandoahHeapRegion::block_start_const(const void* p) const {
503 assert(MemRegion(bottom(), end()).contains(p),
504 "p (" PTR_FORMAT ") not in space [" PTR_FORMAT ", " PTR_FORMAT ")",
505 p2i(p), p2i(bottom()), p2i(end()));
506 if (p >= top()) {
507 return top();
508 } else {
509 HeapWord* last = bottom() + ShenandoahForwarding::word_size();
510 HeapWord* cur = last;
511 while (cur <= p) {
512 last = cur;
513 cur += oop(cur)->size() + ShenandoahForwarding::word_size();
514 }
515 shenandoah_assert_correct(NULL, oop(last));
516 return last;
517 }
518 }
519
520 void ShenandoahHeapRegion::setup_sizes(size_t initial_heap_size, size_t max_heap_size) {
521 // Absolute minimums we should not ever break.
522 static const size_t MIN_REGION_SIZE = 256*K;
523
524 if (FLAG_IS_DEFAULT(ShenandoahMinRegionSize)) {
525 FLAG_SET_DEFAULT(ShenandoahMinRegionSize, MIN_REGION_SIZE);
526 }
527
528 size_t region_size;
529 if (FLAG_IS_DEFAULT(ShenandoahHeapRegionSize)) {
530 if (ShenandoahMinRegionSize > initial_heap_size / MIN_NUM_REGIONS) {
531 err_msg message("Initial heap size (" SIZE_FORMAT "K) is too low to afford the minimum number "
532 "of regions (" SIZE_FORMAT ") of minimum region size (" SIZE_FORMAT "K).",
533 initial_heap_size/K, MIN_NUM_REGIONS, ShenandoahMinRegionSize/K);
534 vm_exit_during_initialization("Invalid -XX:ShenandoahMinRegionSize option", message);
535 }
536 if (ShenandoahMinRegionSize < MIN_REGION_SIZE) {
537 err_msg message("" SIZE_FORMAT "K should not be lower than minimum region size (" SIZE_FORMAT "K).",
538 ShenandoahMinRegionSize/K, MIN_REGION_SIZE/K);
539 vm_exit_during_initialization("Invalid -XX:ShenandoahMinRegionSize option", message);
540 }
541 if (ShenandoahMinRegionSize < MinTLABSize) {
542 err_msg message("" SIZE_FORMAT "K should not be lower than TLAB size size (" SIZE_FORMAT "K).",
543 ShenandoahMinRegionSize/K, MinTLABSize/K);
544 vm_exit_during_initialization("Invalid -XX:ShenandoahMinRegionSize option", message);
545 }
546 if (ShenandoahMaxRegionSize < MIN_REGION_SIZE) {
547 err_msg message("" SIZE_FORMAT "K should not be lower than min region size (" SIZE_FORMAT "K).",
548 ShenandoahMaxRegionSize/K, MIN_REGION_SIZE/K);
549 vm_exit_during_initialization("Invalid -XX:ShenandoahMaxRegionSize option", message);
550 }
551 if (ShenandoahMinRegionSize > ShenandoahMaxRegionSize) {
552 err_msg message("Minimum (" SIZE_FORMAT "K) should be larger than maximum (" SIZE_FORMAT "K).",
553 ShenandoahMinRegionSize/K, ShenandoahMaxRegionSize/K);
554 vm_exit_during_initialization("Invalid -XX:ShenandoahMinRegionSize or -XX:ShenandoahMaxRegionSize", message);
555 }
556
557 // We rapidly expand to max_heap_size in most scenarios, so that is the measure
558 // for usual heap sizes. Do not depend on initial_heap_size here.
559 region_size = max_heap_size / ShenandoahTargetNumRegions;
560
561 // Now make sure that we don't go over or under our limits.
562 region_size = MAX2(ShenandoahMinRegionSize, region_size);
563 region_size = MIN2(ShenandoahMaxRegionSize, region_size);
564
565 } else {
566 if (ShenandoahHeapRegionSize > initial_heap_size / MIN_NUM_REGIONS) {
567 err_msg message("Initial heap size (" SIZE_FORMAT "K) is too low to afford the minimum number "
568 "of regions (" SIZE_FORMAT ") of requested size (" SIZE_FORMAT "K).",
569 initial_heap_size/K, MIN_NUM_REGIONS, ShenandoahHeapRegionSize/K);
570 vm_exit_during_initialization("Invalid -XX:ShenandoahHeapRegionSize option", message);
571 }
572 if (ShenandoahHeapRegionSize < ShenandoahMinRegionSize) {
573 err_msg message("Heap region size (" SIZE_FORMAT "K) should be larger than min region size (" SIZE_FORMAT "K).",
574 ShenandoahHeapRegionSize/K, ShenandoahMinRegionSize/K);
575 vm_exit_during_initialization("Invalid -XX:ShenandoahHeapRegionSize option", message);
576 }
577 if (ShenandoahHeapRegionSize > ShenandoahMaxRegionSize) {
578 err_msg message("Heap region size (" SIZE_FORMAT "K) should be lower than max region size (" SIZE_FORMAT "K).",
579 ShenandoahHeapRegionSize/K, ShenandoahMaxRegionSize/K);
580 vm_exit_during_initialization("Invalid -XX:ShenandoahHeapRegionSize option", message);
581 }
582 region_size = ShenandoahHeapRegionSize;
583 }
584
585 // Make sure region size is at least one large page, if enabled.
586 // Otherwise, uncommitting one region may falsely uncommit the adjacent
587 // regions too.
588 // Also see shenandoahArguments.cpp, where it handles UseLargePages.
589 if (UseLargePages && ShenandoahUncommit) {
590 region_size = MAX2(region_size, os::large_page_size());
591 }
592
593 int region_size_log = log2_long((jlong) region_size);
594 // Recalculate the region size to make sure it's a power of
595 // 2. This means that region_size is the largest power of 2 that's
596 // <= what we've calculated so far.
597 region_size = size_t(1) << region_size_log;
598
599 // Now, set up the globals.
600 guarantee(RegionSizeBytesShift == 0, "we should only set it once");
601 RegionSizeBytesShift = (size_t)region_size_log;
602
603 guarantee(RegionSizeWordsShift == 0, "we should only set it once");
604 RegionSizeWordsShift = RegionSizeBytesShift - LogHeapWordSize;
605
606 guarantee(RegionSizeBytes == 0, "we should only set it once");
607 RegionSizeBytes = region_size;
608 RegionSizeWords = RegionSizeBytes >> LogHeapWordSize;
609 assert (RegionSizeWords*HeapWordSize == RegionSizeBytes, "sanity");
610
611 guarantee(RegionSizeWordsMask == 0, "we should only set it once");
612 RegionSizeWordsMask = RegionSizeWords - 1;
613
614 guarantee(RegionSizeBytesMask == 0, "we should only set it once");
615 RegionSizeBytesMask = RegionSizeBytes - 1;
616
617 guarantee(RegionCount == 0, "we should only set it once");
618 RegionCount = max_heap_size / RegionSizeBytes;
619 guarantee(RegionCount >= MIN_NUM_REGIONS, "Should have at least minimum regions");
620
621 guarantee(HumongousThresholdWords == 0, "we should only set it once");
622 HumongousThresholdWords = RegionSizeWords * ShenandoahHumongousThreshold / 100;
623 HumongousThresholdWords = align_down(HumongousThresholdWords, MinObjAlignment);
624 assert (HumongousThresholdWords <= RegionSizeWords, "sanity");
625
626 guarantee(HumongousThresholdBytes == 0, "we should only set it once");
627 HumongousThresholdBytes = HumongousThresholdWords * HeapWordSize;
628 assert (HumongousThresholdBytes <= RegionSizeBytes, "sanity");
629
630 // The rationale for trimming the TLAB sizes has to do with the raciness in
631 // TLAB allocation machinery. It may happen that TLAB sizing policy polls Shenandoah
632 // about next free size, gets the answer for region #N, goes away for a while, then
633 // tries to allocate in region #N, and fail because some other thread have claimed part
634 // of the region #N, and then the freeset allocation code has to retire the region #N,
635 // before moving the allocation to region #N+1.
636 //
637 // The worst case realizes when "answer" is "region size", which means it could
638 // prematurely retire an entire region. Having smaller TLABs does not fix that
639 // completely, but reduces the probability of too wasteful region retirement.
640 // With current divisor, we will waste no more than 1/8 of region size in the worst
641 // case. This also has a secondary effect on collection set selection: even under
642 // the race, the regions would be at least 7/8 used, which allows relying on
643 // "used" - "live" for cset selection. Otherwise, we can get the fragmented region
644 // below the garbage threshold that would never be considered for collection.
645 //
646 // The whole thing is mitigated if Elastic TLABs are enabled.
647 //
648 guarantee(MaxTLABSizeWords == 0, "we should only set it once");
649 MaxTLABSizeWords = MIN2(ShenandoahElasticTLAB ? RegionSizeWords : (RegionSizeWords / 8), HumongousThresholdWords);
650 MaxTLABSizeWords = align_down(MaxTLABSizeWords, MinObjAlignment);
651
652 guarantee(MaxTLABSizeBytes == 0, "we should only set it once");
653 MaxTLABSizeBytes = MaxTLABSizeWords * HeapWordSize;
654 assert (MaxTLABSizeBytes > MinTLABSize, "should be larger");
655
656 log_info(gc, init)("Regions: " SIZE_FORMAT " x " SIZE_FORMAT "%s",
657 RegionCount, byte_size_in_proper_unit(RegionSizeBytes), proper_unit_for_byte_size(RegionSizeBytes));
658 log_info(gc, init)("Humongous object threshold: " SIZE_FORMAT "%s",
659 byte_size_in_proper_unit(HumongousThresholdBytes), proper_unit_for_byte_size(HumongousThresholdBytes));
660 log_info(gc, init)("Max TLAB size: " SIZE_FORMAT "%s",
661 byte_size_in_proper_unit(MaxTLABSizeBytes), proper_unit_for_byte_size(MaxTLABSizeBytes));
662 }
663
664 void ShenandoahHeapRegion::do_commit() {
665 if (!_heap->is_heap_region_special() && !os::commit_memory((char *) _reserved.start(), _reserved.byte_size(), false)) {
666 report_java_out_of_memory("Unable to commit region");
667 }
668 if (!_heap->commit_bitmap_slice(this)) {
669 report_java_out_of_memory("Unable to commit bitmaps for region");
670 }
671 _heap->increase_committed(ShenandoahHeapRegion::region_size_bytes());
672 }
673
674 void ShenandoahHeapRegion::do_uncommit() {
675 if (!_heap->is_heap_region_special() && !os::uncommit_memory((char *) _reserved.start(), _reserved.byte_size())) {
676 report_java_out_of_memory("Unable to uncommit region");
677 }
678 if (!_heap->uncommit_bitmap_slice(this)) {
679 report_java_out_of_memory("Unable to uncommit bitmaps for region");
680 }
681 _heap->decrease_committed(ShenandoahHeapRegion::region_size_bytes());
682 }
683
684 void ShenandoahHeapRegion::set_state(RegionState to) {
685 EventShenandoahHeapRegionStateChange evt;
686 if (evt.should_commit()){
687 evt.set_index(region_number());
688 evt.set_start((uintptr_t)bottom());
689 evt.set_used(used());
690 evt.set_from(_state);
691 evt.set_to(to);
692 evt.commit();
693 }
694 _state = to;
695 }