1 /*
2 * Copyright (c) 2013, 2019, Red Hat, Inc. All rights reserved.
3 *
4 * This code is free software; you can redistribute it and/or modify it
5 * under the terms of the GNU General Public License version 2 only, as
6 * published by the Free Software Foundation.
7 *
8 * This code is distributed in the hope that it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
11 * version 2 for more details (a copy is included in the LICENSE file that
12 * accompanied this code).
13 *
14 * You should have received a copy of the GNU General Public License version
15 * 2 along with this work; if not, write to the Free Software Foundation,
16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
17 *
18 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
19 * or visit www.oracle.com if you need additional information or have any
20 * questions.
21 *
22 */
23
24 #include "precompiled.hpp"
25 #include "memory/allocation.hpp"
26 #include "gc/shenandoah/shenandoahHeapRegionSet.inline.hpp"
27 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
28 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
29 #include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"
30 #include "gc/shenandoah/shenandoahTraversalGC.hpp"
31 #include "gc/shared/space.inline.hpp"
32 #include "jfr/jfrEvents.hpp"
33 #include "memory/iterator.inline.hpp"
34 #include "memory/resourceArea.hpp"
35 #include "memory/universe.hpp"
36 #include "oops/oop.inline.hpp"
37 #include "runtime/java.hpp"
38 #include "runtime/mutexLocker.hpp"
39 #include "runtime/os.hpp"
40 #include "runtime/safepoint.hpp"
41
42 size_t ShenandoahHeapRegion::RegionCount = 0;
43 size_t ShenandoahHeapRegion::RegionSizeBytes = 0;
44 size_t ShenandoahHeapRegion::RegionSizeWords = 0;
45 size_t ShenandoahHeapRegion::RegionSizeBytesShift = 0;
46 size_t ShenandoahHeapRegion::RegionSizeWordsShift = 0;
47 size_t ShenandoahHeapRegion::RegionSizeBytesMask = 0;
48 size_t ShenandoahHeapRegion::RegionSizeWordsMask = 0;
49 size_t ShenandoahHeapRegion::HumongousThresholdBytes = 0;
50 size_t ShenandoahHeapRegion::HumongousThresholdWords = 0;
51 size_t ShenandoahHeapRegion::MaxTLABSizeBytes = 0;
52 size_t ShenandoahHeapRegion::MaxTLABSizeWords = 0;
53
54 ShenandoahHeapRegion::PaddedAllocSeqNum ShenandoahHeapRegion::_alloc_seq_num;
55
56 ShenandoahHeapRegion::ShenandoahHeapRegion(ShenandoahHeap* heap, HeapWord* start,
57 size_t size_words, size_t index, bool committed) :
58 _heap(heap),
59 _reserved(MemRegion(start, size_words)),
60 _region_number(index),
61 _new_top(NULL),
62 _critical_pins(0),
63 _empty_time(os::elapsedTime()),
64 _state(committed ? _empty_committed : _empty_uncommitted),
65 _tlab_allocs(0),
66 _gclab_allocs(0),
67 _shared_allocs(0),
68 _seqnum_first_alloc_mutator(0),
69 _seqnum_first_alloc_gc(0),
70 _seqnum_last_alloc_mutator(0),
71 _seqnum_last_alloc_gc(0),
72 _live_data(0) {
73
74 ContiguousSpace::initialize(_reserved, true, committed);
75 }
76
77 size_t ShenandoahHeapRegion::region_number() const {
78 return _region_number;
79 }
80
81 void ShenandoahHeapRegion::report_illegal_transition(const char *method) {
82 ResourceMark rm;
83 stringStream ss;
84 ss.print("Illegal region state transition from \"%s\", at %s\n ", region_state_to_string(_state), method);
85 print_on(&ss);
86 fatal("%s", ss.as_string());
87 }
88
89 void ShenandoahHeapRegion::make_regular_allocation() {
90 _heap->assert_heaplock_owned_by_current_thread();
91
92 switch (_state) {
93 case _empty_uncommitted:
94 do_commit();
95 case _empty_committed:
96 set_state(_regular);
97 case _regular:
98 case _pinned:
99 return;
100 default:
101 report_illegal_transition("regular allocation");
102 }
103 }
104
105 void ShenandoahHeapRegion::make_regular_bypass() {
106 _heap->assert_heaplock_owned_by_current_thread();
107 assert (_heap->is_full_gc_in_progress() || _heap->is_degenerated_gc_in_progress(),
108 "only for full or degen GC");
109
110 switch (_state) {
111 case _empty_uncommitted:
112 do_commit();
113 case _empty_committed:
114 case _cset:
115 case _humongous_start:
116 case _humongous_cont:
117 set_state(_regular);
118 return;
119 case _pinned_cset:
120 set_state(_pinned);
121 return;
122 case _regular:
123 case _pinned:
124 return;
125 default:
126 report_illegal_transition("regular bypass");
127 }
128 }
129
130 void ShenandoahHeapRegion::make_humongous_start() {
131 _heap->assert_heaplock_owned_by_current_thread();
132 switch (_state) {
133 case _empty_uncommitted:
134 do_commit();
135 case _empty_committed:
136 set_state(_humongous_start);
137 return;
138 default:
139 report_illegal_transition("humongous start allocation");
140 }
141 }
142
143 void ShenandoahHeapRegion::make_humongous_start_bypass() {
144 _heap->assert_heaplock_owned_by_current_thread();
145 assert (_heap->is_full_gc_in_progress(), "only for full GC");
146
147 switch (_state) {
148 case _empty_committed:
149 case _regular:
150 case _humongous_start:
151 case _humongous_cont:
152 set_state(_humongous_start);
153 return;
154 default:
155 report_illegal_transition("humongous start bypass");
156 }
157 }
158
159 void ShenandoahHeapRegion::make_humongous_cont() {
160 _heap->assert_heaplock_owned_by_current_thread();
161 switch (_state) {
162 case _empty_uncommitted:
163 do_commit();
164 case _empty_committed:
165 set_state(_humongous_cont);
166 return;
167 default:
168 report_illegal_transition("humongous continuation allocation");
169 }
170 }
171
172 void ShenandoahHeapRegion::make_humongous_cont_bypass() {
173 _heap->assert_heaplock_owned_by_current_thread();
174 assert (_heap->is_full_gc_in_progress(), "only for full GC");
175
176 switch (_state) {
177 case _empty_committed:
178 case _regular:
179 case _humongous_start:
180 case _humongous_cont:
181 set_state(_humongous_cont);
182 return;
183 default:
184 report_illegal_transition("humongous continuation bypass");
185 }
186 }
187
188 void ShenandoahHeapRegion::make_pinned() {
189 _heap->assert_heaplock_owned_by_current_thread();
190 switch (_state) {
191 case _regular:
192 assert (_critical_pins == 0, "sanity");
193 set_state(_pinned);
194 case _pinned_cset:
195 case _pinned:
196 _critical_pins++;
197 return;
198 case _humongous_start:
199 assert (_critical_pins == 0, "sanity");
200 set_state(_pinned_humongous_start);
201 case _pinned_humongous_start:
202 _critical_pins++;
203 return;
204 case _cset:
205 guarantee(_heap->cancelled_gc(), "only valid when evac has been cancelled");
206 assert (_critical_pins == 0, "sanity");
207 _state = _pinned_cset;
208 _critical_pins++;
209 return;
210 default:
211 report_illegal_transition("pinning");
212 }
213 }
214
215 void ShenandoahHeapRegion::make_unpinned() {
216 _heap->assert_heaplock_owned_by_current_thread();
217 switch (_state) {
218 case _pinned:
219 assert (_critical_pins > 0, "sanity");
220 _critical_pins--;
221 if (_critical_pins == 0) {
222 set_state(_regular);
223 }
224 return;
225 case _regular:
226 case _humongous_start:
227 assert (_critical_pins == 0, "sanity");
228 return;
229 case _pinned_cset:
230 guarantee(_heap->cancelled_gc(), "only valid when evac has been cancelled");
231 assert (_critical_pins > 0, "sanity");
232 _critical_pins--;
233 if (_critical_pins == 0) {
234 set_state(_cset);
235 }
236 return;
237 case _pinned_humongous_start:
238 assert (_critical_pins > 0, "sanity");
239 _critical_pins--;
240 if (_critical_pins == 0) {
241 set_state(_humongous_start);
242 }
243 return;
244 default:
245 report_illegal_transition("unpinning");
246 }
247 }
248
249 void ShenandoahHeapRegion::make_cset() {
250 _heap->assert_heaplock_owned_by_current_thread();
251 switch (_state) {
252 case _regular:
253 set_state(_cset);
254 case _cset:
255 return;
256 default:
257 report_illegal_transition("cset");
258 }
259 }
260
261 void ShenandoahHeapRegion::make_trash() {
262 _heap->assert_heaplock_owned_by_current_thread();
263 switch (_state) {
264 case _cset:
265 // Reclaiming cset regions
266 case _humongous_start:
267 case _humongous_cont:
268 // Reclaiming humongous regions
269 case _regular:
270 // Immediate region reclaim
271 set_state(_trash);
272 return;
273 default:
274 report_illegal_transition("trashing");
275 }
276 }
277
278 void ShenandoahHeapRegion::make_trash_immediate() {
279 make_trash();
280
281 // On this path, we know there are no marked objects in the region,
282 // tell marking context about it to bypass bitmap resets.
283 _heap->complete_marking_context()->reset_top_bitmap(this);
284 }
285
286 void ShenandoahHeapRegion::make_empty() {
287 _heap->assert_heaplock_owned_by_current_thread();
288 switch (_state) {
289 case _trash:
290 set_state(_empty_committed);
291 _empty_time = os::elapsedTime();
292 return;
293 default:
294 report_illegal_transition("emptying");
295 }
296 }
297
298 void ShenandoahHeapRegion::make_uncommitted() {
299 _heap->assert_heaplock_owned_by_current_thread();
300 switch (_state) {
301 case _empty_committed:
302 do_uncommit();
303 set_state(_empty_uncommitted);
304 return;
305 default:
306 report_illegal_transition("uncommiting");
307 }
308 }
309
310 void ShenandoahHeapRegion::make_committed_bypass() {
311 _heap->assert_heaplock_owned_by_current_thread();
312 assert (_heap->is_full_gc_in_progress(), "only for full GC");
313
314 switch (_state) {
315 case _empty_uncommitted:
316 do_commit();
317 set_state(_empty_committed);
318 return;
319 default:
320 report_illegal_transition("commit bypass");
321 }
322 }
323
324 void ShenandoahHeapRegion::clear_live_data() {
325 OrderAccess::release_store_fence<size_t>(&_live_data, 0);
326 }
327
328 void ShenandoahHeapRegion::reset_alloc_metadata() {
329 _tlab_allocs = 0;
330 _gclab_allocs = 0;
331 _shared_allocs = 0;
332 _seqnum_first_alloc_mutator = 0;
333 _seqnum_last_alloc_mutator = 0;
334 _seqnum_first_alloc_gc = 0;
335 _seqnum_last_alloc_gc = 0;
336 }
337
338 void ShenandoahHeapRegion::reset_alloc_metadata_to_shared() {
339 if (used() > 0) {
340 _tlab_allocs = 0;
341 _gclab_allocs = 0;
342 _shared_allocs = used() >> LogHeapWordSize;
343 uint64_t next = _alloc_seq_num.value++;
344 _seqnum_first_alloc_mutator = next;
345 _seqnum_last_alloc_mutator = next;
346 _seqnum_first_alloc_gc = 0;
347 _seqnum_last_alloc_gc = 0;
348 } else {
349 reset_alloc_metadata();
350 }
351 }
352
353 size_t ShenandoahHeapRegion::get_shared_allocs() const {
354 return _shared_allocs * HeapWordSize;
355 }
356
357 size_t ShenandoahHeapRegion::get_tlab_allocs() const {
358 return _tlab_allocs * HeapWordSize;
359 }
360
361 size_t ShenandoahHeapRegion::get_gclab_allocs() const {
362 return _gclab_allocs * HeapWordSize;
363 }
364
365 void ShenandoahHeapRegion::set_live_data(size_t s) {
366 assert(Thread::current()->is_VM_thread(), "by VM thread");
367 _live_data = (s >> LogHeapWordSize);
368 }
369
370 size_t ShenandoahHeapRegion::get_live_data_words() const {
371 return OrderAccess::load_acquire(&_live_data);
372 }
373
374 size_t ShenandoahHeapRegion::get_live_data_bytes() const {
375 return get_live_data_words() * HeapWordSize;
376 }
377
378 bool ShenandoahHeapRegion::has_live() const {
379 return get_live_data_words() != 0;
380 }
381
382 size_t ShenandoahHeapRegion::garbage() const {
383 assert(used() >= get_live_data_bytes(), "Live Data must be a subset of used() live: " SIZE_FORMAT " used: " SIZE_FORMAT,
384 get_live_data_bytes(), used());
385
386 size_t result = used() - get_live_data_bytes();
387 return result;
388 }
389
390 void ShenandoahHeapRegion::print_on(outputStream* st) const {
391 st->print("|");
392 st->print(SIZE_FORMAT_W(5), this->_region_number);
393
394 switch (_state) {
395 case _empty_uncommitted:
396 st->print("|EU ");
397 break;
398 case _empty_committed:
399 st->print("|EC ");
400 break;
401 case _regular:
402 st->print("|R ");
403 break;
404 case _humongous_start:
405 st->print("|H ");
406 break;
407 case _pinned_humongous_start:
408 st->print("|HP ");
409 break;
410 case _humongous_cont:
411 st->print("|HC ");
412 break;
413 case _cset:
414 st->print("|CS ");
415 break;
416 case _trash:
417 st->print("|T ");
418 break;
419 case _pinned:
420 st->print("|P ");
421 break;
422 case _pinned_cset:
423 st->print("|CSP");
424 break;
425 default:
426 ShouldNotReachHere();
427 }
428 st->print("|BTE " INTPTR_FORMAT_W(12) ", " INTPTR_FORMAT_W(12) ", " INTPTR_FORMAT_W(12),
429 p2i(bottom()), p2i(top()), p2i(end()));
430 st->print("|TAMS " INTPTR_FORMAT_W(12),
431 p2i(_heap->marking_context()->top_at_mark_start(const_cast<ShenandoahHeapRegion*>(this))));
432 st->print("|U " SIZE_FORMAT_W(5) "%1s", byte_size_in_proper_unit(used()), proper_unit_for_byte_size(used()));
433 st->print("|T " SIZE_FORMAT_W(5) "%1s", byte_size_in_proper_unit(get_tlab_allocs()), proper_unit_for_byte_size(get_tlab_allocs()));
434 st->print("|G " SIZE_FORMAT_W(5) "%1s", byte_size_in_proper_unit(get_gclab_allocs()), proper_unit_for_byte_size(get_gclab_allocs()));
435 st->print("|S " SIZE_FORMAT_W(5) "%1s", byte_size_in_proper_unit(get_shared_allocs()), proper_unit_for_byte_size(get_shared_allocs()));
436 st->print("|L " SIZE_FORMAT_W(5) "%1s", byte_size_in_proper_unit(get_live_data_bytes()), proper_unit_for_byte_size(get_live_data_bytes()));
437 st->print("|CP " SIZE_FORMAT_W(3), _critical_pins);
438 st->print("|SN " UINT64_FORMAT_X_W(12) ", " UINT64_FORMAT_X_W(8) ", " UINT64_FORMAT_X_W(8) ", " UINT64_FORMAT_X_W(8),
439 seqnum_first_alloc_mutator(), seqnum_last_alloc_mutator(),
440 seqnum_first_alloc_gc(), seqnum_last_alloc_gc());
441 st->cr();
442 }
443
444 void ShenandoahHeapRegion::oop_iterate(OopIterateClosure* blk) {
445 if (!is_active()) return;
446 if (is_humongous()) {
447 oop_iterate_humongous(blk);
448 } else {
449 oop_iterate_objects(blk);
450 }
451 }
452
453 void ShenandoahHeapRegion::oop_iterate_objects(OopIterateClosure* blk) {
454 assert(! is_humongous(), "no humongous region here");
455 HeapWord* obj_addr = bottom();
456 HeapWord* t = top();
457 // Could call objects iterate, but this is easier.
458 while (obj_addr < t) {
459 oop obj = oop(obj_addr);
460 obj_addr += obj->oop_iterate_size(blk);
461 }
462 }
463
464 void ShenandoahHeapRegion::oop_iterate_humongous(OopIterateClosure* blk) {
465 assert(is_humongous(), "only humongous region here");
466 // Find head.
467 ShenandoahHeapRegion* r = humongous_start_region();
468 assert(r->is_humongous_start(), "need humongous head here");
469 oop obj = oop(r->bottom());
470 obj->oop_iterate(blk, MemRegion(bottom(), top()));
471 }
472
473 ShenandoahHeapRegion* ShenandoahHeapRegion::humongous_start_region() const {
474 assert(is_humongous(), "Must be a part of the humongous region");
475 size_t reg_num = region_number();
476 ShenandoahHeapRegion* r = const_cast<ShenandoahHeapRegion*>(this);
477 while (!r->is_humongous_start()) {
478 assert(reg_num > 0, "Sanity");
479 reg_num --;
480 r = _heap->get_region(reg_num);
481 assert(r->is_humongous(), "Must be a part of the humongous region");
482 }
483 assert(r->is_humongous_start(), "Must be");
484 return r;
485 }
486
487 void ShenandoahHeapRegion::recycle() {
488 ContiguousSpace::clear(false);
489 if (ZapUnusedHeapArea) {
490 ContiguousSpace::mangle_unused_area_complete();
491 }
492 clear_live_data();
493
494 reset_alloc_metadata();
495
496 _heap->marking_context()->reset_top_at_mark_start(this);
497
498 make_empty();
499 }
500
501 HeapWord* ShenandoahHeapRegion::block_start_const(const void* p) const {
502 assert(MemRegion(bottom(), end()).contains(p),
503 "p (" PTR_FORMAT ") not in space [" PTR_FORMAT ", " PTR_FORMAT ")",
504 p2i(p), p2i(bottom()), p2i(end()));
505 if (p >= top()) {
506 return top();
507 } else {
508 HeapWord* last = bottom();
509 HeapWord* cur = last;
510 while (cur <= p) {
511 last = cur;
512 cur += oop(cur)->size();
513 }
514 shenandoah_assert_correct(NULL, oop(last));
515 return last;
516 }
517 }
518
519 void ShenandoahHeapRegion::setup_sizes(size_t max_heap_size) {
520 // Absolute minimums we should not ever break.
521 static const size_t MIN_REGION_SIZE = 256*K;
522
523 if (FLAG_IS_DEFAULT(ShenandoahMinRegionSize)) {
524 FLAG_SET_DEFAULT(ShenandoahMinRegionSize, MIN_REGION_SIZE);
525 }
526
527 size_t region_size;
528 if (FLAG_IS_DEFAULT(ShenandoahHeapRegionSize)) {
529 if (ShenandoahMinRegionSize > max_heap_size / MIN_NUM_REGIONS) {
530 err_msg message("Max heap size (" SIZE_FORMAT "K) is too low to afford the minimum number "
531 "of regions (" SIZE_FORMAT ") of minimum region size (" SIZE_FORMAT "K).",
532 max_heap_size/K, MIN_NUM_REGIONS, ShenandoahMinRegionSize/K);
533 vm_exit_during_initialization("Invalid -XX:ShenandoahMinRegionSize option", message);
534 }
535 if (ShenandoahMinRegionSize < MIN_REGION_SIZE) {
536 err_msg message("" SIZE_FORMAT "K should not be lower than minimum region size (" SIZE_FORMAT "K).",
537 ShenandoahMinRegionSize/K, MIN_REGION_SIZE/K);
538 vm_exit_during_initialization("Invalid -XX:ShenandoahMinRegionSize option", message);
539 }
540 if (ShenandoahMinRegionSize < MinTLABSize) {
541 err_msg message("" SIZE_FORMAT "K should not be lower than TLAB size size (" SIZE_FORMAT "K).",
542 ShenandoahMinRegionSize/K, MinTLABSize/K);
543 vm_exit_during_initialization("Invalid -XX:ShenandoahMinRegionSize option", message);
544 }
545 if (ShenandoahMaxRegionSize < MIN_REGION_SIZE) {
546 err_msg message("" SIZE_FORMAT "K should not be lower than min region size (" SIZE_FORMAT "K).",
547 ShenandoahMaxRegionSize/K, MIN_REGION_SIZE/K);
548 vm_exit_during_initialization("Invalid -XX:ShenandoahMaxRegionSize option", message);
549 }
550 if (ShenandoahMinRegionSize > ShenandoahMaxRegionSize) {
551 err_msg message("Minimum (" SIZE_FORMAT "K) should be larger than maximum (" SIZE_FORMAT "K).",
552 ShenandoahMinRegionSize/K, ShenandoahMaxRegionSize/K);
553 vm_exit_during_initialization("Invalid -XX:ShenandoahMinRegionSize or -XX:ShenandoahMaxRegionSize", message);
554 }
555
556 // We rapidly expand to max_heap_size in most scenarios, so that is the measure
557 // for usual heap sizes. Do not depend on initial_heap_size here.
558 region_size = max_heap_size / ShenandoahTargetNumRegions;
559
560 // Now make sure that we don't go over or under our limits.
561 region_size = MAX2(ShenandoahMinRegionSize, region_size);
562 region_size = MIN2(ShenandoahMaxRegionSize, region_size);
563
564 } else {
565 if (ShenandoahHeapRegionSize > max_heap_size / MIN_NUM_REGIONS) {
566 err_msg message("Max heap size (" SIZE_FORMAT "K) is too low to afford the minimum number "
567 "of regions (" SIZE_FORMAT ") of requested size (" SIZE_FORMAT "K).",
568 max_heap_size/K, MIN_NUM_REGIONS, ShenandoahHeapRegionSize/K);
569 vm_exit_during_initialization("Invalid -XX:ShenandoahHeapRegionSize option", message);
570 }
571 if (ShenandoahHeapRegionSize < ShenandoahMinRegionSize) {
572 err_msg message("Heap region size (" SIZE_FORMAT "K) should be larger than min region size (" SIZE_FORMAT "K).",
573 ShenandoahHeapRegionSize/K, ShenandoahMinRegionSize/K);
574 vm_exit_during_initialization("Invalid -XX:ShenandoahHeapRegionSize option", message);
575 }
576 if (ShenandoahHeapRegionSize > ShenandoahMaxRegionSize) {
577 err_msg message("Heap region size (" SIZE_FORMAT "K) should be lower than max region size (" SIZE_FORMAT "K).",
578 ShenandoahHeapRegionSize/K, ShenandoahMaxRegionSize/K);
579 vm_exit_during_initialization("Invalid -XX:ShenandoahHeapRegionSize option", message);
580 }
581 region_size = ShenandoahHeapRegionSize;
582 }
583
584 // Make sure region size is at least one large page, if enabled.
585 // Otherwise, uncommitting one region may falsely uncommit the adjacent
586 // regions too.
587 // Also see shenandoahArguments.cpp, where it handles UseLargePages.
588 if (UseLargePages && ShenandoahUncommit) {
589 region_size = MAX2(region_size, os::large_page_size());
590 }
591
592 int region_size_log = log2_long((jlong) region_size);
593 // Recalculate the region size to make sure it's a power of
594 // 2. This means that region_size is the largest power of 2 that's
595 // <= what we've calculated so far.
596 region_size = size_t(1) << region_size_log;
597
598 // Now, set up the globals.
599 guarantee(RegionSizeBytesShift == 0, "we should only set it once");
600 RegionSizeBytesShift = (size_t)region_size_log;
601
602 guarantee(RegionSizeWordsShift == 0, "we should only set it once");
603 RegionSizeWordsShift = RegionSizeBytesShift - LogHeapWordSize;
604
605 guarantee(RegionSizeBytes == 0, "we should only set it once");
606 RegionSizeBytes = region_size;
607 RegionSizeWords = RegionSizeBytes >> LogHeapWordSize;
608 assert (RegionSizeWords*HeapWordSize == RegionSizeBytes, "sanity");
609
610 guarantee(RegionSizeWordsMask == 0, "we should only set it once");
611 RegionSizeWordsMask = RegionSizeWords - 1;
612
613 guarantee(RegionSizeBytesMask == 0, "we should only set it once");
614 RegionSizeBytesMask = RegionSizeBytes - 1;
615
616 guarantee(RegionCount == 0, "we should only set it once");
617 RegionCount = max_heap_size / RegionSizeBytes;
618 guarantee(RegionCount >= MIN_NUM_REGIONS, "Should have at least minimum regions");
619
620 guarantee(HumongousThresholdWords == 0, "we should only set it once");
621 HumongousThresholdWords = RegionSizeWords * ShenandoahHumongousThreshold / 100;
622 HumongousThresholdWords = align_down(HumongousThresholdWords, MinObjAlignment);
623 assert (HumongousThresholdWords <= RegionSizeWords, "sanity");
624
625 guarantee(HumongousThresholdBytes == 0, "we should only set it once");
626 HumongousThresholdBytes = HumongousThresholdWords * HeapWordSize;
627 assert (HumongousThresholdBytes <= RegionSizeBytes, "sanity");
628
629 // The rationale for trimming the TLAB sizes has to do with the raciness in
630 // TLAB allocation machinery. It may happen that TLAB sizing policy polls Shenandoah
631 // about next free size, gets the answer for region #N, goes away for a while, then
632 // tries to allocate in region #N, and fail because some other thread have claimed part
633 // of the region #N, and then the freeset allocation code has to retire the region #N,
634 // before moving the allocation to region #N+1.
635 //
636 // The worst case realizes when "answer" is "region size", which means it could
637 // prematurely retire an entire region. Having smaller TLABs does not fix that
638 // completely, but reduces the probability of too wasteful region retirement.
639 // With current divisor, we will waste no more than 1/8 of region size in the worst
640 // case. This also has a secondary effect on collection set selection: even under
641 // the race, the regions would be at least 7/8 used, which allows relying on
642 // "used" - "live" for cset selection. Otherwise, we can get the fragmented region
643 // below the garbage threshold that would never be considered for collection.
644 //
645 // The whole thing is mitigated if Elastic TLABs are enabled.
646 //
647 guarantee(MaxTLABSizeWords == 0, "we should only set it once");
648 MaxTLABSizeWords = MIN2(ShenandoahElasticTLAB ? RegionSizeWords : (RegionSizeWords / 8), HumongousThresholdWords);
649 MaxTLABSizeWords = align_down(MaxTLABSizeWords, MinObjAlignment);
650
651 guarantee(MaxTLABSizeBytes == 0, "we should only set it once");
652 MaxTLABSizeBytes = MaxTLABSizeWords * HeapWordSize;
653 assert (MaxTLABSizeBytes > MinTLABSize, "should be larger");
654
655 log_info(gc, init)("Regions: " SIZE_FORMAT " x " SIZE_FORMAT "%s",
656 RegionCount, byte_size_in_proper_unit(RegionSizeBytes), proper_unit_for_byte_size(RegionSizeBytes));
657 log_info(gc, init)("Humongous object threshold: " SIZE_FORMAT "%s",
658 byte_size_in_proper_unit(HumongousThresholdBytes), proper_unit_for_byte_size(HumongousThresholdBytes));
659 log_info(gc, init)("Max TLAB size: " SIZE_FORMAT "%s",
660 byte_size_in_proper_unit(MaxTLABSizeBytes), proper_unit_for_byte_size(MaxTLABSizeBytes));
661 }
662
663 void ShenandoahHeapRegion::do_commit() {
664 if (!_heap->is_heap_region_special() && !os::commit_memory((char *) _reserved.start(), _reserved.byte_size(), false)) {
665 report_java_out_of_memory("Unable to commit region");
666 }
667 if (!_heap->commit_bitmap_slice(this)) {
668 report_java_out_of_memory("Unable to commit bitmaps for region");
669 }
670 _heap->increase_committed(ShenandoahHeapRegion::region_size_bytes());
671 }
672
673 void ShenandoahHeapRegion::do_uncommit() {
674 if (!_heap->is_heap_region_special() && !os::uncommit_memory((char *) _reserved.start(), _reserved.byte_size())) {
675 report_java_out_of_memory("Unable to uncommit region");
676 }
677 if (!_heap->uncommit_bitmap_slice(this)) {
678 report_java_out_of_memory("Unable to uncommit bitmaps for region");
679 }
680 _heap->decrease_committed(ShenandoahHeapRegion::region_size_bytes());
681 }
682
683 void ShenandoahHeapRegion::set_state(RegionState to) {
684 EventShenandoahHeapRegionStateChange evt;
685 if (evt.should_commit()){
686 evt.set_index((unsigned)region_number());
687 evt.set_start((uintptr_t)bottom());
688 evt.set_used(used());
689 evt.set_from(_state);
690 evt.set_to(to);
691 evt.commit();
692 }
693 _state = to;
694 }