1 /*
2 * Copyright (c) 2013, 2019, Red Hat, Inc. All rights reserved.
3 *
4 * This code is free software; you can redistribute it and/or modify it
5 * under the terms of the GNU General Public License version 2 only, as
6 * published by the Free Software Foundation.
7 *
8 * This code is distributed in the hope that it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
11 * version 2 for more details (a copy is included in the LICENSE file that
12 * accompanied this code).
13 *
14 * You should have received a copy of the GNU General Public License version
15 * 2 along with this work; if not, write to the Free Software Foundation,
16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
17 *
18 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
19 * or visit www.oracle.com if you need additional information or have any
20 * questions.
21 *
22 */
23
24 #include "precompiled.hpp"
25 #include "memory/allocation.hpp"
26 #include "gc/shenandoah/shenandoahHeapRegionSet.inline.hpp"
27 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
28 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
29 #include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"
30 #include "gc/shared/space.inline.hpp"
31 #include "jfr/jfrEvents.hpp"
32 #include "memory/iterator.inline.hpp"
33 #include "memory/resourceArea.hpp"
34 #include "memory/universe.hpp"
35 #include "oops/oop.inline.hpp"
36 #include "runtime/java.hpp"
37 #include "runtime/mutexLocker.hpp"
38 #include "runtime/os.hpp"
39 #include "runtime/safepoint.hpp"
40
41 size_t ShenandoahHeapRegion::RegionCount = 0;
42 size_t ShenandoahHeapRegion::RegionSizeBytes = 0;
43 size_t ShenandoahHeapRegion::RegionSizeWords = 0;
44 size_t ShenandoahHeapRegion::RegionSizeBytesShift = 0;
45 size_t ShenandoahHeapRegion::RegionSizeWordsShift = 0;
46 size_t ShenandoahHeapRegion::RegionSizeBytesMask = 0;
47 size_t ShenandoahHeapRegion::RegionSizeWordsMask = 0;
48 size_t ShenandoahHeapRegion::HumongousThresholdBytes = 0;
49 size_t ShenandoahHeapRegion::HumongousThresholdWords = 0;
50 size_t ShenandoahHeapRegion::MaxTLABSizeBytes = 0;
51 size_t ShenandoahHeapRegion::MaxTLABSizeWords = 0;
52
53 ShenandoahHeapRegion::ShenandoahHeapRegion(HeapWord* start, size_t index, bool committed) :
54 _index(index),
55 _bottom(start),
56 _end(start + RegionSizeWords),
57 _new_top(NULL),
58 _empty_time(os::elapsedTime()),
59 _state(committed ? _empty_committed : _empty_uncommitted),
60 _top(start),
61 _tlab_allocs(0),
62 _gclab_allocs(0),
63 _live_data(0),
64 _critical_pins(0),
65 _update_watermark(start) {
66
67 assert(Universe::on_page_boundary(_bottom) && Universe::on_page_boundary(_end),
68 "invalid space boundaries");
69 if (ZapUnusedHeapArea && committed) {
70 SpaceMangler::mangle_region(MemRegion(_bottom, _end));
71 }
72 }
73
74 void ShenandoahHeapRegion::report_illegal_transition(const char *method) {
75 ResourceMark rm;
76 stringStream ss;
77 ss.print("Illegal region state transition from \"%s\", at %s\n ", region_state_to_string(_state), method);
78 print_on(&ss);
79 fatal("%s", ss.as_string());
80 }
81
82 void ShenandoahHeapRegion::make_regular_allocation() {
83 shenandoah_assert_heaplocked();
84
85 switch (_state) {
86 case _empty_uncommitted:
87 do_commit();
88 case _empty_committed:
89 set_state(_regular);
90 case _regular:
91 case _pinned:
92 return;
93 default:
94 report_illegal_transition("regular allocation");
95 }
96 }
97
98 void ShenandoahHeapRegion::make_regular_bypass() {
99 shenandoah_assert_heaplocked();
100 assert (ShenandoahHeap::heap()->is_full_gc_in_progress() || ShenandoahHeap::heap()->is_degenerated_gc_in_progress(),
101 "only for full or degen GC");
102
103 switch (_state) {
104 case _empty_uncommitted:
105 do_commit();
106 case _empty_committed:
107 case _cset:
108 case _humongous_start:
109 case _humongous_cont:
110 set_state(_regular);
111 return;
112 case _pinned_cset:
113 set_state(_pinned);
114 return;
115 case _regular:
116 case _pinned:
117 return;
118 default:
119 report_illegal_transition("regular bypass");
120 }
121 }
122
123 void ShenandoahHeapRegion::make_humongous_start() {
124 shenandoah_assert_heaplocked();
125 switch (_state) {
126 case _empty_uncommitted:
127 do_commit();
128 case _empty_committed:
129 set_state(_humongous_start);
130 return;
131 default:
132 report_illegal_transition("humongous start allocation");
133 }
134 }
135
136 void ShenandoahHeapRegion::make_humongous_start_bypass() {
137 shenandoah_assert_heaplocked();
138 assert (ShenandoahHeap::heap()->is_full_gc_in_progress(), "only for full GC");
139
140 switch (_state) {
141 case _empty_committed:
142 case _regular:
143 case _humongous_start:
144 case _humongous_cont:
145 set_state(_humongous_start);
146 return;
147 default:
148 report_illegal_transition("humongous start bypass");
149 }
150 }
151
152 void ShenandoahHeapRegion::make_humongous_cont() {
153 shenandoah_assert_heaplocked();
154 switch (_state) {
155 case _empty_uncommitted:
156 do_commit();
157 case _empty_committed:
158 set_state(_humongous_cont);
159 return;
160 default:
161 report_illegal_transition("humongous continuation allocation");
162 }
163 }
164
165 void ShenandoahHeapRegion::make_humongous_cont_bypass() {
166 shenandoah_assert_heaplocked();
167 assert (ShenandoahHeap::heap()->is_full_gc_in_progress(), "only for full GC");
168
169 switch (_state) {
170 case _empty_committed:
171 case _regular:
172 case _humongous_start:
173 case _humongous_cont:
174 set_state(_humongous_cont);
175 return;
176 default:
177 report_illegal_transition("humongous continuation bypass");
178 }
179 }
180
181 void ShenandoahHeapRegion::make_pinned() {
182 shenandoah_assert_heaplocked();
183 assert(pin_count() > 0, "Should have pins: " SIZE_FORMAT, pin_count());
184
185 switch (_state) {
186 case _regular:
187 set_state(_pinned);
188 case _pinned_cset:
189 case _pinned:
190 return;
191 case _humongous_start:
192 set_state(_pinned_humongous_start);
193 case _pinned_humongous_start:
194 return;
195 case _cset:
196 _state = _pinned_cset;
197 return;
198 default:
199 report_illegal_transition("pinning");
200 }
201 }
202
203 void ShenandoahHeapRegion::make_unpinned() {
204 shenandoah_assert_heaplocked();
205 assert(pin_count() == 0, "Should not have pins: " SIZE_FORMAT, pin_count());
206
207 switch (_state) {
208 case _pinned:
209 set_state(_regular);
210 return;
211 case _regular:
212 case _humongous_start:
213 return;
214 case _pinned_cset:
215 set_state(_cset);
216 return;
217 case _pinned_humongous_start:
218 set_state(_humongous_start);
219 return;
220 default:
221 report_illegal_transition("unpinning");
222 }
223 }
224
225 void ShenandoahHeapRegion::make_cset() {
226 shenandoah_assert_heaplocked();
227 switch (_state) {
228 case _regular:
229 set_state(_cset);
230 case _cset:
231 return;
232 default:
233 report_illegal_transition("cset");
234 }
235 }
236
237 void ShenandoahHeapRegion::make_trash() {
238 shenandoah_assert_heaplocked();
239 switch (_state) {
240 case _cset:
241 // Reclaiming cset regions
242 case _humongous_start:
243 case _humongous_cont:
244 // Reclaiming humongous regions
245 case _regular:
246 // Immediate region reclaim
247 set_state(_trash);
248 return;
249 default:
250 report_illegal_transition("trashing");
251 }
252 }
253
254 void ShenandoahHeapRegion::make_trash_immediate() {
255 make_trash();
256
257 // On this path, we know there are no marked objects in the region,
258 // tell marking context about it to bypass bitmap resets.
259 ShenandoahHeap::heap()->complete_marking_context()->reset_top_bitmap(this);
260 }
261
262 void ShenandoahHeapRegion::make_empty() {
263 shenandoah_assert_heaplocked();
264 switch (_state) {
265 case _trash:
266 set_state(_empty_committed);
267 _empty_time = os::elapsedTime();
268 return;
269 default:
270 report_illegal_transition("emptying");
271 }
272 }
273
274 void ShenandoahHeapRegion::make_uncommitted() {
275 shenandoah_assert_heaplocked();
276 switch (_state) {
277 case _empty_committed:
278 do_uncommit();
279 set_state(_empty_uncommitted);
280 return;
281 default:
282 report_illegal_transition("uncommiting");
283 }
284 }
285
286 void ShenandoahHeapRegion::make_committed_bypass() {
287 shenandoah_assert_heaplocked();
288 assert (ShenandoahHeap::heap()->is_full_gc_in_progress(), "only for full GC");
289
290 switch (_state) {
291 case _empty_uncommitted:
292 do_commit();
293 set_state(_empty_committed);
294 return;
295 default:
296 report_illegal_transition("commit bypass");
297 }
298 }
299
300 void ShenandoahHeapRegion::reset_alloc_metadata() {
301 _tlab_allocs = 0;
302 _gclab_allocs = 0;
303 }
304
305 size_t ShenandoahHeapRegion::get_shared_allocs() const {
306 return used() - (_tlab_allocs + _gclab_allocs) * HeapWordSize;
307 }
308
309 size_t ShenandoahHeapRegion::get_tlab_allocs() const {
310 return _tlab_allocs * HeapWordSize;
311 }
312
313 size_t ShenandoahHeapRegion::get_gclab_allocs() const {
314 return _gclab_allocs * HeapWordSize;
315 }
316
317 void ShenandoahHeapRegion::set_live_data(size_t s) {
318 assert(Thread::current()->is_VM_thread(), "by VM thread");
319 _live_data = (s >> LogHeapWordSize);
320 }
321
322 void ShenandoahHeapRegion::print_on(outputStream* st) const {
323 st->print("|");
324 st->print(SIZE_FORMAT_W(5), this->_index);
325
326 switch (_state) {
327 case _empty_uncommitted:
328 st->print("|EU ");
329 break;
330 case _empty_committed:
331 st->print("|EC ");
332 break;
333 case _regular:
334 st->print("|R ");
335 break;
336 case _humongous_start:
337 st->print("|H ");
338 break;
339 case _pinned_humongous_start:
340 st->print("|HP ");
341 break;
342 case _humongous_cont:
343 st->print("|HC ");
344 break;
345 case _cset:
346 st->print("|CS ");
347 break;
348 case _trash:
349 st->print("|T ");
350 break;
351 case _pinned:
352 st->print("|P ");
353 break;
354 case _pinned_cset:
355 st->print("|CSP");
356 break;
357 default:
358 ShouldNotReachHere();
359 }
360 st->print("|BTE " INTPTR_FORMAT_W(12) ", " INTPTR_FORMAT_W(12) ", " INTPTR_FORMAT_W(12),
361 p2i(bottom()), p2i(top()), p2i(end()));
362 st->print("|TAMS " INTPTR_FORMAT_W(12),
363 p2i(ShenandoahHeap::heap()->marking_context()->top_at_mark_start(const_cast<ShenandoahHeapRegion*>(this))));
364 st->print("|UWM " INTPTR_FORMAT_W(12),
365 p2i(_update_watermark));
366 st->print("|U " SIZE_FORMAT_W(5) "%1s", byte_size_in_proper_unit(used()), proper_unit_for_byte_size(used()));
367 st->print("|T " SIZE_FORMAT_W(5) "%1s", byte_size_in_proper_unit(get_tlab_allocs()), proper_unit_for_byte_size(get_tlab_allocs()));
368 st->print("|G " SIZE_FORMAT_W(5) "%1s", byte_size_in_proper_unit(get_gclab_allocs()), proper_unit_for_byte_size(get_gclab_allocs()));
369 st->print("|S " SIZE_FORMAT_W(5) "%1s", byte_size_in_proper_unit(get_shared_allocs()), proper_unit_for_byte_size(get_shared_allocs()));
370 st->print("|L " SIZE_FORMAT_W(5) "%1s", byte_size_in_proper_unit(get_live_data_bytes()), proper_unit_for_byte_size(get_live_data_bytes()));
371 st->print("|CP " SIZE_FORMAT_W(3), pin_count());
372 st->cr();
373 }
374
375 void ShenandoahHeapRegion::oop_iterate(OopIterateClosure* blk) {
376 if (!is_active()) return;
377 if (is_humongous()) {
378 oop_iterate_humongous(blk);
379 } else {
380 oop_iterate_objects(blk);
381 }
382 }
383
384 void ShenandoahHeapRegion::oop_iterate_objects(OopIterateClosure* blk) {
385 assert(! is_humongous(), "no humongous region here");
386 HeapWord* obj_addr = bottom();
387 HeapWord* t = top();
388 // Could call objects iterate, but this is easier.
389 while (obj_addr < t) {
390 oop obj = oop(obj_addr);
391 obj_addr += obj->oop_iterate_size(blk);
392 }
393 }
394
395 void ShenandoahHeapRegion::oop_iterate_humongous(OopIterateClosure* blk) {
396 assert(is_humongous(), "only humongous region here");
397 // Find head.
398 ShenandoahHeapRegion* r = humongous_start_region();
399 assert(r->is_humongous_start(), "need humongous head here");
400 oop obj = oop(r->bottom());
401 obj->oop_iterate(blk, MemRegion(bottom(), top()));
402 }
403
404 ShenandoahHeapRegion* ShenandoahHeapRegion::humongous_start_region() const {
405 ShenandoahHeap* heap = ShenandoahHeap::heap();
406 assert(is_humongous(), "Must be a part of the humongous region");
407 size_t i = index();
408 ShenandoahHeapRegion* r = const_cast<ShenandoahHeapRegion*>(this);
409 while (!r->is_humongous_start()) {
410 assert(i > 0, "Sanity");
411 i--;
412 r = heap->get_region(i);
413 assert(r->is_humongous(), "Must be a part of the humongous region");
414 }
415 assert(r->is_humongous_start(), "Must be");
416 return r;
417 }
418
419 void ShenandoahHeapRegion::recycle() {
420 set_top(bottom());
421 clear_live_data();
422
423 reset_alloc_metadata();
424
425 ShenandoahHeap::heap()->marking_context()->reset_top_at_mark_start(this);
426 set_update_watermark(bottom());
427
428 make_empty();
429
430 if (ZapUnusedHeapArea) {
431 SpaceMangler::mangle_region(MemRegion(bottom(), end()));
432 }
433 }
434
435 HeapWord* ShenandoahHeapRegion::block_start(const void* p) const {
436 assert(MemRegion(bottom(), end()).contains(p),
437 "p (" PTR_FORMAT ") not in space [" PTR_FORMAT ", " PTR_FORMAT ")",
438 p2i(p), p2i(bottom()), p2i(end()));
439 if (p >= top()) {
440 return top();
441 } else {
442 HeapWord* last = bottom();
443 HeapWord* cur = last;
444 while (cur <= p) {
445 last = cur;
446 cur += oop(cur)->size();
447 }
448 shenandoah_assert_correct(NULL, oop(last));
449 return last;
450 }
451 }
452
453 size_t ShenandoahHeapRegion::block_size(const HeapWord* p) const {
454 assert(MemRegion(bottom(), end()).contains(p),
455 "p (" PTR_FORMAT ") not in space [" PTR_FORMAT ", " PTR_FORMAT ")",
456 p2i(p), p2i(bottom()), p2i(end()));
457 if (p < top()) {
458 return oop(p)->size();
459 } else {
460 assert(p == top(), "just checking");
461 return pointer_delta(end(), (HeapWord*) p);
462 }
463 }
464
465 void ShenandoahHeapRegion::setup_sizes(size_t max_heap_size) {
466 // Absolute minimums we should not ever break.
467 static const size_t MIN_REGION_SIZE = 256*K;
468
469 if (FLAG_IS_DEFAULT(ShenandoahMinRegionSize)) {
470 FLAG_SET_DEFAULT(ShenandoahMinRegionSize, MIN_REGION_SIZE);
471 }
472
473 size_t region_size;
474 if (FLAG_IS_DEFAULT(ShenandoahRegionSize)) {
475 if (ShenandoahMinRegionSize > max_heap_size / MIN_NUM_REGIONS) {
476 err_msg message("Max heap size (" SIZE_FORMAT "%s) is too low to afford the minimum number "
477 "of regions (" SIZE_FORMAT ") of minimum region size (" SIZE_FORMAT "%s).",
478 byte_size_in_proper_unit(max_heap_size), proper_unit_for_byte_size(max_heap_size),
479 MIN_NUM_REGIONS,
480 byte_size_in_proper_unit(ShenandoahMinRegionSize), proper_unit_for_byte_size(ShenandoahMinRegionSize));
481 vm_exit_during_initialization("Invalid -XX:ShenandoahMinRegionSize option", message);
482 }
483 if (ShenandoahMinRegionSize < MIN_REGION_SIZE) {
484 err_msg message("" SIZE_FORMAT "%s should not be lower than minimum region size (" SIZE_FORMAT "%s).",
485 byte_size_in_proper_unit(ShenandoahMinRegionSize), proper_unit_for_byte_size(ShenandoahMinRegionSize),
486 byte_size_in_proper_unit(MIN_REGION_SIZE), proper_unit_for_byte_size(MIN_REGION_SIZE));
487 vm_exit_during_initialization("Invalid -XX:ShenandoahMinRegionSize option", message);
488 }
489 if (ShenandoahMinRegionSize < MinTLABSize) {
490 err_msg message("" SIZE_FORMAT "%s should not be lower than TLAB size size (" SIZE_FORMAT "%s).",
491 byte_size_in_proper_unit(ShenandoahMinRegionSize), proper_unit_for_byte_size(ShenandoahMinRegionSize),
492 byte_size_in_proper_unit(MinTLABSize), proper_unit_for_byte_size(MinTLABSize));
493 vm_exit_during_initialization("Invalid -XX:ShenandoahMinRegionSize option", message);
494 }
495 if (ShenandoahMaxRegionSize < MIN_REGION_SIZE) {
496 err_msg message("" SIZE_FORMAT "%s should not be lower than min region size (" SIZE_FORMAT "%s).",
497 byte_size_in_proper_unit(ShenandoahMaxRegionSize), proper_unit_for_byte_size(ShenandoahMaxRegionSize),
498 byte_size_in_proper_unit(MIN_REGION_SIZE), proper_unit_for_byte_size(MIN_REGION_SIZE));
499 vm_exit_during_initialization("Invalid -XX:ShenandoahMaxRegionSize option", message);
500 }
501 if (ShenandoahMinRegionSize > ShenandoahMaxRegionSize) {
502 err_msg message("Minimum (" SIZE_FORMAT "%s) should be larger than maximum (" SIZE_FORMAT "%s).",
503 byte_size_in_proper_unit(ShenandoahMinRegionSize), proper_unit_for_byte_size(ShenandoahMinRegionSize),
504 byte_size_in_proper_unit(ShenandoahMaxRegionSize), proper_unit_for_byte_size(ShenandoahMaxRegionSize));
505 vm_exit_during_initialization("Invalid -XX:ShenandoahMinRegionSize or -XX:ShenandoahMaxRegionSize", message);
506 }
507
508 // We rapidly expand to max_heap_size in most scenarios, so that is the measure
509 // for usual heap sizes. Do not depend on initial_heap_size here.
510 region_size = max_heap_size / ShenandoahTargetNumRegions;
511
512 // Now make sure that we don't go over or under our limits.
513 region_size = MAX2(ShenandoahMinRegionSize, region_size);
514 region_size = MIN2(ShenandoahMaxRegionSize, region_size);
515
516 } else {
517 if (ShenandoahRegionSize > max_heap_size / MIN_NUM_REGIONS) {
518 err_msg message("Max heap size (" SIZE_FORMAT "%s) is too low to afford the minimum number "
519 "of regions (" SIZE_FORMAT ") of requested size (" SIZE_FORMAT "%s).",
520 byte_size_in_proper_unit(max_heap_size), proper_unit_for_byte_size(max_heap_size),
521 MIN_NUM_REGIONS,
522 byte_size_in_proper_unit(ShenandoahRegionSize), proper_unit_for_byte_size(ShenandoahRegionSize));
523 vm_exit_during_initialization("Invalid -XX:ShenandoahRegionSize option", message);
524 }
525 if (ShenandoahRegionSize < ShenandoahMinRegionSize) {
526 err_msg message("Heap region size (" SIZE_FORMAT "%s) should be larger than min region size (" SIZE_FORMAT "%s).",
527 byte_size_in_proper_unit(ShenandoahRegionSize), proper_unit_for_byte_size(ShenandoahRegionSize),
528 byte_size_in_proper_unit(ShenandoahMinRegionSize), proper_unit_for_byte_size(ShenandoahMinRegionSize));
529 vm_exit_during_initialization("Invalid -XX:ShenandoahRegionSize option", message);
530 }
531 if (ShenandoahRegionSize > ShenandoahMaxRegionSize) {
532 err_msg message("Heap region size (" SIZE_FORMAT "%s) should be lower than max region size (" SIZE_FORMAT "%s).",
533 byte_size_in_proper_unit(ShenandoahRegionSize), proper_unit_for_byte_size(ShenandoahRegionSize),
534 byte_size_in_proper_unit(ShenandoahMaxRegionSize), proper_unit_for_byte_size(ShenandoahMaxRegionSize));
535 vm_exit_during_initialization("Invalid -XX:ShenandoahRegionSize option", message);
536 }
537 region_size = ShenandoahRegionSize;
538 }
539
540 // Make sure region size is at least one large page, if enabled.
541 // Otherwise, uncommitting one region may falsely uncommit the adjacent
542 // regions too.
543 // Also see shenandoahArguments.cpp, where it handles UseLargePages.
544 if (UseLargePages && ShenandoahUncommit) {
545 region_size = MAX2(region_size, os::large_page_size());
546 }
547
548 int region_size_log = log2_long((jlong) region_size);
549 // Recalculate the region size to make sure it's a power of
550 // 2. This means that region_size is the largest power of 2 that's
551 // <= what we've calculated so far.
552 region_size = size_t(1) << region_size_log;
553
554 // Now, set up the globals.
555 guarantee(RegionSizeBytesShift == 0, "we should only set it once");
556 RegionSizeBytesShift = (size_t)region_size_log;
557
558 guarantee(RegionSizeWordsShift == 0, "we should only set it once");
559 RegionSizeWordsShift = RegionSizeBytesShift - LogHeapWordSize;
560
561 guarantee(RegionSizeBytes == 0, "we should only set it once");
562 RegionSizeBytes = region_size;
563 RegionSizeWords = RegionSizeBytes >> LogHeapWordSize;
564 assert (RegionSizeWords*HeapWordSize == RegionSizeBytes, "sanity");
565
566 guarantee(RegionSizeWordsMask == 0, "we should only set it once");
567 RegionSizeWordsMask = RegionSizeWords - 1;
568
569 guarantee(RegionSizeBytesMask == 0, "we should only set it once");
570 RegionSizeBytesMask = RegionSizeBytes - 1;
571
572 guarantee(RegionCount == 0, "we should only set it once");
573 RegionCount = max_heap_size / RegionSizeBytes;
574 guarantee(RegionCount >= MIN_NUM_REGIONS, "Should have at least minimum regions");
575
576 guarantee(HumongousThresholdWords == 0, "we should only set it once");
577 HumongousThresholdWords = RegionSizeWords * ShenandoahHumongousThreshold / 100;
578 HumongousThresholdWords = align_down(HumongousThresholdWords, MinObjAlignment);
579 assert (HumongousThresholdWords <= RegionSizeWords, "sanity");
580
581 guarantee(HumongousThresholdBytes == 0, "we should only set it once");
582 HumongousThresholdBytes = HumongousThresholdWords * HeapWordSize;
583 assert (HumongousThresholdBytes <= RegionSizeBytes, "sanity");
584
585 // The rationale for trimming the TLAB sizes has to do with the raciness in
586 // TLAB allocation machinery. It may happen that TLAB sizing policy polls Shenandoah
587 // about next free size, gets the answer for region #N, goes away for a while, then
588 // tries to allocate in region #N, and fail because some other thread have claimed part
589 // of the region #N, and then the freeset allocation code has to retire the region #N,
590 // before moving the allocation to region #N+1.
591 //
592 // The worst case realizes when "answer" is "region size", which means it could
593 // prematurely retire an entire region. Having smaller TLABs does not fix that
594 // completely, but reduces the probability of too wasteful region retirement.
595 // With current divisor, we will waste no more than 1/8 of region size in the worst
596 // case. This also has a secondary effect on collection set selection: even under
597 // the race, the regions would be at least 7/8 used, which allows relying on
598 // "used" - "live" for cset selection. Otherwise, we can get the fragmented region
599 // below the garbage threshold that would never be considered for collection.
600 //
601 // The whole thing is mitigated if Elastic TLABs are enabled.
602 //
603 guarantee(MaxTLABSizeWords == 0, "we should only set it once");
604 MaxTLABSizeWords = MIN2(ShenandoahElasticTLAB ? RegionSizeWords : (RegionSizeWords / 8), HumongousThresholdWords);
605 MaxTLABSizeWords = align_down(MaxTLABSizeWords, MinObjAlignment);
606
607 guarantee(MaxTLABSizeBytes == 0, "we should only set it once");
608 MaxTLABSizeBytes = MaxTLABSizeWords * HeapWordSize;
609 assert (MaxTLABSizeBytes > MinTLABSize, "should be larger");
610
611 log_info(gc, init)("Regions: " SIZE_FORMAT " x " SIZE_FORMAT "%s",
612 RegionCount, byte_size_in_proper_unit(RegionSizeBytes), proper_unit_for_byte_size(RegionSizeBytes));
613 log_info(gc, init)("Humongous object threshold: " SIZE_FORMAT "%s",
614 byte_size_in_proper_unit(HumongousThresholdBytes), proper_unit_for_byte_size(HumongousThresholdBytes));
615 log_info(gc, init)("Max TLAB size: " SIZE_FORMAT "%s",
616 byte_size_in_proper_unit(MaxTLABSizeBytes), proper_unit_for_byte_size(MaxTLABSizeBytes));
617 }
618
619 void ShenandoahHeapRegion::do_commit() {
620 ShenandoahHeap* heap = ShenandoahHeap::heap();
621 if (!heap->is_heap_region_special() && !os::commit_memory((char *) bottom(), RegionSizeBytes, false)) {
622 report_java_out_of_memory("Unable to commit region");
623 }
624 if (!heap->commit_bitmap_slice(this)) {
625 report_java_out_of_memory("Unable to commit bitmaps for region");
626 }
627 if (AlwaysPreTouch) {
628 os::pretouch_memory(bottom(), end(), heap->pretouch_heap_page_size());
629 }
630 heap->increase_committed(ShenandoahHeapRegion::region_size_bytes());
631 }
632
633 void ShenandoahHeapRegion::do_uncommit() {
634 ShenandoahHeap* heap = ShenandoahHeap::heap();
635 if (!heap->is_heap_region_special() && !os::uncommit_memory((char *) bottom(), RegionSizeBytes)) {
636 report_java_out_of_memory("Unable to uncommit region");
637 }
638 if (!heap->uncommit_bitmap_slice(this)) {
639 report_java_out_of_memory("Unable to uncommit bitmaps for region");
640 }
641 heap->decrease_committed(ShenandoahHeapRegion::region_size_bytes());
642 }
643
644 void ShenandoahHeapRegion::set_state(RegionState to) {
645 EventShenandoahHeapRegionStateChange evt;
646 if (evt.should_commit()){
647 evt.set_index((unsigned)index());
648 evt.set_start((uintptr_t)bottom());
649 evt.set_used(used());
650 evt.set_from(_state);
651 evt.set_to(to);
652 evt.commit();
653 }
654 _state = to;
655 }
656
657 void ShenandoahHeapRegion::record_pin() {
658 Atomic::add((size_t)1, &_critical_pins);
659 }
660
661 void ShenandoahHeapRegion::record_unpin() {
662 assert(pin_count() > 0, "Region " SIZE_FORMAT " should have non-zero pins", index());
663 Atomic::sub((size_t)1, &_critical_pins);
664 }
665
666 size_t ShenandoahHeapRegion::pin_count() const {
667 return Atomic::load(&_critical_pins);
668 }