1 /*
2 * Copyright (c) 2013, 2019, Red Hat, Inc. All rights reserved.
3 *
4 * This code is free software; you can redistribute it and/or modify it
5 * under the terms of the GNU General Public License version 2 only, as
6 * published by the Free Software Foundation.
7 *
8 * This code is distributed in the hope that it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
11 * version 2 for more details (a copy is included in the LICENSE file that
12 * accompanied this code).
13 *
14 * You should have received a copy of the GNU General Public License version
15 * 2 along with this work; if not, write to the Free Software Foundation,
16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
17 *
18 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
19 * or visit www.oracle.com if you need additional information or have any
20 * questions.
21 *
22 */
23
24 #include "precompiled.hpp"
25 #include "memory/allocation.hpp"
26 #include "gc/shenandoah/shenandoahHeapRegionSet.inline.hpp"
27 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
28 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
29 #include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"
30 #include "gc/shenandoah/shenandoahTraversalGC.hpp"
31 #include "gc/shared/space.inline.hpp"
32 #include "jfr/jfrEvents.hpp"
33 #include "memory/iterator.inline.hpp"
34 #include "memory/resourceArea.hpp"
35 #include "memory/universe.hpp"
36 #include "oops/oop.inline.hpp"
37 #include "runtime/java.hpp"
38 #include "runtime/mutexLocker.hpp"
39 #include "runtime/os.hpp"
40 #include "runtime/safepoint.hpp"
41
42 size_t ShenandoahHeapRegion::RegionCount = 0;
43 size_t ShenandoahHeapRegion::RegionSizeBytes = 0;
44 size_t ShenandoahHeapRegion::RegionSizeWords = 0;
45 size_t ShenandoahHeapRegion::RegionSizeBytesShift = 0;
46 size_t ShenandoahHeapRegion::RegionSizeWordsShift = 0;
47 size_t ShenandoahHeapRegion::RegionSizeBytesMask = 0;
48 size_t ShenandoahHeapRegion::RegionSizeWordsMask = 0;
49 size_t ShenandoahHeapRegion::HumongousThresholdBytes = 0;
50 size_t ShenandoahHeapRegion::HumongousThresholdWords = 0;
51 size_t ShenandoahHeapRegion::MaxTLABSizeBytes = 0;
52 size_t ShenandoahHeapRegion::MaxTLABSizeWords = 0;
53
54 ShenandoahHeapRegion::PaddedAllocSeqNum ShenandoahHeapRegion::_alloc_seq_num;
55
56 ShenandoahHeapRegion::ShenandoahHeapRegion(ShenandoahHeap* heap, HeapWord* start,
57 size_t size_words, size_t index, bool committed) :
58 _heap(heap),
59 _reserved(MemRegion(start, size_words)),
60 _region_number(index),
61 _new_top(NULL),
62 _empty_time(os::elapsedTime()),
63 _state(committed ? _empty_committed : _empty_uncommitted),
64 _tlab_allocs(0),
65 _gclab_allocs(0),
66 _shared_allocs(0),
67 _seqnum_first_alloc_mutator(0),
68 _seqnum_first_alloc_gc(0),
69 _seqnum_last_alloc_mutator(0),
70 _seqnum_last_alloc_gc(0),
71 _live_data(0),
72 _critical_pins(0) {
73
74 ContiguousSpace::initialize(_reserved, true, committed);
75 }
76
77 size_t ShenandoahHeapRegion::region_number() const {
78 return _region_number;
79 }
80
81 void ShenandoahHeapRegion::report_illegal_transition(const char *method) {
82 ResourceMark rm;
83 stringStream ss;
84 ss.print("Illegal region state transition from \"%s\", at %s\n ", region_state_to_string(_state), method);
85 print_on(&ss);
86 fatal("%s", ss.as_string());
87 }
88
89 void ShenandoahHeapRegion::make_regular_allocation() {
90 _heap->assert_heaplock_owned_by_current_thread();
91
92 switch (_state) {
93 case _empty_uncommitted:
94 do_commit();
95 case _empty_committed:
96 set_state(_regular);
97 case _regular:
98 case _pinned:
99 return;
100 default:
101 report_illegal_transition("regular allocation");
102 }
103 }
104
105 void ShenandoahHeapRegion::make_regular_bypass() {
106 _heap->assert_heaplock_owned_by_current_thread();
107 assert (_heap->is_full_gc_in_progress() || _heap->is_degenerated_gc_in_progress(),
108 "only for full or degen GC");
109
110 switch (_state) {
111 case _empty_uncommitted:
112 do_commit();
113 case _empty_committed:
114 case _cset:
115 case _humongous_start:
116 case _humongous_cont:
117 set_state(_regular);
118 return;
119 case _pinned_cset:
120 set_state(_pinned);
121 return;
122 case _regular:
123 case _pinned:
124 return;
125 default:
126 report_illegal_transition("regular bypass");
127 }
128 }
129
130 void ShenandoahHeapRegion::make_humongous_start() {
131 _heap->assert_heaplock_owned_by_current_thread();
132 switch (_state) {
133 case _empty_uncommitted:
134 do_commit();
135 case _empty_committed:
136 set_state(_humongous_start);
137 return;
138 default:
139 report_illegal_transition("humongous start allocation");
140 }
141 }
142
143 void ShenandoahHeapRegion::make_humongous_start_bypass() {
144 _heap->assert_heaplock_owned_by_current_thread();
145 assert (_heap->is_full_gc_in_progress(), "only for full GC");
146
147 switch (_state) {
148 case _empty_committed:
149 case _regular:
150 case _humongous_start:
151 case _humongous_cont:
152 set_state(_humongous_start);
153 return;
154 default:
155 report_illegal_transition("humongous start bypass");
156 }
157 }
158
159 void ShenandoahHeapRegion::make_humongous_cont() {
160 _heap->assert_heaplock_owned_by_current_thread();
161 switch (_state) {
162 case _empty_uncommitted:
163 do_commit();
164 case _empty_committed:
165 set_state(_humongous_cont);
166 return;
167 default:
168 report_illegal_transition("humongous continuation allocation");
169 }
170 }
171
172 void ShenandoahHeapRegion::make_humongous_cont_bypass() {
173 _heap->assert_heaplock_owned_by_current_thread();
174 assert (_heap->is_full_gc_in_progress(), "only for full GC");
175
176 switch (_state) {
177 case _empty_committed:
178 case _regular:
179 case _humongous_start:
180 case _humongous_cont:
181 set_state(_humongous_cont);
182 return;
183 default:
184 report_illegal_transition("humongous continuation bypass");
185 }
186 }
187
188 void ShenandoahHeapRegion::make_pinned() {
189 _heap->assert_heaplock_owned_by_current_thread();
190 assert(pin_count() > 0, "Should have pins: " SIZE_FORMAT, pin_count());
191
192 switch (_state) {
193 case _regular:
194 set_state(_pinned);
195 case _pinned_cset:
196 case _pinned:
197 return;
198 case _humongous_start:
199 set_state(_pinned_humongous_start);
200 case _pinned_humongous_start:
201 return;
202 case _cset:
203 _state = _pinned_cset;
204 return;
205 default:
206 report_illegal_transition("pinning");
207 }
208 }
209
210 void ShenandoahHeapRegion::make_unpinned() {
211 _heap->assert_heaplock_owned_by_current_thread();
212 assert(pin_count() == 0, "Should not have pins: " SIZE_FORMAT, pin_count());
213
214 switch (_state) {
215 case _pinned:
216 set_state(_regular);
217 return;
218 case _regular:
219 case _humongous_start:
220 return;
221 case _pinned_cset:
222 set_state(_cset);
223 return;
224 case _pinned_humongous_start:
225 set_state(_humongous_start);
226 return;
227 default:
228 report_illegal_transition("unpinning");
229 }
230 }
231
232 void ShenandoahHeapRegion::make_cset() {
233 _heap->assert_heaplock_owned_by_current_thread();
234 switch (_state) {
235 case _regular:
236 set_state(_cset);
237 case _cset:
238 return;
239 default:
240 report_illegal_transition("cset");
241 }
242 }
243
244 void ShenandoahHeapRegion::make_trash() {
245 _heap->assert_heaplock_owned_by_current_thread();
246 switch (_state) {
247 case _cset:
248 // Reclaiming cset regions
249 case _humongous_start:
250 case _humongous_cont:
251 // Reclaiming humongous regions
252 case _regular:
253 // Immediate region reclaim
254 set_state(_trash);
255 return;
256 default:
257 report_illegal_transition("trashing");
258 }
259 }
260
261 void ShenandoahHeapRegion::make_trash_immediate() {
262 make_trash();
263
264 // On this path, we know there are no marked objects in the region,
265 // tell marking context about it to bypass bitmap resets.
266 _heap->complete_marking_context()->reset_top_bitmap(this);
267 }
268
269 void ShenandoahHeapRegion::make_empty() {
270 _heap->assert_heaplock_owned_by_current_thread();
271 switch (_state) {
272 case _trash:
273 set_state(_empty_committed);
274 _empty_time = os::elapsedTime();
275 return;
276 default:
277 report_illegal_transition("emptying");
278 }
279 }
280
281 void ShenandoahHeapRegion::make_uncommitted() {
282 _heap->assert_heaplock_owned_by_current_thread();
283 switch (_state) {
284 case _empty_committed:
285 do_uncommit();
286 set_state(_empty_uncommitted);
287 return;
288 default:
289 report_illegal_transition("uncommiting");
290 }
291 }
292
293 void ShenandoahHeapRegion::make_committed_bypass() {
294 _heap->assert_heaplock_owned_by_current_thread();
295 assert (_heap->is_full_gc_in_progress(), "only for full GC");
296
297 switch (_state) {
298 case _empty_uncommitted:
299 do_commit();
300 set_state(_empty_committed);
301 return;
302 default:
303 report_illegal_transition("commit bypass");
304 }
305 }
306
307 void ShenandoahHeapRegion::clear_live_data() {
308 Atomic::release_store_fence<size_t>(&_live_data, 0);
309 }
310
311 void ShenandoahHeapRegion::reset_alloc_metadata() {
312 _tlab_allocs = 0;
313 _gclab_allocs = 0;
314 _shared_allocs = 0;
315 _seqnum_first_alloc_mutator = 0;
316 _seqnum_last_alloc_mutator = 0;
317 _seqnum_first_alloc_gc = 0;
318 _seqnum_last_alloc_gc = 0;
319 }
320
321 void ShenandoahHeapRegion::reset_alloc_metadata_to_shared() {
322 if (used() > 0) {
323 _tlab_allocs = 0;
324 _gclab_allocs = 0;
325 _shared_allocs = used() >> LogHeapWordSize;
326 uint64_t next = _alloc_seq_num.value++;
327 _seqnum_first_alloc_mutator = next;
328 _seqnum_last_alloc_mutator = next;
329 _seqnum_first_alloc_gc = 0;
330 _seqnum_last_alloc_gc = 0;
331 } else {
332 reset_alloc_metadata();
333 }
334 }
335
336 size_t ShenandoahHeapRegion::get_shared_allocs() const {
337 return _shared_allocs * HeapWordSize;
338 }
339
340 size_t ShenandoahHeapRegion::get_tlab_allocs() const {
341 return _tlab_allocs * HeapWordSize;
342 }
343
344 size_t ShenandoahHeapRegion::get_gclab_allocs() const {
345 return _gclab_allocs * HeapWordSize;
346 }
347
348 void ShenandoahHeapRegion::set_live_data(size_t s) {
349 assert(Thread::current()->is_VM_thread(), "by VM thread");
350 _live_data = (s >> LogHeapWordSize);
351 }
352
353 size_t ShenandoahHeapRegion::get_live_data_words() const {
354 return Atomic::load_acquire(&_live_data);
355 }
356
357 size_t ShenandoahHeapRegion::get_live_data_bytes() const {
358 return get_live_data_words() * HeapWordSize;
359 }
360
361 bool ShenandoahHeapRegion::has_live() const {
362 return get_live_data_words() != 0;
363 }
364
365 size_t ShenandoahHeapRegion::garbage() const {
366 assert(used() >= get_live_data_bytes(), "Live Data must be a subset of used() live: " SIZE_FORMAT " used: " SIZE_FORMAT,
367 get_live_data_bytes(), used());
368
369 size_t result = used() - get_live_data_bytes();
370 return result;
371 }
372
373 void ShenandoahHeapRegion::print_on(outputStream* st) const {
374 st->print("|");
375 st->print(SIZE_FORMAT_W(5), this->_region_number);
376
377 switch (_state) {
378 case _empty_uncommitted:
379 st->print("|EU ");
380 break;
381 case _empty_committed:
382 st->print("|EC ");
383 break;
384 case _regular:
385 st->print("|R ");
386 break;
387 case _humongous_start:
388 st->print("|H ");
389 break;
390 case _pinned_humongous_start:
391 st->print("|HP ");
392 break;
393 case _humongous_cont:
394 st->print("|HC ");
395 break;
396 case _cset:
397 st->print("|CS ");
398 break;
399 case _trash:
400 st->print("|T ");
401 break;
402 case _pinned:
403 st->print("|P ");
404 break;
405 case _pinned_cset:
406 st->print("|CSP");
407 break;
408 default:
409 ShouldNotReachHere();
410 }
411 st->print("|BTE " INTPTR_FORMAT_W(12) ", " INTPTR_FORMAT_W(12) ", " INTPTR_FORMAT_W(12),
412 p2i(bottom()), p2i(top()), p2i(end()));
413 st->print("|TAMS " INTPTR_FORMAT_W(12),
414 p2i(_heap->marking_context()->top_at_mark_start(const_cast<ShenandoahHeapRegion*>(this))));
415 st->print("|U " SIZE_FORMAT_W(5) "%1s", byte_size_in_proper_unit(used()), proper_unit_for_byte_size(used()));
416 st->print("|T " SIZE_FORMAT_W(5) "%1s", byte_size_in_proper_unit(get_tlab_allocs()), proper_unit_for_byte_size(get_tlab_allocs()));
417 st->print("|G " SIZE_FORMAT_W(5) "%1s", byte_size_in_proper_unit(get_gclab_allocs()), proper_unit_for_byte_size(get_gclab_allocs()));
418 st->print("|S " SIZE_FORMAT_W(5) "%1s", byte_size_in_proper_unit(get_shared_allocs()), proper_unit_for_byte_size(get_shared_allocs()));
419 st->print("|L " SIZE_FORMAT_W(5) "%1s", byte_size_in_proper_unit(get_live_data_bytes()), proper_unit_for_byte_size(get_live_data_bytes()));
420 st->print("|CP " SIZE_FORMAT_W(3), pin_count());
421 st->print("|SN " UINT64_FORMAT_X_W(12) ", " UINT64_FORMAT_X_W(8) ", " UINT64_FORMAT_X_W(8) ", " UINT64_FORMAT_X_W(8),
422 seqnum_first_alloc_mutator(), seqnum_last_alloc_mutator(),
423 seqnum_first_alloc_gc(), seqnum_last_alloc_gc());
424 st->cr();
425 }
426
427 void ShenandoahHeapRegion::oop_iterate(OopIterateClosure* blk) {
428 if (!is_active()) return;
429 if (is_humongous()) {
430 oop_iterate_humongous(blk);
431 } else {
432 oop_iterate_objects(blk);
433 }
434 }
435
436 void ShenandoahHeapRegion::oop_iterate_objects(OopIterateClosure* blk) {
437 assert(! is_humongous(), "no humongous region here");
438 HeapWord* obj_addr = bottom();
439 HeapWord* t = top();
440 // Could call objects iterate, but this is easier.
441 while (obj_addr < t) {
442 oop obj = oop(obj_addr);
443 obj_addr += obj->oop_iterate_size(blk);
444 }
445 }
446
447 void ShenandoahHeapRegion::oop_iterate_humongous(OopIterateClosure* blk) {
448 assert(is_humongous(), "only humongous region here");
449 // Find head.
450 ShenandoahHeapRegion* r = humongous_start_region();
451 assert(r->is_humongous_start(), "need humongous head here");
452 oop obj = oop(r->bottom());
453 obj->oop_iterate(blk, MemRegion(bottom(), top()));
454 }
455
456 ShenandoahHeapRegion* ShenandoahHeapRegion::humongous_start_region() const {
457 assert(is_humongous(), "Must be a part of the humongous region");
458 size_t reg_num = region_number();
459 ShenandoahHeapRegion* r = const_cast<ShenandoahHeapRegion*>(this);
460 while (!r->is_humongous_start()) {
461 assert(reg_num > 0, "Sanity");
462 reg_num --;
463 r = _heap->get_region(reg_num);
464 assert(r->is_humongous(), "Must be a part of the humongous region");
465 }
466 assert(r->is_humongous_start(), "Must be");
467 return r;
468 }
469
470 void ShenandoahHeapRegion::recycle() {
471 ContiguousSpace::clear(false);
472 if (ZapUnusedHeapArea) {
473 ContiguousSpace::mangle_unused_area_complete();
474 }
475 clear_live_data();
476
477 reset_alloc_metadata();
478
479 _heap->marking_context()->reset_top_at_mark_start(this);
480
481 make_empty();
482 }
483
484 HeapWord* ShenandoahHeapRegion::block_start_const(const void* p) const {
485 assert(MemRegion(bottom(), end()).contains(p),
486 "p (" PTR_FORMAT ") not in space [" PTR_FORMAT ", " PTR_FORMAT ")",
487 p2i(p), p2i(bottom()), p2i(end()));
488 if (p >= top()) {
489 return top();
490 } else {
491 HeapWord* last = bottom();
492 HeapWord* cur = last;
493 while (cur <= p) {
494 last = cur;
495 cur += oop(cur)->size();
496 }
497 shenandoah_assert_correct(NULL, oop(last));
498 return last;
499 }
500 }
501
502 void ShenandoahHeapRegion::setup_sizes(size_t max_heap_size) {
503 // Absolute minimums we should not ever break.
504 static const size_t MIN_REGION_SIZE = 256*K;
505
506 if (FLAG_IS_DEFAULT(ShenandoahMinRegionSize)) {
507 FLAG_SET_DEFAULT(ShenandoahMinRegionSize, MIN_REGION_SIZE);
508 }
509
510 size_t region_size;
511 if (FLAG_IS_DEFAULT(ShenandoahHeapRegionSize)) {
512 if (ShenandoahMinRegionSize > max_heap_size / MIN_NUM_REGIONS) {
513 err_msg message("Max heap size (" SIZE_FORMAT "%s) is too low to afford the minimum number "
514 "of regions (" SIZE_FORMAT ") of minimum region size (" SIZE_FORMAT "%s).",
515 byte_size_in_proper_unit(max_heap_size), proper_unit_for_byte_size(max_heap_size),
516 MIN_NUM_REGIONS,
517 byte_size_in_proper_unit(ShenandoahMinRegionSize), proper_unit_for_byte_size(ShenandoahMinRegionSize));
518 vm_exit_during_initialization("Invalid -XX:ShenandoahMinRegionSize option", message);
519 }
520 if (ShenandoahMinRegionSize < MIN_REGION_SIZE) {
521 err_msg message("" SIZE_FORMAT "%s should not be lower than minimum region size (" SIZE_FORMAT "%s).",
522 byte_size_in_proper_unit(ShenandoahMinRegionSize), proper_unit_for_byte_size(ShenandoahMinRegionSize),
523 byte_size_in_proper_unit(MIN_REGION_SIZE), proper_unit_for_byte_size(MIN_REGION_SIZE));
524 vm_exit_during_initialization("Invalid -XX:ShenandoahMinRegionSize option", message);
525 }
526 if (ShenandoahMinRegionSize < MinTLABSize) {
527 err_msg message("" SIZE_FORMAT "%s should not be lower than TLAB size size (" SIZE_FORMAT "%s).",
528 byte_size_in_proper_unit(ShenandoahMinRegionSize), proper_unit_for_byte_size(ShenandoahMinRegionSize),
529 byte_size_in_proper_unit(MinTLABSize), proper_unit_for_byte_size(MinTLABSize));
530 vm_exit_during_initialization("Invalid -XX:ShenandoahMinRegionSize option", message);
531 }
532 if (ShenandoahMaxRegionSize < MIN_REGION_SIZE) {
533 err_msg message("" SIZE_FORMAT "%s should not be lower than min region size (" SIZE_FORMAT "%s).",
534 byte_size_in_proper_unit(ShenandoahMaxRegionSize), proper_unit_for_byte_size(ShenandoahMaxRegionSize),
535 byte_size_in_proper_unit(MIN_REGION_SIZE), proper_unit_for_byte_size(MIN_REGION_SIZE));
536 vm_exit_during_initialization("Invalid -XX:ShenandoahMaxRegionSize option", message);
537 }
538 if (ShenandoahMinRegionSize > ShenandoahMaxRegionSize) {
539 err_msg message("Minimum (" SIZE_FORMAT "%s) should be larger than maximum (" SIZE_FORMAT "%s).",
540 byte_size_in_proper_unit(ShenandoahMinRegionSize), proper_unit_for_byte_size(ShenandoahMinRegionSize),
541 byte_size_in_proper_unit(ShenandoahMaxRegionSize), proper_unit_for_byte_size(ShenandoahMaxRegionSize));
542 vm_exit_during_initialization("Invalid -XX:ShenandoahMinRegionSize or -XX:ShenandoahMaxRegionSize", message);
543 }
544
545 // We rapidly expand to max_heap_size in most scenarios, so that is the measure
546 // for usual heap sizes. Do not depend on initial_heap_size here.
547 region_size = max_heap_size / ShenandoahTargetNumRegions;
548
549 // Now make sure that we don't go over or under our limits.
550 region_size = MAX2(ShenandoahMinRegionSize, region_size);
551 region_size = MIN2(ShenandoahMaxRegionSize, region_size);
552
553 } else {
554 if (ShenandoahHeapRegionSize > max_heap_size / MIN_NUM_REGIONS) {
555 err_msg message("Max heap size (" SIZE_FORMAT "%s) is too low to afford the minimum number "
556 "of regions (" SIZE_FORMAT ") of requested size (" SIZE_FORMAT "%s).",
557 byte_size_in_proper_unit(max_heap_size), proper_unit_for_byte_size(max_heap_size),
558 MIN_NUM_REGIONS,
559 byte_size_in_proper_unit(ShenandoahHeapRegionSize), proper_unit_for_byte_size(ShenandoahHeapRegionSize));
560 vm_exit_during_initialization("Invalid -XX:ShenandoahHeapRegionSize option", message);
561 }
562 if (ShenandoahHeapRegionSize < ShenandoahMinRegionSize) {
563 err_msg message("Heap region size (" SIZE_FORMAT "%s) should be larger than min region size (" SIZE_FORMAT "%s).",
564 byte_size_in_proper_unit(ShenandoahHeapRegionSize), proper_unit_for_byte_size(ShenandoahHeapRegionSize),
565 byte_size_in_proper_unit(ShenandoahMinRegionSize), proper_unit_for_byte_size(ShenandoahMinRegionSize));
566 vm_exit_during_initialization("Invalid -XX:ShenandoahHeapRegionSize option", message);
567 }
568 if (ShenandoahHeapRegionSize > ShenandoahMaxRegionSize) {
569 err_msg message("Heap region size (" SIZE_FORMAT "%s) should be lower than max region size (" SIZE_FORMAT "%s).",
570 byte_size_in_proper_unit(ShenandoahHeapRegionSize), proper_unit_for_byte_size(ShenandoahHeapRegionSize),
571 byte_size_in_proper_unit(ShenandoahMaxRegionSize), proper_unit_for_byte_size(ShenandoahMaxRegionSize));
572 vm_exit_during_initialization("Invalid -XX:ShenandoahHeapRegionSize option", message);
573 }
574 region_size = ShenandoahHeapRegionSize;
575 }
576
577 // Make sure region size is at least one large page, if enabled.
578 // Otherwise, uncommitting one region may falsely uncommit the adjacent
579 // regions too.
580 // Also see shenandoahArguments.cpp, where it handles UseLargePages.
581 if (UseLargePages && ShenandoahUncommit) {
582 region_size = MAX2(region_size, os::large_page_size());
583 }
584
585 int region_size_log = log2_long((jlong) region_size);
586 // Recalculate the region size to make sure it's a power of
587 // 2. This means that region_size is the largest power of 2 that's
588 // <= what we've calculated so far.
589 region_size = size_t(1) << region_size_log;
590
591 // Now, set up the globals.
592 guarantee(RegionSizeBytesShift == 0, "we should only set it once");
593 RegionSizeBytesShift = (size_t)region_size_log;
594
595 guarantee(RegionSizeWordsShift == 0, "we should only set it once");
596 RegionSizeWordsShift = RegionSizeBytesShift - LogHeapWordSize;
597
598 guarantee(RegionSizeBytes == 0, "we should only set it once");
599 RegionSizeBytes = region_size;
600 RegionSizeWords = RegionSizeBytes >> LogHeapWordSize;
601 assert (RegionSizeWords*HeapWordSize == RegionSizeBytes, "sanity");
602
603 guarantee(RegionSizeWordsMask == 0, "we should only set it once");
604 RegionSizeWordsMask = RegionSizeWords - 1;
605
606 guarantee(RegionSizeBytesMask == 0, "we should only set it once");
607 RegionSizeBytesMask = RegionSizeBytes - 1;
608
609 guarantee(RegionCount == 0, "we should only set it once");
610 RegionCount = max_heap_size / RegionSizeBytes;
611 guarantee(RegionCount >= MIN_NUM_REGIONS, "Should have at least minimum regions");
612
613 guarantee(HumongousThresholdWords == 0, "we should only set it once");
614 HumongousThresholdWords = RegionSizeWords * ShenandoahHumongousThreshold / 100;
615 HumongousThresholdWords = align_down(HumongousThresholdWords, MinObjAlignment);
616 assert (HumongousThresholdWords <= RegionSizeWords, "sanity");
617
618 guarantee(HumongousThresholdBytes == 0, "we should only set it once");
619 HumongousThresholdBytes = HumongousThresholdWords * HeapWordSize;
620 assert (HumongousThresholdBytes <= RegionSizeBytes, "sanity");
621
622 // The rationale for trimming the TLAB sizes has to do with the raciness in
623 // TLAB allocation machinery. It may happen that TLAB sizing policy polls Shenandoah
624 // about next free size, gets the answer for region #N, goes away for a while, then
625 // tries to allocate in region #N, and fail because some other thread have claimed part
626 // of the region #N, and then the freeset allocation code has to retire the region #N,
627 // before moving the allocation to region #N+1.
628 //
629 // The worst case realizes when "answer" is "region size", which means it could
630 // prematurely retire an entire region. Having smaller TLABs does not fix that
631 // completely, but reduces the probability of too wasteful region retirement.
632 // With current divisor, we will waste no more than 1/8 of region size in the worst
633 // case. This also has a secondary effect on collection set selection: even under
634 // the race, the regions would be at least 7/8 used, which allows relying on
635 // "used" - "live" for cset selection. Otherwise, we can get the fragmented region
636 // below the garbage threshold that would never be considered for collection.
637 //
638 // The whole thing is mitigated if Elastic TLABs are enabled.
639 //
640 guarantee(MaxTLABSizeWords == 0, "we should only set it once");
641 MaxTLABSizeWords = MIN2(ShenandoahElasticTLAB ? RegionSizeWords : (RegionSizeWords / 8), HumongousThresholdWords);
642 MaxTLABSizeWords = align_down(MaxTLABSizeWords, MinObjAlignment);
643
644 guarantee(MaxTLABSizeBytes == 0, "we should only set it once");
645 MaxTLABSizeBytes = MaxTLABSizeWords * HeapWordSize;
646 assert (MaxTLABSizeBytes > MinTLABSize, "should be larger");
647
648 log_info(gc, init)("Regions: " SIZE_FORMAT " x " SIZE_FORMAT "%s",
649 RegionCount, byte_size_in_proper_unit(RegionSizeBytes), proper_unit_for_byte_size(RegionSizeBytes));
650 log_info(gc, init)("Humongous object threshold: " SIZE_FORMAT "%s",
651 byte_size_in_proper_unit(HumongousThresholdBytes), proper_unit_for_byte_size(HumongousThresholdBytes));
652 log_info(gc, init)("Max TLAB size: " SIZE_FORMAT "%s",
653 byte_size_in_proper_unit(MaxTLABSizeBytes), proper_unit_for_byte_size(MaxTLABSizeBytes));
654 }
655
656 void ShenandoahHeapRegion::do_commit() {
657 if (!_heap->is_heap_region_special() && !os::commit_memory((char *) _reserved.start(), _reserved.byte_size(), false)) {
658 report_java_out_of_memory("Unable to commit region");
659 }
660 if (!_heap->commit_bitmap_slice(this)) {
661 report_java_out_of_memory("Unable to commit bitmaps for region");
662 }
663 _heap->increase_committed(ShenandoahHeapRegion::region_size_bytes());
664 }
665
666 void ShenandoahHeapRegion::do_uncommit() {
667 if (!_heap->is_heap_region_special() && !os::uncommit_memory((char *) _reserved.start(), _reserved.byte_size())) {
668 report_java_out_of_memory("Unable to uncommit region");
669 }
670 if (!_heap->uncommit_bitmap_slice(this)) {
671 report_java_out_of_memory("Unable to uncommit bitmaps for region");
672 }
673 _heap->decrease_committed(ShenandoahHeapRegion::region_size_bytes());
674 }
675
676 void ShenandoahHeapRegion::set_state(RegionState to) {
677 EventShenandoahHeapRegionStateChange evt;
678 if (evt.should_commit()){
679 evt.set_index((unsigned)region_number());
680 evt.set_start((uintptr_t)bottom());
681 evt.set_used(used());
682 evt.set_from(_state);
683 evt.set_to(to);
684 evt.commit();
685 }
686 _state = to;
687 }
688
689 void ShenandoahHeapRegion::record_pin() {
690 Atomic::add((size_t)1, &_critical_pins);
691 }
692
693 void ShenandoahHeapRegion::record_unpin() {
694 assert(pin_count() > 0, "Region " SIZE_FORMAT " should have non-zero pins", region_number());
695 Atomic::sub((size_t)1, &_critical_pins);
696 }
697
698 size_t ShenandoahHeapRegion::pin_count() const {
699 return Atomic::load(&_critical_pins);
700 }