1 /*
2 * Copyright (c) 2013, 2015, Red Hat, Inc. and/or its affiliates.
3 *
4 * This code is free software; you can redistribute it and/or modify it
5 * under the terms of the GNU General Public License version 2 only, as
6 * published by the Free Software Foundation.
7 *
8 * This code is distributed in the hope that it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
11 * version 2 for more details (a copy is included in the LICENSE file that
12 * accompanied this code).
13 *
14 * You should have received a copy of the GNU General Public License version
15 * 2 along with this work; if not, write to the Free Software Foundation,
16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
17 *
18 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
19 * or visit www.oracle.com if you need additional information or have any
20 * questions.
21 *
22 */
23
24 #include "precompiled.hpp"
25 #include "memory/allocation.hpp"
26 #include "gc/shenandoah/brooksPointer.hpp"
27 #include "gc/shenandoah/shenandoahConnectionMatrix.hpp"
28 #include "gc/shenandoah/shenandoahHeap.hpp"
29 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
30 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
31 #include "gc/shared/space.inline.hpp"
32 #include "memory/universe.hpp"
33 #include "oops/oop.inline.hpp"
34 #include "runtime/java.hpp"
35 #include "runtime/mutexLocker.hpp"
36 #include "runtime/os.hpp"
37 #include "runtime/safepoint.hpp"
38
39 size_t ShenandoahHeapRegion::RegionSizeBytes = 0;
40 size_t ShenandoahHeapRegion::RegionSizeWords = 0;
41 size_t ShenandoahHeapRegion::RegionSizeBytesShift = 0;
42 size_t ShenandoahHeapRegion::RegionSizeWordsShift = 0;
43 size_t ShenandoahHeapRegion::RegionSizeBytesMask = 0;
44 size_t ShenandoahHeapRegion::RegionSizeWordsMask = 0;
45 size_t ShenandoahHeapRegion::HumongousThresholdBytes = 0;
46 size_t ShenandoahHeapRegion::HumongousThresholdWords = 0;
47 size_t ShenandoahHeapRegion::MaxTLABSizeBytes = 0;
48
49 // start with 1, reserve 0 for uninitialized value
50 uint64_t ShenandoahHeapRegion::AllocSeqNum = 1;
51
52 ShenandoahHeapRegion::ShenandoahHeapRegion(ShenandoahHeap* heap, HeapWord* start,
53 size_t size_words, size_t index, bool committed) :
54 _heap(heap),
55 _pacer(ShenandoahPacing ? heap->pacer() : NULL),
56 _region_number(index),
57 _live_data(0),
58 _tlab_allocs(0),
59 _gclab_allocs(0),
60 _shared_allocs(0),
61 _reserved(MemRegion(start, size_words)),
62 _root(false),
63 _new_top(NULL),
64 _seqnum_first_alloc_mutator(0),
65 _seqnum_last_alloc_mutator(0),
66 _seqnum_first_alloc_gc(0),
67 _seqnum_last_alloc_gc(0),
68 _state(committed ? _empty_committed : _empty_uncommitted),
69 _empty_time(os::elapsedTime()),
70 _initialized(false),
71 _critical_pins(0) {
72
73 ContiguousSpace::initialize(_reserved, true, committed);
74 }
75
76 size_t ShenandoahHeapRegion::region_number() const {
77 return _region_number;
78 }
79
80 void ShenandoahHeapRegion::report_illegal_transition(const char *method) {
81 ResourceMark rm;
82 stringStream ss;
83 ss.print("Illegal region state transition from \"%s\", at %s\n ", region_state_to_string(_state), method);
84 print_on(&ss);
85 fatal("%s", ss.as_string());
86 }
87
88 void ShenandoahHeapRegion::make_regular_allocation() {
89 _heap->assert_heaplock_owned_by_current_thread();
90
91 switch (_state) {
92 case _empty_uncommitted:
93 do_commit();
94 case _empty_committed:
95 _state = _regular;
96 case _regular:
97 case _pinned:
98 return;
99 default:
100 report_illegal_transition("regular allocation");
101 }
102 }
103
104 void ShenandoahHeapRegion::make_regular_bypass() {
105 _heap->assert_heaplock_owned_by_current_thread();
106 assert (_heap->is_full_gc_in_progress() || _heap->is_degenerated_gc_in_progress(),
107 "only for full or degen GC");
108
109 switch (_state) {
110 case _empty_uncommitted:
111 do_commit();
112 case _empty_committed:
113 case _cset:
114 case _humongous_start:
115 case _humongous_cont:
116 _state = _regular;
117 return;
118 case _pinned_cset:
119 _state = _pinned;
120 return;
121 case _regular:
122 case _pinned:
123 return;
124 default:
125 report_illegal_transition("regular bypass");
126 }
127 }
128
129 void ShenandoahHeapRegion::make_humongous_start() {
130 _heap->assert_heaplock_owned_by_current_thread();
131 switch (_state) {
132 case _empty_uncommitted:
133 do_commit();
134 case _empty_committed:
135 _state = _humongous_start;
136 return;
137 default:
138 report_illegal_transition("humongous start allocation");
139 }
140 }
141
142 void ShenandoahHeapRegion::make_humongous_start_bypass() {
143 _heap->assert_heaplock_owned_by_current_thread();
144 assert (_heap->is_full_gc_in_progress(), "only for full GC");
145
146 switch (_state) {
147 case _empty_committed:
148 case _regular:
149 case _humongous_start:
150 case _humongous_cont:
151 _state = _humongous_start;
152 return;
153 default:
154 report_illegal_transition("humongous start bypass");
155 }
156 }
157
158 void ShenandoahHeapRegion::make_humongous_cont() {
159 _heap->assert_heaplock_owned_by_current_thread();
160 switch (_state) {
161 case _empty_uncommitted:
162 do_commit();
163 case _empty_committed:
164 _state = _humongous_cont;
165 return;
166 default:
167 report_illegal_transition("humongous continuation allocation");
168 }
169 }
170
171 void ShenandoahHeapRegion::make_humongous_cont_bypass() {
172 _heap->assert_heaplock_owned_by_current_thread();
173 assert (_heap->is_full_gc_in_progress(), "only for full GC");
174
175 switch (_state) {
176 case _empty_committed:
177 case _regular:
178 case _humongous_start:
179 case _humongous_cont:
180 _state = _humongous_cont;
181 return;
182 default:
183 report_illegal_transition("humongous continuation bypass");
184 }
185 }
186
187 void ShenandoahHeapRegion::make_pinned() {
188 _heap->assert_heaplock_owned_by_current_thread();
189 switch (_state) {
190 case _regular:
191 assert (_critical_pins == 0, "sanity");
192 _state = _pinned;
193 case _pinned_cset:
194 case _pinned:
195 _critical_pins++;
196 return;
197 case _humongous_start:
198 assert (_critical_pins == 0, "sanity");
199 _state = _pinned_humongous_start;
200 case _pinned_humongous_start:
201 _critical_pins++;
202 return;
203 case _cset:
204 guarantee(_heap->cancelled_concgc(), "only valid when evac has been cancelled");
205 assert (_critical_pins == 0, "sanity");
206 _state = _pinned_cset;
207 _critical_pins++;
208 return;
209 default:
210 report_illegal_transition("pinning");
211 }
212 }
213
214 void ShenandoahHeapRegion::make_unpinned() {
215 _heap->assert_heaplock_owned_by_current_thread();
216 switch (_state) {
217 case _pinned:
218 assert (_critical_pins > 0, "sanity");
219 _critical_pins--;
220 if (_critical_pins == 0) {
221 _state = _regular;
222 }
223 return;
224 case _regular:
225 case _humongous_start:
226 assert (_critical_pins == 0, "sanity");
227 return;
228 case _pinned_cset:
229 guarantee(_heap->cancelled_concgc(), "only valid when evac has been cancelled");
230 assert (_critical_pins > 0, "sanity");
231 _critical_pins--;
232 if (_critical_pins == 0) {
233 _state = _cset;
234 }
235 return;
236 case _pinned_humongous_start:
237 assert (_critical_pins > 0, "sanity");
238 _critical_pins--;
239 if (_critical_pins == 0) {
240 _state = _humongous_start;
241 }
242 return;
243 default:
244 report_illegal_transition("unpinning");
245 }
246 }
247
248 void ShenandoahHeapRegion::make_cset() {
249 _heap->assert_heaplock_owned_by_current_thread();
250 switch (_state) {
251 case _regular:
252 _state = _cset;
253 case _cset:
254 return;
255 default:
256 report_illegal_transition("cset");
257 }
258 }
259
260 void ShenandoahHeapRegion::make_trash() {
261 _heap->assert_heaplock_owned_by_current_thread();
262 switch (_state) {
263 case _cset:
264 // Reclaiming cset regions
265 case _humongous_start:
266 case _humongous_cont:
267 // Reclaiming humongous regions
268 case _regular:
269 // Immediate region reclaim
270 _state = _trash;
271 return;
272 default:
273 report_illegal_transition("trashing");
274 }
275 }
276
277 void ShenandoahHeapRegion::make_empty() {
278 _heap->assert_heaplock_owned_by_current_thread();
279 switch (_state) {
280 case _trash:
281 _state = _empty_committed;
282 _empty_time = os::elapsedTime();
283 return;
284 default:
285 report_illegal_transition("emptying");
286 }
287 }
288
289 void ShenandoahHeapRegion::make_uncommitted() {
290 _heap->assert_heaplock_owned_by_current_thread();
291 switch (_state) {
292 case _empty_committed:
293 do_uncommit();
294 _state = _empty_uncommitted;
295 return;
296 default:
297 report_illegal_transition("uncommiting");
298 }
299 }
300
301 void ShenandoahHeapRegion::make_committed_bypass() {
302 _heap->assert_heaplock_owned_by_current_thread();
303 assert (_heap->is_full_gc_in_progress(), "only for full GC");
304
305 switch (_state) {
306 case _empty_uncommitted:
307 do_commit();
308 _state = _empty_committed;
309 return;
310 default:
311 report_illegal_transition("commit bypass");
312 }
313 }
314
315 bool ShenandoahHeapRegion::rollback_allocation(uint size) {
316 set_top(top() - size);
317 return true;
318 }
319
320 void ShenandoahHeapRegion::clear_live_data() {
321 OrderAccess::release_store_fence<size_t>(&_live_data, 0);
322 }
323
324 void ShenandoahHeapRegion::reset_alloc_metadata() {
325 _tlab_allocs = 0;
326 _gclab_allocs = 0;
327 _shared_allocs = 0;
328 _seqnum_first_alloc_mutator = 0;
329 _seqnum_last_alloc_mutator = 0;
330 _seqnum_first_alloc_gc = 0;
331 _seqnum_last_alloc_gc = 0;
332 }
333
334 void ShenandoahHeapRegion::reset_alloc_metadata_to_shared() {
335 if (used() > 0) {
336 _tlab_allocs = 0;
337 _gclab_allocs = 0;
338 _shared_allocs = used() >> LogHeapWordSize;
339 uint64_t next = AllocSeqNum++;
340 _seqnum_first_alloc_mutator = next;
341 _seqnum_last_alloc_mutator = next;
342 _seqnum_first_alloc_gc = 0;
343 _seqnum_last_alloc_gc = 0;
344 } else {
345 reset_alloc_metadata();
346 }
347 }
348
349 size_t ShenandoahHeapRegion::get_shared_allocs() const {
350 return _shared_allocs * HeapWordSize;
351 }
352
353 size_t ShenandoahHeapRegion::get_tlab_allocs() const {
354 return _tlab_allocs * HeapWordSize;
355 }
356
357 size_t ShenandoahHeapRegion::get_gclab_allocs() const {
358 return _gclab_allocs * HeapWordSize;
359 }
360
361 void ShenandoahHeapRegion::set_live_data(size_t s) {
362 assert(Thread::current()->is_VM_thread(), "by VM thread");
363 _live_data = (s >> LogHeapWordSize);
364 }
365
366 size_t ShenandoahHeapRegion::get_live_data_words() const {
367 return OrderAccess::load_acquire(&_live_data);
368 }
369
370 size_t ShenandoahHeapRegion::get_live_data_bytes() const {
371 return get_live_data_words() * HeapWordSize;
372 }
373
374 bool ShenandoahHeapRegion::has_live() const {
375 return get_live_data_words() != 0;
376 }
377
378 size_t ShenandoahHeapRegion::garbage() const {
379 assert(used() >= get_live_data_bytes(), "Live Data must be a subset of used() live: "SIZE_FORMAT" used: "SIZE_FORMAT,
380 get_live_data_bytes(), used());
381
382 size_t result = used() - get_live_data_bytes();
383 return result;
384 }
385
386 bool ShenandoahHeapRegion::in_collection_set() const {
387 return _heap->region_in_collection_set(_region_number);
388 }
389
390 void ShenandoahHeapRegion::print_on(outputStream* st) const {
391 st->print("|");
392 st->print(SIZE_FORMAT_W(5), this->_region_number);
393
394 switch (_state) {
395 case _empty_uncommitted:
396 st->print("|EU ");
397 break;
398 case _empty_committed:
399 st->print("|EC ");
400 break;
401 case _regular:
402 st->print("|R ");
403 break;
404 case _humongous_start:
405 st->print("|H ");
406 break;
407 case _pinned_humongous_start:
408 st->print("|HP ");
409 break;
410 case _humongous_cont:
411 st->print("|HC ");
412 break;
413 case _cset:
414 st->print("|CS ");
415 break;
416 case _trash:
417 st->print("|T ");
418 break;
419 case _pinned:
420 st->print("|P ");
421 break;
422 case _pinned_cset:
423 st->print("|CSP");
424 break;
425 default:
426 ShouldNotReachHere();
427 }
428 st->print("|BTE " INTPTR_FORMAT_W(12) ", " INTPTR_FORMAT_W(12) ", " INTPTR_FORMAT_W(12),
429 p2i(bottom()), p2i(top()), p2i(end()));
430 st->print("|TAMS " INTPTR_FORMAT_W(12) ", " INTPTR_FORMAT_W(12),
431 p2i(_heap->complete_top_at_mark_start(_bottom)),
432 p2i(_heap->next_top_at_mark_start(_bottom)));
433 st->print("|U %3d%%", (int) ((double) used() * 100 / capacity()));
434 st->print("|T %3d%%", (int) ((double) get_tlab_allocs() * 100 / capacity()));
435 st->print("|G %3d%%", (int) ((double) get_gclab_allocs() * 100 / capacity()));
436 st->print("|S %3d%%", (int) ((double) get_shared_allocs() * 100 / capacity()));
437 st->print("|L %3d%%", (int) ((double) get_live_data_bytes() * 100 / capacity()));
438 if (is_root()) {
439 st->print("|R");
440 } else {
441 st->print("| ");
442 }
443 st->print("|CP " SIZE_FORMAT_W(3), _critical_pins);
444 st->print("|SN " UINT64_FORMAT_HEX_W(12) ", " UINT64_FORMAT_HEX_W(8) ", " UINT64_FORMAT_HEX_W(8) ", " UINT64_FORMAT_HEX_W(8),
445 seqnum_first_alloc_mutator(), seqnum_last_alloc_mutator(),
446 seqnum_first_alloc_gc(), seqnum_last_alloc_gc());
447 st->cr();
448 }
449
450 void ShenandoahHeapRegion::oop_iterate(ExtendedOopClosure* blk) {
451 if (!is_active()) return;
452 if (is_humongous()) {
453 oop_iterate_humongous(blk);
454 } else {
455 oop_iterate_objects(blk);
456 }
457 }
458
459 void ShenandoahHeapRegion::oop_iterate_objects(ExtendedOopClosure* blk) {
460 assert(! is_humongous(), "no humongous region here");
461 HeapWord* obj_addr = bottom() + BrooksPointer::word_size();
462 HeapWord* t = top();
463 // Could call objects iterate, but this is easier.
464 while (obj_addr < t) {
465 oop obj = oop(obj_addr);
466 obj_addr += obj->oop_iterate_size(blk) + BrooksPointer::word_size();
467 }
468 }
469
470 void ShenandoahHeapRegion::oop_iterate_humongous(ExtendedOopClosure* blk) {
471 assert(is_humongous(), "only humongous region here");
472 // Find head.
473 ShenandoahHeapRegion* r = humongous_start_region();
474 assert(r->is_humongous_start(), "need humongous head here");
475 oop obj = oop(r->bottom() + BrooksPointer::word_size());
476 obj->oop_iterate(blk, MemRegion(bottom(), top()));
477 }
478
479 void ShenandoahHeapRegion::fill_region() {
480 if (free() > (BrooksPointer::word_size() + CollectedHeap::min_fill_size())) {
481 HeapWord* filler = allocate(BrooksPointer::word_size(), ShenandoahHeap::_alloc_shared);
482 HeapWord* obj = allocate(end() - top(), ShenandoahHeap::_alloc_shared);
483 _heap->fill_with_object(obj, end() - obj);
484 BrooksPointer::initialize(oop(obj));
485 }
486 }
487
488 ShenandoahHeapRegion* ShenandoahHeapRegion::humongous_start_region() const {
489 assert(is_humongous(), "Must be a part of the humongous region");
490 size_t reg_num = region_number();
491 ShenandoahHeapRegion* r = const_cast<ShenandoahHeapRegion*>(this);
492 while (!r->is_humongous_start()) {
493 assert(reg_num > 0, "Sanity");
494 reg_num --;
495 r = _heap->get_region(reg_num);
496 assert(r->is_humongous(), "Must be a part of the humongous region");
497 }
498 assert(r->is_humongous_start(), "Must be");
499 return r;
500 }
501
502 void ShenandoahHeapRegion::recycle() {
503 ContiguousSpace::clear(false);
504 if (ZapUnusedHeapArea) {
505 ContiguousSpace::mangle_unused_area_complete();
506 }
507 clear_live_data();
508 _root = false;
509
510 reset_alloc_metadata();
511
512 // Reset C-TAMS pointer to ensure size-based iteration, everything
513 // in that regions is going to be new objects.
514 _heap->set_complete_top_at_mark_start(bottom(), bottom());
515 // We can only safely reset the C-TAMS pointer if the bitmap is clear for that region.
516 assert(_heap->is_complete_bitmap_clear_range(bottom(), end()), "must be clear");
517
518 if (UseShenandoahMatrix) {
519 _heap->connection_matrix()->clear_region(region_number());
520 }
521
522 make_empty();
523 }
524
525 HeapWord* ShenandoahHeapRegion::block_start_const(const void* p) const {
526 assert(MemRegion(bottom(), end()).contains(p),
527 "p ("PTR_FORMAT") not in space ["PTR_FORMAT", "PTR_FORMAT")",
528 p2i(p), p2i(bottom()), p2i(end()));
529 if (p >= top()) {
530 return top();
531 } else {
532 HeapWord* last = bottom() + BrooksPointer::word_size();
533 HeapWord* cur = last;
534 while (cur <= p) {
535 last = cur;
536 cur += oop(cur)->size() + BrooksPointer::word_size();
537 }
538 assert(oopDesc::is_oop(oop(last)),
539 PTR_FORMAT" should be an object start", p2i(last));
540 return last;
541 }
542 }
543
544 void ShenandoahHeapRegion::setup_heap_region_size(size_t initial_heap_size, size_t max_heap_size) {
545 // Absolute minimums we should not ever break.
546 static const size_t MIN_REGION_SIZE = 256*K;
547 static const size_t MIN_NUM_REGIONS = 10;
548
549 if (FLAG_IS_DEFAULT(ShenandoahMinRegionSize)) {
550 FLAG_SET_DEFAULT(ShenandoahMinRegionSize, MIN_REGION_SIZE);
551 }
552
553 uintx region_size;
554 if (FLAG_IS_DEFAULT(ShenandoahHeapRegionSize)) {
555 if (ShenandoahMinRegionSize > initial_heap_size / MIN_NUM_REGIONS) {
556 err_msg message("Initial heap size (" SIZE_FORMAT "K) is too low to afford the minimum number "
557 "of regions (" SIZE_FORMAT ") of minimum region size (" SIZE_FORMAT "K).",
558 initial_heap_size/K, MIN_NUM_REGIONS, ShenandoahMinRegionSize/K);
559 vm_exit_during_initialization("Invalid -XX:ShenandoahMinRegionSize option", message);
560 }
561 if (ShenandoahMinRegionSize < MIN_REGION_SIZE) {
562 err_msg message("" SIZE_FORMAT "K should not be lower than minimum region size (" SIZE_FORMAT "K).",
563 ShenandoahMinRegionSize/K, MIN_REGION_SIZE/K);
564 vm_exit_during_initialization("Invalid -XX:ShenandoahMinRegionSize option", message);
565 }
566 if (ShenandoahMinRegionSize < MinTLABSize) {
567 err_msg message("" SIZE_FORMAT "K should not be lower than TLAB size size (" SIZE_FORMAT "K).",
568 ShenandoahMinRegionSize/K, MinTLABSize/K);
569 vm_exit_during_initialization("Invalid -XX:ShenandoahMinRegionSize option", message);
570 }
571 if (ShenandoahMaxRegionSize < MIN_REGION_SIZE) {
572 err_msg message("" SIZE_FORMAT "K should not be lower than min region size (" SIZE_FORMAT "K).",
573 ShenandoahMaxRegionSize/K, MIN_REGION_SIZE/K);
574 vm_exit_during_initialization("Invalid -XX:ShenandoahMaxRegionSize option", message);
575 }
576 if (ShenandoahMinRegionSize > ShenandoahMaxRegionSize) {
577 err_msg message("Minimum (" SIZE_FORMAT "K) should be larger than maximum (" SIZE_FORMAT "K).",
578 ShenandoahMinRegionSize/K, ShenandoahMaxRegionSize/K);
579 vm_exit_during_initialization("Invalid -XX:ShenandoahMinRegionSize or -XX:ShenandoahMaxRegionSize", message);
580 }
581 size_t average_heap_size = (initial_heap_size + max_heap_size) / 2;
582 region_size = MAX2(average_heap_size / ShenandoahTargetNumRegions,
583 ShenandoahMinRegionSize);
584
585 // Now make sure that we don't go over or under our limits.
586 region_size = MAX2(ShenandoahMinRegionSize, region_size);
587 region_size = MIN2(ShenandoahMaxRegionSize, region_size);
588
589 } else {
590 if (ShenandoahHeapRegionSize > initial_heap_size / MIN_NUM_REGIONS) {
591 err_msg message("Initial heap size (" SIZE_FORMAT "K) is too low to afford the minimum number "
592 "of regions (" SIZE_FORMAT ") of requested size (" SIZE_FORMAT "K).",
593 initial_heap_size/K, MIN_NUM_REGIONS, ShenandoahHeapRegionSize/K);
594 vm_exit_during_initialization("Invalid -XX:ShenandoahHeapRegionSize option", message);
595 }
596 if (ShenandoahHeapRegionSize < ShenandoahMinRegionSize) {
597 err_msg message("Heap region size (" SIZE_FORMAT "K) should be larger than min region size (" SIZE_FORMAT "K).",
598 ShenandoahHeapRegionSize/K, ShenandoahMinRegionSize/K);
599 vm_exit_during_initialization("Invalid -XX:ShenandoahHeapRegionSize option", message);
600 }
601 if (ShenandoahHeapRegionSize > ShenandoahMaxRegionSize) {
602 err_msg message("Heap region size (" SIZE_FORMAT "K) should be lower than max region size (" SIZE_FORMAT "K).",
603 ShenandoahHeapRegionSize/K, ShenandoahMaxRegionSize/K);
604 vm_exit_during_initialization("Invalid -XX:ShenandoahHeapRegionSize option", message);
605 }
606 region_size = ShenandoahHeapRegionSize;
607 }
608
609 // Make sure region size is at least one large page, if enabled.
610 // Otherwise, mem-protecting one region may falsely protect the adjacent
611 // regions too.
612 if (UseLargePages) {
613 region_size = MAX2(region_size, os::large_page_size());
614 }
615
616 int region_size_log = log2_long((jlong) region_size);
617 // Recalculate the region size to make sure it's a power of
618 // 2. This means that region_size is the largest power of 2 that's
619 // <= what we've calculated so far.
620 region_size = ((uintx)1 << region_size_log);
621
622 // Now, set up the globals.
623 guarantee(RegionSizeBytesShift == 0, "we should only set it once");
624 RegionSizeBytesShift = (size_t)region_size_log;
625
626 guarantee(RegionSizeWordsShift == 0, "we should only set it once");
627 RegionSizeWordsShift = RegionSizeBytesShift - LogHeapWordSize;
628
629 guarantee(RegionSizeBytes == 0, "we should only set it once");
630 RegionSizeBytes = (size_t)region_size;
631 RegionSizeWords = RegionSizeBytes >> LogHeapWordSize;
632 assert (RegionSizeWords*HeapWordSize == RegionSizeBytes, "sanity");
633
634 guarantee(RegionSizeWordsMask == 0, "we should only set it once");
635 RegionSizeWordsMask = RegionSizeWords - 1;
636
637 guarantee(RegionSizeBytesMask == 0, "we should only set it once");
638 RegionSizeBytesMask = RegionSizeBytes - 1;
639
640 guarantee(HumongousThresholdWords == 0, "we should only set it once");
641 HumongousThresholdWords = RegionSizeWords * ShenandoahHumongousThreshold / 100;
642 assert (HumongousThresholdWords <= RegionSizeWords, "sanity");
643
644 guarantee(HumongousThresholdBytes == 0, "we should only set it once");
645 HumongousThresholdBytes = HumongousThresholdWords * HeapWordSize;
646 assert (HumongousThresholdBytes <= RegionSizeBytes, "sanity");
647
648 // The rationale for trimming the TLAB sizes has to do with the raciness in
649 // TLAB allocation machinery. It may happen that TLAB sizing policy polls Shenandoah
650 // about next free size, gets the answer for region #N, goes away for a while, then
651 // tries to allocate in region #N, and fail because some other thread have claimed part
652 // of the region #N, and then the freeset allocation code has to retire the region #N,
653 // before moving the allocation to region #N+1.
654 //
655 // The worst case realizes when "answer" is "region size", which means it could
656 // prematurely retire an entire region. Having smaller TLABs does not fix that
657 // completely, but reduces the probability of too wasteful region retirement.
658 // With current divisor, we will waste no more than 1/8 of region size in the worst
659 // case. This also has a secondary effect on collection set selection: even under
660 // the race, the regions would be at least 7/8 used, which allows relying on
661 // "used" - "live" for cset selection. Otherwise, we can get the fragmented region
662 // below the garbage threshold that would never be considered for collection.
663 guarantee(MaxTLABSizeBytes == 0, "we should only set it once");
664 MaxTLABSizeBytes = MIN2(RegionSizeBytes / 8, HumongousThresholdBytes);
665 assert (MaxTLABSizeBytes > MinTLABSize, "should be larger");
666
667 log_info(gc, heap)("Heap region size: " SIZE_FORMAT "M", RegionSizeBytes / M);
668 log_info(gc, init)("Region size in bytes: "SIZE_FORMAT, RegionSizeBytes);
669 log_info(gc, init)("Region size byte shift: "SIZE_FORMAT, RegionSizeBytesShift);
670 log_info(gc, init)("Humongous threshold in bytes: "SIZE_FORMAT, HumongousThresholdBytes);
671 log_info(gc, init)("Max TLAB size in bytes: "SIZE_FORMAT, MaxTLABSizeBytes);
672 log_info(gc, init)("Number of regions: "SIZE_FORMAT, max_heap_size / RegionSizeBytes);
673 }
674
675 CompactibleSpace* ShenandoahHeapRegion::next_compaction_space() const {
676 return _heap->next_compaction_region(this);
677 }
678
679 void ShenandoahHeapRegion::prepare_for_compaction(CompactPoint* cp) {
680 scan_and_forward(this, cp);
681 }
682
683 void ShenandoahHeapRegion::adjust_pointers() {
684 // Check first is there is any work to do.
685 if (used() == 0) {
686 return; // Nothing to do.
687 }
688
689 scan_and_adjust_pointers(this);
690 }
691
692 void ShenandoahHeapRegion::compact() {
693 assert(!is_humongous(), "Shouldn't be compacting humongous regions");
694 scan_and_compact(this);
695 }
696
697 void ShenandoahHeapRegion::do_commit() {
698 if (_initialized && can_idle_region()) {
699 os::activate_memory((char *)_reserved.start(), _reserved.byte_size());
700 _heap->activate_bitmap_slice(this);
701 } else {
702 if (!os::commit_memory((char *) _reserved.start(), _reserved.byte_size(), false)) {
703 report_java_out_of_memory("Unable to commit region");
704 }
705 if (!_heap->commit_bitmap_slice(this)) {
706 report_java_out_of_memory("Unable to commit bitmaps for region");
707 }
708
709 _initialized = true;
710 }
711 _heap->increase_committed(ShenandoahHeapRegion::region_size_bytes());
712 }
713
714 void ShenandoahHeapRegion::do_uncommit() {
715 if (can_idle_region()) {
716 if (!os::idle_memory((char *)_reserved.start(), _reserved.byte_size())) {
717 report_java_out_of_memory("Unable to idle the region");
718 }
719
720 if (!_heap->idle_bitmap_slice(this)) {
721 report_java_out_of_memory("Unable to idle bitmaps for region");
722 }
723 } else {
724 if (!os::uncommit_memory((char *) _reserved.start(), _reserved.byte_size())) {
725 report_java_out_of_memory("Unable to uncommit region");
726 }
727 if (!_heap->uncommit_bitmap_slice(this)) {
728 report_java_out_of_memory("Unable to uncommit bitmaps for region");
729 }
730 }
731 _heap->decrease_committed(ShenandoahHeapRegion::region_size_bytes());
732 }
733
734
735 bool ShenandoahHeapRegion::can_idle_region() const {
736 return LINUX_ONLY(ShenandoahUncommitWithIdle && !UseLargePages) NOT_LINUX(false);
737 }