1 /*
2 * Copyright (c) 2013, 2015, Red Hat, Inc. and/or its affiliates.
3 *
4 * This code is free software; you can redistribute it and/or modify it
5 * under the terms of the GNU General Public License version 2 only, as
6 * published by the Free Software Foundation.
7 *
8 * This code is distributed in the hope that it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
11 * version 2 for more details (a copy is included in the LICENSE file that
12 * accompanied this code).
13 *
14 * You should have received a copy of the GNU General Public License version
15 * 2 along with this work; if not, write to the Free Software Foundation,
16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
17 *
18 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
19 * or visit www.oracle.com if you need additional information or have any
20 * questions.
21 *
22 */
23
24 #include "precompiled.hpp"
25 #include "memory/allocation.hpp"
26 #include "gc/shenandoah/brooksPointer.hpp"
27 #include "gc/shenandoah/shenandoahHeapRegionSet.inline.hpp"
28 #include "gc/shenandoah/shenandoahConnectionMatrix.hpp"
29 #include "gc/shenandoah/shenandoahHeap.hpp"
30 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
31 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
32 #include "gc/shenandoah/shenandoahTraversalGC.hpp"
33 #include "gc/shared/space.inline.hpp"
34 #include "memory/universe.hpp"
35 #include "oops/oop.inline.hpp"
36 #include "runtime/java.hpp"
37 #include "runtime/mutexLocker.hpp"
38 #include "runtime/os.hpp"
39 #include "runtime/safepoint.hpp"
40
41 size_t ShenandoahHeapRegion::RegionSizeBytes = 0;
42 size_t ShenandoahHeapRegion::RegionSizeWords = 0;
43 size_t ShenandoahHeapRegion::RegionSizeBytesShift = 0;
44 size_t ShenandoahHeapRegion::RegionSizeWordsShift = 0;
45 size_t ShenandoahHeapRegion::RegionSizeBytesMask = 0;
46 size_t ShenandoahHeapRegion::RegionSizeWordsMask = 0;
47 size_t ShenandoahHeapRegion::HumongousThresholdBytes = 0;
48 size_t ShenandoahHeapRegion::HumongousThresholdWords = 0;
49 size_t ShenandoahHeapRegion::MaxTLABSizeBytes = 0;
50
51 // start with 1, reserve 0 for uninitialized value
52 uint64_t ShenandoahHeapRegion::AllocSeqNum = 1;
53
54 ShenandoahHeapRegion::ShenandoahHeapRegion(ShenandoahHeap* heap, HeapWord* start,
55 size_t size_words, size_t index, bool committed) :
56 _heap(heap),
57 _pacer(ShenandoahPacing ? heap->pacer() : NULL),
58 _region_number(index),
59 _live_data(0),
60 _tlab_allocs(0),
61 _gclab_allocs(0),
62 _shared_allocs(0),
63 _reserved(MemRegion(start, size_words)),
64 _new_top(NULL),
65 _seqnum_first_alloc_mutator(0),
66 _seqnum_last_alloc_mutator(0),
67 _seqnum_first_alloc_gc(0),
68 _seqnum_last_alloc_gc(0),
69 _state(committed ? _empty_committed : _empty_uncommitted),
70 _empty_time(os::elapsedTime()),
71 _initialized(false),
72 _critical_pins(0) {
73
74 ContiguousSpace::initialize(_reserved, true, committed);
75 }
76
77 size_t ShenandoahHeapRegion::region_number() const {
78 return _region_number;
79 }
80
81 void ShenandoahHeapRegion::report_illegal_transition(const char *method) {
82 ResourceMark rm;
83 stringStream ss;
84 ss.print("Illegal region state transition from \"%s\", at %s\n ", region_state_to_string(_state), method);
85 print_on(&ss);
86 fatal("%s", ss.as_string());
87 }
88
89 void ShenandoahHeapRegion::make_regular_allocation() {
90 _heap->assert_heaplock_owned_by_current_thread();
91
92 switch (_state) {
93 case _empty_uncommitted:
94 do_commit();
95 case _empty_committed:
96 _state = _regular;
97 case _regular:
98 case _pinned:
99 return;
100 default:
101 report_illegal_transition("regular allocation");
102 }
103 }
104
105 void ShenandoahHeapRegion::make_regular_bypass() {
106 _heap->assert_heaplock_owned_by_current_thread();
107 assert (_heap->is_full_gc_in_progress() || _heap->is_degenerated_gc_in_progress(),
108 "only for full or degen GC");
109
110 switch (_state) {
111 case _empty_uncommitted:
112 do_commit();
113 case _empty_committed:
114 case _cset:
115 case _humongous_start:
116 case _humongous_cont:
117 _state = _regular;
118 return;
119 case _pinned_cset:
120 _state = _pinned;
121 return;
122 case _regular:
123 case _pinned:
124 return;
125 default:
126 report_illegal_transition("regular bypass");
127 }
128 }
129
130 void ShenandoahHeapRegion::make_humongous_start() {
131 _heap->assert_heaplock_owned_by_current_thread();
132 switch (_state) {
133 case _empty_uncommitted:
134 do_commit();
135 case _empty_committed:
136 _state = _humongous_start;
137 return;
138 default:
139 report_illegal_transition("humongous start allocation");
140 }
141 }
142
143 void ShenandoahHeapRegion::make_humongous_start_bypass() {
144 _heap->assert_heaplock_owned_by_current_thread();
145 assert (_heap->is_full_gc_in_progress(), "only for full GC");
146
147 switch (_state) {
148 case _empty_committed:
149 case _regular:
150 case _humongous_start:
151 case _humongous_cont:
152 _state = _humongous_start;
153 return;
154 default:
155 report_illegal_transition("humongous start bypass");
156 }
157 }
158
159 void ShenandoahHeapRegion::make_humongous_cont() {
160 _heap->assert_heaplock_owned_by_current_thread();
161 switch (_state) {
162 case _empty_uncommitted:
163 do_commit();
164 case _empty_committed:
165 _state = _humongous_cont;
166 return;
167 default:
168 report_illegal_transition("humongous continuation allocation");
169 }
170 }
171
172 void ShenandoahHeapRegion::make_humongous_cont_bypass() {
173 _heap->assert_heaplock_owned_by_current_thread();
174 assert (_heap->is_full_gc_in_progress(), "only for full GC");
175
176 switch (_state) {
177 case _empty_committed:
178 case _regular:
179 case _humongous_start:
180 case _humongous_cont:
181 _state = _humongous_cont;
182 return;
183 default:
184 report_illegal_transition("humongous continuation bypass");
185 }
186 }
187
188 void ShenandoahHeapRegion::make_pinned() {
189 _heap->assert_heaplock_owned_by_current_thread();
190 switch (_state) {
191 case _regular:
192 assert (_critical_pins == 0, "sanity");
193 _state = _pinned;
194 case _pinned_cset:
195 case _pinned:
196 _critical_pins++;
197 return;
198 case _humongous_start:
199 assert (_critical_pins == 0, "sanity");
200 _state = _pinned_humongous_start;
201 case _pinned_humongous_start:
202 _critical_pins++;
203 return;
204 case _cset:
205 guarantee(_heap->cancelled_concgc(), "only valid when evac has been cancelled");
206 assert (_critical_pins == 0, "sanity");
207 _state = _pinned_cset;
208 _critical_pins++;
209 return;
210 default:
211 report_illegal_transition("pinning");
212 }
213 }
214
215 void ShenandoahHeapRegion::make_unpinned() {
216 _heap->assert_heaplock_owned_by_current_thread();
217 switch (_state) {
218 case _pinned:
219 assert (_critical_pins > 0, "sanity");
220 _critical_pins--;
221 if (_critical_pins == 0) {
222 _state = _regular;
223 }
224 return;
225 case _regular:
226 case _humongous_start:
227 assert (_critical_pins == 0, "sanity");
228 return;
229 case _pinned_cset:
230 guarantee(_heap->cancelled_concgc(), "only valid when evac has been cancelled");
231 assert (_critical_pins > 0, "sanity");
232 _critical_pins--;
233 if (_critical_pins == 0) {
234 _state = _cset;
235 }
236 return;
237 case _pinned_humongous_start:
238 assert (_critical_pins > 0, "sanity");
239 _critical_pins--;
240 if (_critical_pins == 0) {
241 _state = _humongous_start;
242 }
243 return;
244 default:
245 report_illegal_transition("unpinning");
246 }
247 }
248
249 void ShenandoahHeapRegion::make_cset() {
250 _heap->assert_heaplock_owned_by_current_thread();
251 switch (_state) {
252 case _regular:
253 _state = _cset;
254 case _cset:
255 return;
256 default:
257 report_illegal_transition("cset");
258 }
259 }
260
261 void ShenandoahHeapRegion::make_trash() {
262 _heap->assert_heaplock_owned_by_current_thread();
263 switch (_state) {
264 case _cset:
265 // Reclaiming cset regions
266 case _humongous_start:
267 case _humongous_cont:
268 // Reclaiming humongous regions
269 case _regular:
270 // Immediate region reclaim
271 _state = _trash;
272 return;
273 default:
274 report_illegal_transition("trashing");
275 }
276 }
277
278 void ShenandoahHeapRegion::make_empty() {
279 _heap->assert_heaplock_owned_by_current_thread();
280 switch (_state) {
281 case _trash:
282 _state = _empty_committed;
283 _empty_time = os::elapsedTime();
284 return;
285 default:
286 report_illegal_transition("emptying");
287 }
288 }
289
290 void ShenandoahHeapRegion::make_uncommitted() {
291 _heap->assert_heaplock_owned_by_current_thread();
292 switch (_state) {
293 case _empty_committed:
294 do_uncommit();
295 _state = _empty_uncommitted;
296 return;
297 default:
298 report_illegal_transition("uncommiting");
299 }
300 }
301
302 void ShenandoahHeapRegion::make_committed_bypass() {
303 _heap->assert_heaplock_owned_by_current_thread();
304 assert (_heap->is_full_gc_in_progress(), "only for full GC");
305
306 switch (_state) {
307 case _empty_uncommitted:
308 do_commit();
309 _state = _empty_committed;
310 return;
311 default:
312 report_illegal_transition("commit bypass");
313 }
314 }
315
316 bool ShenandoahHeapRegion::rollback_allocation(uint size) {
317 set_top(top() - size);
318 return true;
319 }
320
321 void ShenandoahHeapRegion::clear_live_data() {
322 OrderAccess::release_store_fence<size_t>(&_live_data, 0);
323 }
324
325 void ShenandoahHeapRegion::reset_alloc_metadata() {
326 _tlab_allocs = 0;
327 _gclab_allocs = 0;
328 _shared_allocs = 0;
329 _seqnum_first_alloc_mutator = 0;
330 _seqnum_last_alloc_mutator = 0;
331 _seqnum_first_alloc_gc = 0;
332 _seqnum_last_alloc_gc = 0;
333 }
334
335 void ShenandoahHeapRegion::reset_alloc_metadata_to_shared() {
336 if (used() > 0) {
337 _tlab_allocs = 0;
338 _gclab_allocs = 0;
339 _shared_allocs = used() >> LogHeapWordSize;
340 uint64_t next = AllocSeqNum++;
341 _seqnum_first_alloc_mutator = next;
342 _seqnum_last_alloc_mutator = next;
343 _seqnum_first_alloc_gc = 0;
344 _seqnum_last_alloc_gc = 0;
345 } else {
346 reset_alloc_metadata();
347 }
348 }
349
350 size_t ShenandoahHeapRegion::get_shared_allocs() const {
351 return _shared_allocs * HeapWordSize;
352 }
353
354 size_t ShenandoahHeapRegion::get_tlab_allocs() const {
355 return _tlab_allocs * HeapWordSize;
356 }
357
358 size_t ShenandoahHeapRegion::get_gclab_allocs() const {
359 return _gclab_allocs * HeapWordSize;
360 }
361
362 void ShenandoahHeapRegion::set_live_data(size_t s) {
363 assert(Thread::current()->is_VM_thread(), "by VM thread");
364 _live_data = (s >> LogHeapWordSize);
365 }
366
367 size_t ShenandoahHeapRegion::get_live_data_words() const {
368 return OrderAccess::load_acquire(&_live_data);
369 }
370
371 size_t ShenandoahHeapRegion::get_live_data_bytes() const {
372 return get_live_data_words() * HeapWordSize;
373 }
374
375 bool ShenandoahHeapRegion::has_live() const {
376 return get_live_data_words() != 0;
377 }
378
379 size_t ShenandoahHeapRegion::garbage() const {
380 assert(used() >= get_live_data_bytes(), "Live Data must be a subset of used() live: "SIZE_FORMAT" used: "SIZE_FORMAT,
381 get_live_data_bytes(), used());
382
383 size_t result = used() - get_live_data_bytes();
384 return result;
385 }
386
387 bool ShenandoahHeapRegion::in_collection_set() const {
388 return _heap->region_in_collection_set(_region_number);
389 }
390
391 void ShenandoahHeapRegion::print_on(outputStream* st) const {
392 st->print("|");
393 st->print(SIZE_FORMAT_W(5), this->_region_number);
394
395 switch (_state) {
396 case _empty_uncommitted:
397 st->print("|EU ");
398 break;
399 case _empty_committed:
400 st->print("|EC ");
401 break;
402 case _regular:
403 st->print("|R ");
404 break;
405 case _humongous_start:
406 st->print("|H ");
407 break;
408 case _pinned_humongous_start:
409 st->print("|HP ");
410 break;
411 case _humongous_cont:
412 st->print("|HC ");
413 break;
414 case _cset:
415 st->print("|CS ");
416 break;
417 case _trash:
418 st->print("|T ");
419 break;
420 case _pinned:
421 st->print("|P ");
422 break;
423 case _pinned_cset:
424 st->print("|CSP");
425 break;
426 default:
427 ShouldNotReachHere();
428 }
429 st->print("|BTE " INTPTR_FORMAT_W(12) ", " INTPTR_FORMAT_W(12) ", " INTPTR_FORMAT_W(12),
430 p2i(bottom()), p2i(top()), p2i(end()));
431 st->print("|TAMS " INTPTR_FORMAT_W(12) ", " INTPTR_FORMAT_W(12),
432 p2i(_heap->complete_top_at_mark_start(_bottom)),
433 p2i(_heap->next_top_at_mark_start(_bottom)));
434 st->print("|U %3d%%", (int) ((double) used() * 100 / capacity()));
435 st->print("|T %3d%%", (int) ((double) get_tlab_allocs() * 100 / capacity()));
436 st->print("|G %3d%%", (int) ((double) get_gclab_allocs() * 100 / capacity()));
437 st->print("|S %3d%%", (int) ((double) get_shared_allocs() * 100 / capacity()));
438 st->print("|L %3d%%", (int) ((double) get_live_data_bytes() * 100 / capacity()));
439 if (_heap->traversal_gc() != NULL && _heap->traversal_gc()->root_regions()->is_in(region_number())) {
440 st->print("|R");
441 } else {
442 st->print("| ");
443 }
444 st->print("|CP " SIZE_FORMAT_W(3), _critical_pins);
445 st->print("|SN " UINT64_FORMAT_HEX_W(12) ", " UINT64_FORMAT_HEX_W(8) ", " UINT64_FORMAT_HEX_W(8) ", " UINT64_FORMAT_HEX_W(8),
446 seqnum_first_alloc_mutator(), seqnum_last_alloc_mutator(),
447 seqnum_first_alloc_gc(), seqnum_last_alloc_gc());
448 st->cr();
449 }
450
451 void ShenandoahHeapRegion::oop_iterate(ExtendedOopClosure* blk) {
452 if (!is_active()) return;
453 if (is_humongous()) {
454 oop_iterate_humongous(blk);
455 } else {
456 oop_iterate_objects(blk);
457 }
458 }
459
460 void ShenandoahHeapRegion::oop_iterate_objects(ExtendedOopClosure* blk) {
461 assert(! is_humongous(), "no humongous region here");
462 HeapWord* obj_addr = bottom() + BrooksPointer::word_size();
463 HeapWord* t = top();
464 // Could call objects iterate, but this is easier.
465 while (obj_addr < t) {
466 oop obj = oop(obj_addr);
467 obj_addr += obj->oop_iterate_size(blk) + BrooksPointer::word_size();
468 }
469 }
470
471 void ShenandoahHeapRegion::oop_iterate_humongous(ExtendedOopClosure* blk) {
472 assert(is_humongous(), "only humongous region here");
473 // Find head.
474 ShenandoahHeapRegion* r = humongous_start_region();
475 assert(r->is_humongous_start(), "need humongous head here");
476 oop obj = oop(r->bottom() + BrooksPointer::word_size());
477 obj->oop_iterate(blk, MemRegion(bottom(), top()));
478 }
479
480 void ShenandoahHeapRegion::fill_region() {
481 if (free() > (BrooksPointer::word_size() + CollectedHeap::min_fill_size())) {
482 HeapWord* filler = allocate(BrooksPointer::word_size(), ShenandoahHeap::_alloc_shared);
483 HeapWord* obj = allocate(end() - top(), ShenandoahHeap::_alloc_shared);
484 _heap->fill_with_object(obj, end() - obj);
485 BrooksPointer::initialize(oop(obj));
486 }
487 }
488
489 ShenandoahHeapRegion* ShenandoahHeapRegion::humongous_start_region() const {
490 assert(is_humongous(), "Must be a part of the humongous region");
491 size_t reg_num = region_number();
492 ShenandoahHeapRegion* r = const_cast<ShenandoahHeapRegion*>(this);
493 while (!r->is_humongous_start()) {
494 assert(reg_num > 0, "Sanity");
495 reg_num --;
496 r = _heap->get_region(reg_num);
497 assert(r->is_humongous(), "Must be a part of the humongous region");
498 }
499 assert(r->is_humongous_start(), "Must be");
500 return r;
501 }
502
503 void ShenandoahHeapRegion::recycle() {
504 ContiguousSpace::clear(false);
505 if (ZapUnusedHeapArea) {
506 ContiguousSpace::mangle_unused_area_complete();
507 }
508 clear_live_data();
509
510 reset_alloc_metadata();
511
512 // Reset C-TAMS pointer to ensure size-based iteration, everything
513 // in that regions is going to be new objects.
514 if (ShenandoahRecycleClearsBitmap && !_heap->is_full_gc_in_progress()) {
515 HeapWord* r_bottom = bottom();
516 HeapWord* top = _heap->complete_top_at_mark_start(r_bottom);
517 if (top > r_bottom) {
518 _heap->complete_mark_bit_map()->clear_range_large(MemRegion(r_bottom, top));
519 }
520
521 assert(_heap->is_next_bitmap_clear_range(bottom(), end()), "must be clear");
522 _heap->set_next_top_at_mark_start(bottom(), bottom());
523 }
524
525 // We can only safely reset the C-TAMS pointer if the bitmap is clear for that region.
526 assert(_heap->is_complete_bitmap_clear_range(bottom(), end()), "must be clear");
527
528 _heap->set_complete_top_at_mark_start(bottom(), bottom());
529
530 if (UseShenandoahMatrix) {
531 _heap->connection_matrix()->clear_region(region_number());
532 }
533
534 make_empty();
535 }
536
537 HeapWord* ShenandoahHeapRegion::block_start_const(const void* p) const {
538 assert(MemRegion(bottom(), end()).contains(p),
539 "p ("PTR_FORMAT") not in space ["PTR_FORMAT", "PTR_FORMAT")",
540 p2i(p), p2i(bottom()), p2i(end()));
541 if (p >= top()) {
542 return top();
543 } else {
544 HeapWord* last = bottom() + BrooksPointer::word_size();
545 HeapWord* cur = last;
546 while (cur <= p) {
547 last = cur;
548 cur += oop(cur)->size() + BrooksPointer::word_size();
549 }
550 assert(oopDesc::is_oop(oop(last)),
551 PTR_FORMAT" should be an object start", p2i(last));
552 return last;
553 }
554 }
555
556 void ShenandoahHeapRegion::setup_heap_region_size(size_t initial_heap_size, size_t max_heap_size) {
557 // Absolute minimums we should not ever break.
558 static const size_t MIN_REGION_SIZE = 256*K;
559 static const size_t MIN_NUM_REGIONS = 10;
560
561 if (FLAG_IS_DEFAULT(ShenandoahMinRegionSize)) {
562 FLAG_SET_DEFAULT(ShenandoahMinRegionSize, MIN_REGION_SIZE);
563 }
564
565 uintx region_size;
566 if (FLAG_IS_DEFAULT(ShenandoahHeapRegionSize)) {
567 if (ShenandoahMinRegionSize > initial_heap_size / MIN_NUM_REGIONS) {
568 err_msg message("Initial heap size (" SIZE_FORMAT "K) is too low to afford the minimum number "
569 "of regions (" SIZE_FORMAT ") of minimum region size (" SIZE_FORMAT "K).",
570 initial_heap_size/K, MIN_NUM_REGIONS, ShenandoahMinRegionSize/K);
571 vm_exit_during_initialization("Invalid -XX:ShenandoahMinRegionSize option", message);
572 }
573 if (ShenandoahMinRegionSize < MIN_REGION_SIZE) {
574 err_msg message("" SIZE_FORMAT "K should not be lower than minimum region size (" SIZE_FORMAT "K).",
575 ShenandoahMinRegionSize/K, MIN_REGION_SIZE/K);
576 vm_exit_during_initialization("Invalid -XX:ShenandoahMinRegionSize option", message);
577 }
578 if (ShenandoahMinRegionSize < MinTLABSize) {
579 err_msg message("" SIZE_FORMAT "K should not be lower than TLAB size size (" SIZE_FORMAT "K).",
580 ShenandoahMinRegionSize/K, MinTLABSize/K);
581 vm_exit_during_initialization("Invalid -XX:ShenandoahMinRegionSize option", message);
582 }
583 if (ShenandoahMaxRegionSize < MIN_REGION_SIZE) {
584 err_msg message("" SIZE_FORMAT "K should not be lower than min region size (" SIZE_FORMAT "K).",
585 ShenandoahMaxRegionSize/K, MIN_REGION_SIZE/K);
586 vm_exit_during_initialization("Invalid -XX:ShenandoahMaxRegionSize option", message);
587 }
588 if (ShenandoahMinRegionSize > ShenandoahMaxRegionSize) {
589 err_msg message("Minimum (" SIZE_FORMAT "K) should be larger than maximum (" SIZE_FORMAT "K).",
590 ShenandoahMinRegionSize/K, ShenandoahMaxRegionSize/K);
591 vm_exit_during_initialization("Invalid -XX:ShenandoahMinRegionSize or -XX:ShenandoahMaxRegionSize", message);
592 }
593 size_t average_heap_size = (initial_heap_size + max_heap_size) / 2;
594 region_size = MAX2(average_heap_size / ShenandoahTargetNumRegions,
595 ShenandoahMinRegionSize);
596
597 // Now make sure that we don't go over or under our limits.
598 region_size = MAX2(ShenandoahMinRegionSize, region_size);
599 region_size = MIN2(ShenandoahMaxRegionSize, region_size);
600
601 } else {
602 if (ShenandoahHeapRegionSize > initial_heap_size / MIN_NUM_REGIONS) {
603 err_msg message("Initial heap size (" SIZE_FORMAT "K) is too low to afford the minimum number "
604 "of regions (" SIZE_FORMAT ") of requested size (" SIZE_FORMAT "K).",
605 initial_heap_size/K, MIN_NUM_REGIONS, ShenandoahHeapRegionSize/K);
606 vm_exit_during_initialization("Invalid -XX:ShenandoahHeapRegionSize option", message);
607 }
608 if (ShenandoahHeapRegionSize < ShenandoahMinRegionSize) {
609 err_msg message("Heap region size (" SIZE_FORMAT "K) should be larger than min region size (" SIZE_FORMAT "K).",
610 ShenandoahHeapRegionSize/K, ShenandoahMinRegionSize/K);
611 vm_exit_during_initialization("Invalid -XX:ShenandoahHeapRegionSize option", message);
612 }
613 if (ShenandoahHeapRegionSize > ShenandoahMaxRegionSize) {
614 err_msg message("Heap region size (" SIZE_FORMAT "K) should be lower than max region size (" SIZE_FORMAT "K).",
615 ShenandoahHeapRegionSize/K, ShenandoahMaxRegionSize/K);
616 vm_exit_during_initialization("Invalid -XX:ShenandoahHeapRegionSize option", message);
617 }
618 region_size = ShenandoahHeapRegionSize;
619 }
620
621 // Make sure region size is at least one large page, if enabled.
622 // Otherwise, mem-protecting one region may falsely protect the adjacent
623 // regions too.
624 if (UseLargePages) {
625 region_size = MAX2(region_size, os::large_page_size());
626 }
627
628 int region_size_log = log2_long((jlong) region_size);
629 // Recalculate the region size to make sure it's a power of
630 // 2. This means that region_size is the largest power of 2 that's
631 // <= what we've calculated so far.
632 region_size = ((uintx)1 << region_size_log);
633
634 // Now, set up the globals.
635 guarantee(RegionSizeBytesShift == 0, "we should only set it once");
636 RegionSizeBytesShift = (size_t)region_size_log;
637
638 guarantee(RegionSizeWordsShift == 0, "we should only set it once");
639 RegionSizeWordsShift = RegionSizeBytesShift - LogHeapWordSize;
640
641 guarantee(RegionSizeBytes == 0, "we should only set it once");
642 RegionSizeBytes = (size_t)region_size;
643 RegionSizeWords = RegionSizeBytes >> LogHeapWordSize;
644 assert (RegionSizeWords*HeapWordSize == RegionSizeBytes, "sanity");
645
646 guarantee(RegionSizeWordsMask == 0, "we should only set it once");
647 RegionSizeWordsMask = RegionSizeWords - 1;
648
649 guarantee(RegionSizeBytesMask == 0, "we should only set it once");
650 RegionSizeBytesMask = RegionSizeBytes - 1;
651
652 guarantee(HumongousThresholdWords == 0, "we should only set it once");
653 HumongousThresholdWords = RegionSizeWords * ShenandoahHumongousThreshold / 100;
654 assert (HumongousThresholdWords <= RegionSizeWords, "sanity");
655
656 guarantee(HumongousThresholdBytes == 0, "we should only set it once");
657 HumongousThresholdBytes = HumongousThresholdWords * HeapWordSize;
658 assert (HumongousThresholdBytes <= RegionSizeBytes, "sanity");
659
660 // The rationale for trimming the TLAB sizes has to do with the raciness in
661 // TLAB allocation machinery. It may happen that TLAB sizing policy polls Shenandoah
662 // about next free size, gets the answer for region #N, goes away for a while, then
663 // tries to allocate in region #N, and fail because some other thread have claimed part
664 // of the region #N, and then the freeset allocation code has to retire the region #N,
665 // before moving the allocation to region #N+1.
666 //
667 // The worst case realizes when "answer" is "region size", which means it could
668 // prematurely retire an entire region. Having smaller TLABs does not fix that
669 // completely, but reduces the probability of too wasteful region retirement.
670 // With current divisor, we will waste no more than 1/8 of region size in the worst
671 // case. This also has a secondary effect on collection set selection: even under
672 // the race, the regions would be at least 7/8 used, which allows relying on
673 // "used" - "live" for cset selection. Otherwise, we can get the fragmented region
674 // below the garbage threshold that would never be considered for collection.
675 guarantee(MaxTLABSizeBytes == 0, "we should only set it once");
676 MaxTLABSizeBytes = MIN2(RegionSizeBytes / 8, HumongousThresholdBytes);
677 assert (MaxTLABSizeBytes > MinTLABSize, "should be larger");
678
679 log_info(gc, heap)("Heap region size: " SIZE_FORMAT "M", RegionSizeBytes / M);
680 log_info(gc, init)("Region size in bytes: "SIZE_FORMAT, RegionSizeBytes);
681 log_info(gc, init)("Region size byte shift: "SIZE_FORMAT, RegionSizeBytesShift);
682 log_info(gc, init)("Humongous threshold in bytes: "SIZE_FORMAT, HumongousThresholdBytes);
683 log_info(gc, init)("Max TLAB size in bytes: "SIZE_FORMAT, MaxTLABSizeBytes);
684 log_info(gc, init)("Number of regions: "SIZE_FORMAT, max_heap_size / RegionSizeBytes);
685 }
686
687 CompactibleSpace* ShenandoahHeapRegion::next_compaction_space() const {
688 return _heap->next_compaction_region(this);
689 }
690
691 void ShenandoahHeapRegion::prepare_for_compaction(CompactPoint* cp) {
692 scan_and_forward(this, cp);
693 }
694
695 void ShenandoahHeapRegion::adjust_pointers() {
696 // Check first is there is any work to do.
697 if (used() == 0) {
698 return; // Nothing to do.
699 }
700
701 scan_and_adjust_pointers(this);
702 }
703
704 void ShenandoahHeapRegion::compact() {
705 assert(!is_humongous(), "Shouldn't be compacting humongous regions");
706 scan_and_compact(this);
707 }
708
709 void ShenandoahHeapRegion::do_commit() {
710 if (_initialized && can_idle_region()) {
711 os::activate_memory((char *)_reserved.start(), _reserved.byte_size());
712 _heap->activate_bitmap_slice(this);
713 } else {
714 if (!os::commit_memory((char *) _reserved.start(), _reserved.byte_size(), false)) {
715 report_java_out_of_memory("Unable to commit region");
716 }
717 if (!_heap->commit_bitmap_slice(this)) {
718 report_java_out_of_memory("Unable to commit bitmaps for region");
719 }
720
721 _initialized = true;
722 }
723 _heap->increase_committed(ShenandoahHeapRegion::region_size_bytes());
724 }
725
726 void ShenandoahHeapRegion::do_uncommit() {
727 if (can_idle_region()) {
728 if (!os::idle_memory((char *)_reserved.start(), _reserved.byte_size())) {
729 report_java_out_of_memory("Unable to idle the region");
730 }
731
732 if (!_heap->idle_bitmap_slice(this)) {
733 report_java_out_of_memory("Unable to idle bitmaps for region");
734 }
735 } else {
736 if (!os::uncommit_memory((char *) _reserved.start(), _reserved.byte_size())) {
737 report_java_out_of_memory("Unable to uncommit region");
738 }
739 if (!_heap->uncommit_bitmap_slice(this)) {
740 report_java_out_of_memory("Unable to uncommit bitmaps for region");
741 }
742 }
743 _heap->decrease_committed(ShenandoahHeapRegion::region_size_bytes());
744 }
745
746
747 bool ShenandoahHeapRegion::can_idle_region() const {
748 return LINUX_ONLY(ShenandoahUncommitWithIdle && !UseLargePages) NOT_LINUX(false);
749 }