1 /*
2 * Copyright (c) 2013, 2015, Red Hat, Inc. and/or its affiliates.
3 *
4 * This code is free software; you can redistribute it and/or modify it
5 * under the terms of the GNU General Public License version 2 only, as
6 * published by the Free Software Foundation.
7 *
8 * This code is distributed in the hope that it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
11 * version 2 for more details (a copy is included in the LICENSE file that
12 * accompanied this code).
13 *
14 * You should have received a copy of the GNU General Public License version
15 * 2 along with this work; if not, write to the Free Software Foundation,
16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
17 *
18 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
19 * or visit www.oracle.com if you need additional information or have any
20 * questions.
21 *
22 */
23
24 #include "precompiled.hpp"
25 #include "memory/allocation.hpp"
26 #include "gc/shenandoah/brooksPointer.hpp"
27 #include "gc/shenandoah/shenandoahHeapRegionSet.inline.hpp"
28 #include "gc/shenandoah/shenandoahConnectionMatrix.hpp"
29 #include "gc/shenandoah/shenandoahHeap.hpp"
30 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
31 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
32 #include "gc/shenandoah/shenandoahTraversalGC.hpp"
33 #include "gc/shared/space.inline.hpp"
34 #include "memory/universe.hpp"
35 #include "oops/oop.inline.hpp"
36 #include "runtime/java.hpp"
37 #include "runtime/mutexLocker.hpp"
38 #include "runtime/os.hpp"
39 #include "runtime/safepoint.hpp"
40
41 size_t ShenandoahHeapRegion::RegionSizeBytes = 0;
42 size_t ShenandoahHeapRegion::RegionSizeWords = 0;
43 size_t ShenandoahHeapRegion::RegionSizeBytesShift = 0;
44 size_t ShenandoahHeapRegion::RegionSizeWordsShift = 0;
45 size_t ShenandoahHeapRegion::RegionSizeBytesMask = 0;
46 size_t ShenandoahHeapRegion::RegionSizeWordsMask = 0;
47 size_t ShenandoahHeapRegion::HumongousThresholdBytes = 0;
48 size_t ShenandoahHeapRegion::HumongousThresholdWords = 0;
49 size_t ShenandoahHeapRegion::MaxTLABSizeBytes = 0;
50
51 // start with 1, reserve 0 for uninitialized value
52 uint64_t ShenandoahHeapRegion::AllocSeqNum = 1;
53
54 ShenandoahHeapRegion::ShenandoahHeapRegion(ShenandoahHeap* heap, HeapWord* start,
55 size_t size_words, size_t index, bool committed) :
56 _heap(heap),
57 _pacer(ShenandoahPacing ? heap->pacer() : NULL),
58 _region_number(index),
59 _live_data(0),
60 _tlab_allocs(0),
61 _gclab_allocs(0),
62 _shared_allocs(0),
63 _reserved(MemRegion(start, size_words)),
64 _new_top(NULL),
65 _seqnum_first_alloc_mutator(0),
66 _seqnum_last_alloc_mutator(0),
67 _seqnum_first_alloc_gc(0),
68 _seqnum_last_alloc_gc(0),
69 _state(committed ? _empty_committed : _empty_uncommitted),
70 _empty_time(os::elapsedTime()),
71 _initialized(false),
72 _critical_pins(0) {
73
74 ContiguousSpace::initialize(_reserved, true, committed);
75 }
76
77 size_t ShenandoahHeapRegion::region_number() const {
78 return _region_number;
79 }
80
81 void ShenandoahHeapRegion::report_illegal_transition(const char *method) {
82 ResourceMark rm;
83 stringStream ss;
84 ss.print("Illegal region state transition from \"%s\", at %s\n ", region_state_to_string(_state), method);
85 print_on(&ss);
86 fatal("%s", ss.as_string());
87 }
88
89 void ShenandoahHeapRegion::make_regular_allocation() {
90 _heap->assert_heaplock_owned_by_current_thread();
91
92 switch (_state) {
93 case _empty_uncommitted:
94 do_commit();
95 case _empty_committed:
96 _state = _regular;
97 case _regular:
98 case _pinned:
99 return;
100 default:
101 report_illegal_transition("regular allocation");
102 }
103 }
104
105 void ShenandoahHeapRegion::make_regular_bypass() {
106 _heap->assert_heaplock_owned_by_current_thread();
107 assert (_heap->is_full_gc_in_progress() || _heap->is_degenerated_gc_in_progress(),
108 "only for full or degen GC");
109
110 switch (_state) {
111 case _empty_uncommitted:
112 do_commit();
113 case _empty_committed:
114 case _cset:
115 case _humongous_start:
116 case _humongous_cont:
117 _state = _regular;
118 return;
119 case _pinned_cset:
120 _state = _pinned;
121 return;
122 case _regular:
123 case _pinned:
124 return;
125 default:
126 report_illegal_transition("regular bypass");
127 }
128 }
129
130 void ShenandoahHeapRegion::make_humongous_start() {
131 _heap->assert_heaplock_owned_by_current_thread();
132 switch (_state) {
133 case _empty_uncommitted:
134 do_commit();
135 case _empty_committed:
136 _state = _humongous_start;
137 return;
138 default:
139 report_illegal_transition("humongous start allocation");
140 }
141 }
142
143 void ShenandoahHeapRegion::make_humongous_start_bypass() {
144 _heap->assert_heaplock_owned_by_current_thread();
145 assert (_heap->is_full_gc_in_progress(), "only for full GC");
146
147 switch (_state) {
148 case _empty_committed:
149 case _regular:
150 case _humongous_start:
151 case _humongous_cont:
152 _state = _humongous_start;
153 return;
154 default:
155 report_illegal_transition("humongous start bypass");
156 }
157 }
158
159 void ShenandoahHeapRegion::make_humongous_cont() {
160 _heap->assert_heaplock_owned_by_current_thread();
161 switch (_state) {
162 case _empty_uncommitted:
163 do_commit();
164 case _empty_committed:
165 _state = _humongous_cont;
166 return;
167 default:
168 report_illegal_transition("humongous continuation allocation");
169 }
170 }
171
172 void ShenandoahHeapRegion::make_humongous_cont_bypass() {
173 _heap->assert_heaplock_owned_by_current_thread();
174 assert (_heap->is_full_gc_in_progress(), "only for full GC");
175
176 switch (_state) {
177 case _empty_committed:
178 case _regular:
179 case _humongous_start:
180 case _humongous_cont:
181 _state = _humongous_cont;
182 return;
183 default:
184 report_illegal_transition("humongous continuation bypass");
185 }
186 }
187
188 void ShenandoahHeapRegion::make_pinned() {
189 _heap->assert_heaplock_owned_by_current_thread();
190 switch (_state) {
191 case _regular:
192 assert (_critical_pins == 0, "sanity");
193 _state = _pinned;
194 case _pinned_cset:
195 case _pinned:
196 _critical_pins++;
197 return;
198 case _humongous_start:
199 assert (_critical_pins == 0, "sanity");
200 _state = _pinned_humongous_start;
201 case _pinned_humongous_start:
202 _critical_pins++;
203 return;
204 case _cset:
205 guarantee(_heap->cancelled_concgc(), "only valid when evac has been cancelled");
206 assert (_critical_pins == 0, "sanity");
207 _state = _pinned_cset;
208 _critical_pins++;
209 return;
210 default:
211 report_illegal_transition("pinning");
212 }
213 }
214
215 void ShenandoahHeapRegion::make_unpinned() {
216 _heap->assert_heaplock_owned_by_current_thread();
217 switch (_state) {
218 case _pinned:
219 assert (_critical_pins > 0, "sanity");
220 _critical_pins--;
221 if (_critical_pins == 0) {
222 _state = _regular;
223 }
224 return;
225 case _regular:
226 case _humongous_start:
227 assert (_critical_pins == 0, "sanity");
228 return;
229 case _pinned_cset:
230 guarantee(_heap->cancelled_concgc(), "only valid when evac has been cancelled");
231 assert (_critical_pins > 0, "sanity");
232 _critical_pins--;
233 if (_critical_pins == 0) {
234 _state = _cset;
235 }
236 return;
237 case _pinned_humongous_start:
238 assert (_critical_pins > 0, "sanity");
239 _critical_pins--;
240 if (_critical_pins == 0) {
241 _state = _humongous_start;
242 }
243 return;
244 default:
245 report_illegal_transition("unpinning");
246 }
247 }
248
249 void ShenandoahHeapRegion::make_cset() {
250 _heap->assert_heaplock_owned_by_current_thread();
251 switch (_state) {
252 case _regular:
253 _state = _cset;
254 case _cset:
255 return;
256 default:
257 report_illegal_transition("cset");
258 }
259 }
260
261 void ShenandoahHeapRegion::make_trash() {
262 _heap->assert_heaplock_owned_by_current_thread();
263 switch (_state) {
264 case _cset:
265 // Reclaiming cset regions
266 case _humongous_start:
267 case _humongous_cont:
268 // Reclaiming humongous regions
269 case _regular:
270 // Immediate region reclaim
271 _state = _trash;
272 return;
273 default:
274 report_illegal_transition("trashing");
275 }
276 }
277
278 void ShenandoahHeapRegion::make_empty() {
279 _heap->assert_heaplock_owned_by_current_thread();
280 switch (_state) {
281 case _trash:
282 _state = _empty_committed;
283 _empty_time = os::elapsedTime();
284 return;
285 default:
286 report_illegal_transition("emptying");
287 }
288 }
289
290 void ShenandoahHeapRegion::make_uncommitted() {
291 _heap->assert_heaplock_owned_by_current_thread();
292 switch (_state) {
293 case _empty_committed:
294 do_uncommit();
295 _state = _empty_uncommitted;
296 return;
297 default:
298 report_illegal_transition("uncommiting");
299 }
300 }
301
302 void ShenandoahHeapRegion::make_committed_bypass() {
303 _heap->assert_heaplock_owned_by_current_thread();
304 assert (_heap->is_full_gc_in_progress(), "only for full GC");
305
306 switch (_state) {
307 case _empty_uncommitted:
308 do_commit();
309 _state = _empty_committed;
310 return;
311 default:
312 report_illegal_transition("commit bypass");
313 }
314 }
315
316 bool ShenandoahHeapRegion::rollback_allocation(uint size) {
317 set_top(top() - size);
318 return true;
319 }
320
321 void ShenandoahHeapRegion::clear_live_data() {
322 OrderAccess::release_store_fence<size_t>(&_live_data, 0);
323 }
324
325 void ShenandoahHeapRegion::reset_alloc_metadata() {
326 _tlab_allocs = 0;
327 _gclab_allocs = 0;
328 _shared_allocs = 0;
329 _seqnum_first_alloc_mutator = 0;
330 _seqnum_last_alloc_mutator = 0;
331 _seqnum_first_alloc_gc = 0;
332 _seqnum_last_alloc_gc = 0;
333 }
334
335 void ShenandoahHeapRegion::reset_alloc_metadata_to_shared() {
336 if (used() > 0) {
337 _tlab_allocs = 0;
338 _gclab_allocs = 0;
339 _shared_allocs = used() >> LogHeapWordSize;
340 uint64_t next = AllocSeqNum++;
341 _seqnum_first_alloc_mutator = next;
342 _seqnum_last_alloc_mutator = next;
343 _seqnum_first_alloc_gc = 0;
344 _seqnum_last_alloc_gc = 0;
345 } else {
346 reset_alloc_metadata();
347 }
348 }
349
350 size_t ShenandoahHeapRegion::get_shared_allocs() const {
351 return _shared_allocs * HeapWordSize;
352 }
353
354 size_t ShenandoahHeapRegion::get_tlab_allocs() const {
355 return _tlab_allocs * HeapWordSize;
356 }
357
358 size_t ShenandoahHeapRegion::get_gclab_allocs() const {
359 return _gclab_allocs * HeapWordSize;
360 }
361
362 void ShenandoahHeapRegion::set_live_data(size_t s) {
363 assert(Thread::current()->is_VM_thread(), "by VM thread");
364 _live_data = (s >> LogHeapWordSize);
365 }
366
367 size_t ShenandoahHeapRegion::get_live_data_words() const {
368 return OrderAccess::load_acquire(&_live_data);
369 }
370
371 size_t ShenandoahHeapRegion::get_live_data_bytes() const {
372 return get_live_data_words() * HeapWordSize;
373 }
374
375 bool ShenandoahHeapRegion::has_live() const {
376 return get_live_data_words() != 0;
377 }
378
379 size_t ShenandoahHeapRegion::garbage() const {
380 assert(used() >= get_live_data_bytes(), "Live Data must be a subset of used() live: "SIZE_FORMAT" used: "SIZE_FORMAT,
381 get_live_data_bytes(), used());
382
383 size_t result = used() - get_live_data_bytes();
384 return result;
385 }
386
387 bool ShenandoahHeapRegion::in_collection_set() const {
388 return _heap->region_in_collection_set(_region_number);
389 }
390
391 void ShenandoahHeapRegion::print_on(outputStream* st) const {
392 st->print("|");
393 st->print(SIZE_FORMAT_W(5), this->_region_number);
394
395 switch (_state) {
396 case _empty_uncommitted:
397 st->print("|EU ");
398 break;
399 case _empty_committed:
400 st->print("|EC ");
401 break;
402 case _regular:
403 st->print("|R ");
404 break;
405 case _humongous_start:
406 st->print("|H ");
407 break;
408 case _pinned_humongous_start:
409 st->print("|HP ");
410 break;
411 case _humongous_cont:
412 st->print("|HC ");
413 break;
414 case _cset:
415 st->print("|CS ");
416 break;
417 case _trash:
418 st->print("|T ");
419 break;
420 case _pinned:
421 st->print("|P ");
422 break;
423 case _pinned_cset:
424 st->print("|CSP");
425 break;
426 default:
427 ShouldNotReachHere();
428 }
429 st->print("|BTE " INTPTR_FORMAT_W(12) ", " INTPTR_FORMAT_W(12) ", " INTPTR_FORMAT_W(12),
430 p2i(bottom()), p2i(top()), p2i(end()));
431 st->print("|TAMS " INTPTR_FORMAT_W(12) ", " INTPTR_FORMAT_W(12),
432 p2i(_heap->complete_top_at_mark_start(_bottom)),
433 p2i(_heap->next_top_at_mark_start(_bottom)));
434 st->print("|U %3d%%", (int) ((double) used() * 100 / capacity()));
435 st->print("|T %3d%%", (int) ((double) get_tlab_allocs() * 100 / capacity()));
436 st->print("|G %3d%%", (int) ((double) get_gclab_allocs() * 100 / capacity()));
437 st->print("|S %3d%%", (int) ((double) get_shared_allocs() * 100 / capacity()));
438 st->print("|L %3d%%", (int) ((double) get_live_data_bytes() * 100 / capacity()));
439 if (_heap->traversal_gc() != NULL && _heap->traversal_gc()->root_regions()->is_in(region_number())) {
440 st->print("|R");
441 } else {
442 st->print("| ");
443 }
444 st->print("|CP " SIZE_FORMAT_W(3), _critical_pins);
445 st->print("|SN " UINT64_FORMAT_HEX_W(12) ", " UINT64_FORMAT_HEX_W(8) ", " UINT64_FORMAT_HEX_W(8) ", " UINT64_FORMAT_HEX_W(8),
446 seqnum_first_alloc_mutator(), seqnum_last_alloc_mutator(),
447 seqnum_first_alloc_gc(), seqnum_last_alloc_gc());
448 st->cr();
449 }
450
451 void ShenandoahHeapRegion::oop_iterate(ExtendedOopClosure* blk) {
452 if (!is_active()) return;
453 if (is_humongous()) {
454 oop_iterate_humongous(blk);
455 } else {
456 oop_iterate_objects(blk);
457 }
458 }
459
460 void ShenandoahHeapRegion::oop_iterate_objects(ExtendedOopClosure* blk) {
461 assert(! is_humongous(), "no humongous region here");
462 HeapWord* obj_addr = bottom() + BrooksPointer::word_size();
463 HeapWord* t = top();
464 // Could call objects iterate, but this is easier.
465 while (obj_addr < t) {
466 oop obj = oop(obj_addr);
467 obj_addr += obj->oop_iterate_size(blk) + BrooksPointer::word_size();
468 }
469 }
470
471 void ShenandoahHeapRegion::oop_iterate_humongous(ExtendedOopClosure* blk) {
472 assert(is_humongous(), "only humongous region here");
473 // Find head.
474 ShenandoahHeapRegion* r = humongous_start_region();
475 assert(r->is_humongous_start(), "need humongous head here");
476 oop obj = oop(r->bottom() + BrooksPointer::word_size());
477 obj->oop_iterate(blk, MemRegion(bottom(), top()));
478 }
479
480 void ShenandoahHeapRegion::fill_region() {
481 if (free() > (BrooksPointer::word_size() + CollectedHeap::min_fill_size())) {
482 HeapWord* obj = allocate(end() - top(), ShenandoahHeap::_alloc_shared);
483 _heap->fill_with_object(obj, end() - obj);
484 }
485 }
486
487 ShenandoahHeapRegion* ShenandoahHeapRegion::humongous_start_region() const {
488 assert(is_humongous(), "Must be a part of the humongous region");
489 size_t reg_num = region_number();
490 ShenandoahHeapRegion* r = const_cast<ShenandoahHeapRegion*>(this);
491 while (!r->is_humongous_start()) {
492 assert(reg_num > 0, "Sanity");
493 reg_num --;
494 r = _heap->get_region(reg_num);
495 assert(r->is_humongous(), "Must be a part of the humongous region");
496 }
497 assert(r->is_humongous_start(), "Must be");
498 return r;
499 }
500
501 void ShenandoahHeapRegion::recycle() {
502 ContiguousSpace::clear(false);
503 if (ZapUnusedHeapArea) {
504 ContiguousSpace::mangle_unused_area_complete();
505 }
506 clear_live_data();
507
508 reset_alloc_metadata();
509
510 // Reset C-TAMS pointer to ensure size-based iteration, everything
511 // in that regions is going to be new objects.
512 if (ShenandoahRecycleClearsBitmap && !_heap->is_full_gc_in_progress()) {
513 HeapWord* r_bottom = bottom();
514 HeapWord* top = _heap->complete_top_at_mark_start(r_bottom);
515 if (top > r_bottom) {
516 _heap->complete_mark_bit_map()->clear_range_large(MemRegion(r_bottom, top));
517 }
518
519 assert(_heap->is_next_bitmap_clear_range(bottom(), end()), "must be clear");
520 _heap->set_next_top_at_mark_start(bottom(), bottom());
521 }
522
523 // We can only safely reset the C-TAMS pointer if the bitmap is clear for that region.
524 assert(_heap->is_complete_bitmap_clear_range(bottom(), end()), "must be clear");
525
526 _heap->set_complete_top_at_mark_start(bottom(), bottom());
527
528 if (UseShenandoahMatrix) {
529 _heap->connection_matrix()->clear_region(region_number());
530 }
531
532 make_empty();
533 }
534
535 HeapWord* ShenandoahHeapRegion::block_start_const(const void* p) const {
536 assert(MemRegion(bottom(), end()).contains(p),
537 "p ("PTR_FORMAT") not in space ["PTR_FORMAT", "PTR_FORMAT")",
538 p2i(p), p2i(bottom()), p2i(end()));
539 if (p >= top()) {
540 return top();
541 } else {
542 HeapWord* last = bottom() + BrooksPointer::word_size();
543 HeapWord* cur = last;
544 while (cur <= p) {
545 last = cur;
546 cur += oop(cur)->size() + BrooksPointer::word_size();
547 }
548 assert(oopDesc::is_oop(oop(last)),
549 PTR_FORMAT" should be an object start", p2i(last));
550 return last;
551 }
552 }
553
554 void ShenandoahHeapRegion::setup_heap_region_size(size_t initial_heap_size, size_t max_heap_size) {
555 // Absolute minimums we should not ever break.
556 static const size_t MIN_REGION_SIZE = 256*K;
557 static const size_t MIN_NUM_REGIONS = 10;
558
559 if (FLAG_IS_DEFAULT(ShenandoahMinRegionSize)) {
560 FLAG_SET_DEFAULT(ShenandoahMinRegionSize, MIN_REGION_SIZE);
561 }
562
563 uintx region_size;
564 if (FLAG_IS_DEFAULT(ShenandoahHeapRegionSize)) {
565 if (ShenandoahMinRegionSize > initial_heap_size / MIN_NUM_REGIONS) {
566 err_msg message("Initial heap size (" SIZE_FORMAT "K) is too low to afford the minimum number "
567 "of regions (" SIZE_FORMAT ") of minimum region size (" SIZE_FORMAT "K).",
568 initial_heap_size/K, MIN_NUM_REGIONS, ShenandoahMinRegionSize/K);
569 vm_exit_during_initialization("Invalid -XX:ShenandoahMinRegionSize option", message);
570 }
571 if (ShenandoahMinRegionSize < MIN_REGION_SIZE) {
572 err_msg message("" SIZE_FORMAT "K should not be lower than minimum region size (" SIZE_FORMAT "K).",
573 ShenandoahMinRegionSize/K, MIN_REGION_SIZE/K);
574 vm_exit_during_initialization("Invalid -XX:ShenandoahMinRegionSize option", message);
575 }
576 if (ShenandoahMinRegionSize < MinTLABSize) {
577 err_msg message("" SIZE_FORMAT "K should not be lower than TLAB size size (" SIZE_FORMAT "K).",
578 ShenandoahMinRegionSize/K, MinTLABSize/K);
579 vm_exit_during_initialization("Invalid -XX:ShenandoahMinRegionSize option", message);
580 }
581 if (ShenandoahMaxRegionSize < MIN_REGION_SIZE) {
582 err_msg message("" SIZE_FORMAT "K should not be lower than min region size (" SIZE_FORMAT "K).",
583 ShenandoahMaxRegionSize/K, MIN_REGION_SIZE/K);
584 vm_exit_during_initialization("Invalid -XX:ShenandoahMaxRegionSize option", message);
585 }
586 if (ShenandoahMinRegionSize > ShenandoahMaxRegionSize) {
587 err_msg message("Minimum (" SIZE_FORMAT "K) should be larger than maximum (" SIZE_FORMAT "K).",
588 ShenandoahMinRegionSize/K, ShenandoahMaxRegionSize/K);
589 vm_exit_during_initialization("Invalid -XX:ShenandoahMinRegionSize or -XX:ShenandoahMaxRegionSize", message);
590 }
591 size_t average_heap_size = (initial_heap_size + max_heap_size) / 2;
592 region_size = MAX2(average_heap_size / ShenandoahTargetNumRegions,
593 ShenandoahMinRegionSize);
594
595 // Now make sure that we don't go over or under our limits.
596 region_size = MAX2(ShenandoahMinRegionSize, region_size);
597 region_size = MIN2(ShenandoahMaxRegionSize, region_size);
598
599 } else {
600 if (ShenandoahHeapRegionSize > initial_heap_size / MIN_NUM_REGIONS) {
601 err_msg message("Initial heap size (" SIZE_FORMAT "K) is too low to afford the minimum number "
602 "of regions (" SIZE_FORMAT ") of requested size (" SIZE_FORMAT "K).",
603 initial_heap_size/K, MIN_NUM_REGIONS, ShenandoahHeapRegionSize/K);
604 vm_exit_during_initialization("Invalid -XX:ShenandoahHeapRegionSize option", message);
605 }
606 if (ShenandoahHeapRegionSize < ShenandoahMinRegionSize) {
607 err_msg message("Heap region size (" SIZE_FORMAT "K) should be larger than min region size (" SIZE_FORMAT "K).",
608 ShenandoahHeapRegionSize/K, ShenandoahMinRegionSize/K);
609 vm_exit_during_initialization("Invalid -XX:ShenandoahHeapRegionSize option", message);
610 }
611 if (ShenandoahHeapRegionSize > ShenandoahMaxRegionSize) {
612 err_msg message("Heap region size (" SIZE_FORMAT "K) should be lower than max region size (" SIZE_FORMAT "K).",
613 ShenandoahHeapRegionSize/K, ShenandoahMaxRegionSize/K);
614 vm_exit_during_initialization("Invalid -XX:ShenandoahHeapRegionSize option", message);
615 }
616 region_size = ShenandoahHeapRegionSize;
617 }
618
619 // Make sure region size is at least one large page, if enabled.
620 // Otherwise, mem-protecting one region may falsely protect the adjacent
621 // regions too.
622 if (UseLargePages) {
623 region_size = MAX2(region_size, os::large_page_size());
624 }
625
626 int region_size_log = log2_long((jlong) region_size);
627 // Recalculate the region size to make sure it's a power of
628 // 2. This means that region_size is the largest power of 2 that's
629 // <= what we've calculated so far.
630 region_size = ((uintx)1 << region_size_log);
631
632 // Now, set up the globals.
633 guarantee(RegionSizeBytesShift == 0, "we should only set it once");
634 RegionSizeBytesShift = (size_t)region_size_log;
635
636 guarantee(RegionSizeWordsShift == 0, "we should only set it once");
637 RegionSizeWordsShift = RegionSizeBytesShift - LogHeapWordSize;
638
639 guarantee(RegionSizeBytes == 0, "we should only set it once");
640 RegionSizeBytes = (size_t)region_size;
641 RegionSizeWords = RegionSizeBytes >> LogHeapWordSize;
642 assert (RegionSizeWords*HeapWordSize == RegionSizeBytes, "sanity");
643
644 guarantee(RegionSizeWordsMask == 0, "we should only set it once");
645 RegionSizeWordsMask = RegionSizeWords - 1;
646
647 guarantee(RegionSizeBytesMask == 0, "we should only set it once");
648 RegionSizeBytesMask = RegionSizeBytes - 1;
649
650 guarantee(HumongousThresholdWords == 0, "we should only set it once");
651 HumongousThresholdWords = RegionSizeWords * ShenandoahHumongousThreshold / 100;
652 assert (HumongousThresholdWords <= RegionSizeWords, "sanity");
653
654 guarantee(HumongousThresholdBytes == 0, "we should only set it once");
655 HumongousThresholdBytes = HumongousThresholdWords * HeapWordSize;
656 assert (HumongousThresholdBytes <= RegionSizeBytes, "sanity");
657
658 // The rationale for trimming the TLAB sizes has to do with the raciness in
659 // TLAB allocation machinery. It may happen that TLAB sizing policy polls Shenandoah
660 // about next free size, gets the answer for region #N, goes away for a while, then
661 // tries to allocate in region #N, and fail because some other thread have claimed part
662 // of the region #N, and then the freeset allocation code has to retire the region #N,
663 // before moving the allocation to region #N+1.
664 //
665 // The worst case realizes when "answer" is "region size", which means it could
666 // prematurely retire an entire region. Having smaller TLABs does not fix that
667 // completely, but reduces the probability of too wasteful region retirement.
668 // With current divisor, we will waste no more than 1/8 of region size in the worst
669 // case. This also has a secondary effect on collection set selection: even under
670 // the race, the regions would be at least 7/8 used, which allows relying on
671 // "used" - "live" for cset selection. Otherwise, we can get the fragmented region
672 // below the garbage threshold that would never be considered for collection.
673 guarantee(MaxTLABSizeBytes == 0, "we should only set it once");
674 MaxTLABSizeBytes = MIN2(RegionSizeBytes / 8, HumongousThresholdBytes);
675 assert (MaxTLABSizeBytes > MinTLABSize, "should be larger");
676
677 log_info(gc, heap)("Heap region size: " SIZE_FORMAT "M", RegionSizeBytes / M);
678 log_info(gc, init)("Region size in bytes: "SIZE_FORMAT, RegionSizeBytes);
679 log_info(gc, init)("Region size byte shift: "SIZE_FORMAT, RegionSizeBytesShift);
680 log_info(gc, init)("Humongous threshold in bytes: "SIZE_FORMAT, HumongousThresholdBytes);
681 log_info(gc, init)("Max TLAB size in bytes: "SIZE_FORMAT, MaxTLABSizeBytes);
682 log_info(gc, init)("Number of regions: "SIZE_FORMAT, max_heap_size / RegionSizeBytes);
683 }
684
685 CompactibleSpace* ShenandoahHeapRegion::next_compaction_space() const {
686 return _heap->next_compaction_region(this);
687 }
688
689 void ShenandoahHeapRegion::prepare_for_compaction(CompactPoint* cp) {
690 scan_and_forward(this, cp);
691 }
692
693 void ShenandoahHeapRegion::adjust_pointers() {
694 // Check first is there is any work to do.
695 if (used() == 0) {
696 return; // Nothing to do.
697 }
698
699 scan_and_adjust_pointers(this);
700 }
701
702 void ShenandoahHeapRegion::compact() {
703 assert(!is_humongous(), "Shouldn't be compacting humongous regions");
704 scan_and_compact(this);
705 }
706
707 void ShenandoahHeapRegion::do_commit() {
708 if (_initialized && can_idle_region()) {
709 os::activate_memory((char *)_reserved.start(), _reserved.byte_size());
710 _heap->activate_bitmap_slice(this);
711 } else {
712 if (!os::commit_memory((char *) _reserved.start(), _reserved.byte_size(), false)) {
713 report_java_out_of_memory("Unable to commit region");
714 }
715 if (!_heap->commit_bitmap_slice(this)) {
716 report_java_out_of_memory("Unable to commit bitmaps for region");
717 }
718
719 _initialized = true;
720 }
721 _heap->increase_committed(ShenandoahHeapRegion::region_size_bytes());
722 }
723
724 void ShenandoahHeapRegion::do_uncommit() {
725 if (can_idle_region()) {
726 if (!os::idle_memory((char *)_reserved.start(), _reserved.byte_size())) {
727 report_java_out_of_memory("Unable to idle the region");
728 }
729
730 if (!_heap->idle_bitmap_slice(this)) {
731 report_java_out_of_memory("Unable to idle bitmaps for region");
732 }
733 } else {
734 if (!os::uncommit_memory((char *) _reserved.start(), _reserved.byte_size())) {
735 report_java_out_of_memory("Unable to uncommit region");
736 }
737 if (!_heap->uncommit_bitmap_slice(this)) {
738 report_java_out_of_memory("Unable to uncommit bitmaps for region");
739 }
740 }
741 _heap->decrease_committed(ShenandoahHeapRegion::region_size_bytes());
742 }
743
744
745 bool ShenandoahHeapRegion::can_idle_region() const {
746 return LINUX_ONLY(ShenandoahUncommitWithIdle && !UseLargePages) NOT_LINUX(false);
747 }