1 /*
2 * Copyright (c) 2006, 2017, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "gc/parallel/mutableNUMASpace.hpp"
27 #include "gc/shared/collectedHeap.hpp"
28 #include "gc/shared/spaceDecorator.hpp"
29 #include "oops/oop.inline.hpp"
30 #include "runtime/atomic.hpp"
31 #include "runtime/thread.inline.hpp"
32 #include "runtime/threadSMR.hpp"
33 #include "utilities/align.hpp"
34
35 MutableNUMASpace::MutableNUMASpace(size_t alignment) : MutableSpace(alignment), _must_use_large_pages(false) {
36 _lgrp_spaces = new (ResourceObj::C_HEAP, mtGC) GrowableArray<LGRPSpace*>(0, true);
37 _page_size = os::vm_page_size();
38 _adaptation_cycles = 0;
39 _samples_count = 0;
40
41 #ifdef LINUX
42 // Changing the page size can lead to freeing of memory. When using large pages
43 // and the memory has been both reserved and committed, Linux does not support
44 // freeing parts of it.
45 if (UseLargePages && !os::can_commit_large_page_memory()) {
46 _must_use_large_pages = true;
47 }
48 #endif // LINUX
49
50 update_layout(true);
51 }
52
53 MutableNUMASpace::~MutableNUMASpace() {
54 for (int i = 0; i < lgrp_spaces()->length(); i++) {
55 delete lgrp_spaces()->at(i);
56 }
57 delete lgrp_spaces();
58 }
59
60 #ifndef PRODUCT
61 void MutableNUMASpace::mangle_unused_area() {
62 // This method should do nothing.
63 // It can be called on a numa space during a full compaction.
64 }
65 void MutableNUMASpace::mangle_unused_area_complete() {
66 // This method should do nothing.
67 // It can be called on a numa space during a full compaction.
68 }
69 void MutableNUMASpace::mangle_region(MemRegion mr) {
70 // This method should do nothing because numa spaces are not mangled.
71 }
72 void MutableNUMASpace::set_top_for_allocations(HeapWord* v) {
73 assert(false, "Do not mangle MutableNUMASpace's");
74 }
75 void MutableNUMASpace::set_top_for_allocations() {
76 // This method should do nothing.
77 }
78 void MutableNUMASpace::check_mangled_unused_area(HeapWord* limit) {
79 // This method should do nothing.
80 }
81 void MutableNUMASpace::check_mangled_unused_area_complete() {
82 // This method should do nothing.
83 }
84 #endif // NOT_PRODUCT
85
86 // There may be unallocated holes in the middle chunks
87 // that should be filled with dead objects to ensure parsability.
88 void MutableNUMASpace::ensure_parsability() {
89 for (int i = 0; i < lgrp_spaces()->length(); i++) {
90 LGRPSpace *ls = lgrp_spaces()->at(i);
91 MutableSpace *s = ls->space();
92 if (s->top() < top()) { // For all spaces preceding the one containing top()
93 if (s->free_in_words() > 0) {
94 intptr_t cur_top = (intptr_t)s->top();
95 size_t words_left_to_fill = pointer_delta(s->end(), s->top());;
96 while (words_left_to_fill > 0) {
97 size_t words_to_fill = MIN2(words_left_to_fill, CollectedHeap::filler_array_max_size());
98 assert(words_to_fill >= CollectedHeap::min_fill_size(),
99 "Remaining size (" SIZE_FORMAT ") is too small to fill (based on " SIZE_FORMAT " and " SIZE_FORMAT ")",
100 words_to_fill, words_left_to_fill, CollectedHeap::filler_array_max_size());
101 CollectedHeap::fill_with_object((HeapWord*)cur_top, words_to_fill);
102 if (!os::numa_has_static_binding()) {
103 size_t touched_words = words_to_fill;
104 #ifndef ASSERT
105 if (!ZapUnusedHeapArea) {
106 touched_words = MIN2((size_t)align_object_size(typeArrayOopDesc::header_size(T_INT)),
107 touched_words);
108 }
109 #endif
110 MemRegion invalid;
111 HeapWord *crossing_start = align_up((HeapWord*)cur_top, os::vm_page_size());
112 HeapWord *crossing_end = align_down((HeapWord*)(cur_top + touched_words), os::vm_page_size());
113 if (crossing_start != crossing_end) {
114 // If object header crossed a small page boundary we mark the area
115 // as invalid rounding it to a page_size().
116 HeapWord *start = MAX2(align_down((HeapWord*)cur_top, page_size()), s->bottom());
117 HeapWord *end = MIN2(align_up((HeapWord*)(cur_top + touched_words), page_size()), s->end());
118 invalid = MemRegion(start, end);
119 }
120
121 ls->add_invalid_region(invalid);
122 }
123 cur_top = cur_top + (words_to_fill * HeapWordSize);
124 words_left_to_fill -= words_to_fill;
125 }
126 }
127 } else {
128 if (!os::numa_has_static_binding()) {
129 #ifdef ASSERT
130 MemRegion invalid(s->top(), s->end());
131 ls->add_invalid_region(invalid);
132 #else
133 if (ZapUnusedHeapArea) {
134 MemRegion invalid(s->top(), s->end());
135 ls->add_invalid_region(invalid);
136 } else {
137 return;
138 }
139 #endif
140 } else {
141 return;
142 }
143 }
144 }
145 }
146
147 size_t MutableNUMASpace::used_in_words() const {
148 size_t s = 0;
149 for (int i = 0; i < lgrp_spaces()->length(); i++) {
150 s += lgrp_spaces()->at(i)->space()->used_in_words();
151 }
152 return s;
153 }
154
155 size_t MutableNUMASpace::free_in_words() const {
156 size_t s = 0;
157 for (int i = 0; i < lgrp_spaces()->length(); i++) {
158 s += lgrp_spaces()->at(i)->space()->free_in_words();
159 }
160 return s;
161 }
162
163
164 size_t MutableNUMASpace::tlab_capacity(Thread *thr) const {
165 guarantee(thr != NULL, "No thread");
166 int lgrp_id = thr->lgrp_id();
167 if (lgrp_id == -1) {
168 // This case can occur after the topology of the system has
169 // changed. Thread can change their location, the new home
170 // group will be determined during the first allocation
171 // attempt. For now we can safely assume that all spaces
172 // have equal size because the whole space will be reinitialized.
173 if (lgrp_spaces()->length() > 0) {
174 return capacity_in_bytes() / lgrp_spaces()->length();
175 } else {
176 assert(false, "There should be at least one locality group");
177 return 0;
178 }
179 }
180 // That's the normal case, where we know the locality group of the thread.
181 int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals);
182 if (i == -1) {
183 return 0;
184 }
185 return lgrp_spaces()->at(i)->space()->capacity_in_bytes();
186 }
187
188 size_t MutableNUMASpace::tlab_used(Thread *thr) const {
189 // Please see the comments for tlab_capacity().
190 guarantee(thr != NULL, "No thread");
191 int lgrp_id = thr->lgrp_id();
192 if (lgrp_id == -1) {
193 if (lgrp_spaces()->length() > 0) {
194 return (used_in_bytes()) / lgrp_spaces()->length();
195 } else {
196 assert(false, "There should be at least one locality group");
197 return 0;
198 }
199 }
200 int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals);
201 if (i == -1) {
202 return 0;
203 }
204 return lgrp_spaces()->at(i)->space()->used_in_bytes();
205 }
206
207
208 size_t MutableNUMASpace::unsafe_max_tlab_alloc(Thread *thr) const {
209 // Please see the comments for tlab_capacity().
210 guarantee(thr != NULL, "No thread");
211 int lgrp_id = thr->lgrp_id();
212 if (lgrp_id == -1) {
213 if (lgrp_spaces()->length() > 0) {
214 return free_in_bytes() / lgrp_spaces()->length();
215 } else {
216 assert(false, "There should be at least one locality group");
217 return 0;
218 }
219 }
220 int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals);
221 if (i == -1) {
222 return 0;
223 }
224 return lgrp_spaces()->at(i)->space()->free_in_bytes();
225 }
226
227
228 size_t MutableNUMASpace::capacity_in_words(Thread* thr) const {
229 guarantee(thr != NULL, "No thread");
230 int lgrp_id = thr->lgrp_id();
231 if (lgrp_id == -1) {
232 if (lgrp_spaces()->length() > 0) {
233 return capacity_in_words() / lgrp_spaces()->length();
234 } else {
235 assert(false, "There should be at least one locality group");
236 return 0;
237 }
238 }
239 int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals);
240 if (i == -1) {
241 return 0;
242 }
243 return lgrp_spaces()->at(i)->space()->capacity_in_words();
244 }
245
246 // Check if the NUMA topology has changed. Add and remove spaces if needed.
247 // The update can be forced by setting the force parameter equal to true.
248 bool MutableNUMASpace::update_layout(bool force) {
249 // Check if the topology had changed.
250 bool changed = os::numa_topology_changed();
251 if (force || changed) {
252 // Compute lgrp intersection. Add/remove spaces.
253 int lgrp_limit = (int)os::numa_get_groups_num();
254 int *lgrp_ids = NEW_C_HEAP_ARRAY(int, lgrp_limit, mtGC);
255 int lgrp_num = (int)os::numa_get_leaf_groups(lgrp_ids, lgrp_limit);
256 assert(lgrp_num > 0, "There should be at least one locality group");
257 // Add new spaces for the new nodes
258 for (int i = 0; i < lgrp_num; i++) {
259 bool found = false;
260 for (int j = 0; j < lgrp_spaces()->length(); j++) {
261 if (lgrp_spaces()->at(j)->lgrp_id() == lgrp_ids[i]) {
262 found = true;
263 break;
264 }
265 }
266 if (!found) {
267 lgrp_spaces()->append(new LGRPSpace(lgrp_ids[i], alignment()));
268 }
269 }
270
271 // Remove spaces for the removed nodes.
272 for (int i = 0; i < lgrp_spaces()->length();) {
273 bool found = false;
274 for (int j = 0; j < lgrp_num; j++) {
275 if (lgrp_spaces()->at(i)->lgrp_id() == lgrp_ids[j]) {
276 found = true;
277 break;
278 }
279 }
280 if (!found) {
281 delete lgrp_spaces()->at(i);
282 lgrp_spaces()->remove_at(i);
283 } else {
284 i++;
285 }
286 }
287
288 FREE_C_HEAP_ARRAY(int, lgrp_ids);
289
290 if (changed) {
291 ThreadsListHandle tlh;
292 JavaThreadIterator jti(tlh.list());
293 for (JavaThread *thread = jti.first(); thread != NULL; thread = jti.next()) {
294 thread->set_lgrp_id(-1);
295 }
296 }
297 return true;
298 }
299 return false;
300 }
301
302 // Bias region towards the first-touching lgrp. Set the right page sizes.
303 void MutableNUMASpace::bias_region(MemRegion mr, int lgrp_id) {
304 HeapWord *start = align_up(mr.start(), page_size());
305 HeapWord *end = align_down(mr.end(), page_size());
306 if (end > start) {
307 MemRegion aligned_region(start, end);
308 assert((intptr_t)aligned_region.start() % page_size() == 0 &&
309 (intptr_t)aligned_region.byte_size() % page_size() == 0, "Bad alignment");
310 assert(region().contains(aligned_region), "Sanity");
311 // First we tell the OS which page size we want in the given range. The underlying
312 // large page can be broken down if we require small pages.
313 os::realign_memory((char*)aligned_region.start(), aligned_region.byte_size(), page_size());
314 // Then we uncommit the pages in the range.
315 os::free_memory((char*)aligned_region.start(), aligned_region.byte_size(), page_size());
316 // And make them local/first-touch biased.
317 os::numa_make_local((char*)aligned_region.start(), aligned_region.byte_size(), lgrp_id);
318 }
319 }
320
321 // Free all pages in the region.
322 void MutableNUMASpace::free_region(MemRegion mr) {
323 HeapWord *start = align_up(mr.start(), page_size());
324 HeapWord *end = align_down(mr.end(), page_size());
325 if (end > start) {
326 MemRegion aligned_region(start, end);
327 assert((intptr_t)aligned_region.start() % page_size() == 0 &&
328 (intptr_t)aligned_region.byte_size() % page_size() == 0, "Bad alignment");
329 assert(region().contains(aligned_region), "Sanity");
330 os::free_memory((char*)aligned_region.start(), aligned_region.byte_size(), page_size());
331 }
332 }
333
334 // Update space layout. Perform adaptation.
335 void MutableNUMASpace::update() {
336 if (update_layout(false)) {
337 // If the topology has changed, make all chunks zero-sized.
338 // And clear the alloc-rate statistics.
339 // In future we may want to handle this more gracefully in order
340 // to avoid the reallocation of the pages as much as possible.
341 for (int i = 0; i < lgrp_spaces()->length(); i++) {
342 LGRPSpace *ls = lgrp_spaces()->at(i);
343 MutableSpace *s = ls->space();
344 s->set_end(s->bottom());
345 s->set_top(s->bottom());
346 ls->clear_alloc_rate();
347 }
348 // A NUMA space is never mangled
349 initialize(region(),
350 SpaceDecorator::Clear,
351 SpaceDecorator::DontMangle);
352 } else {
353 bool should_initialize = false;
354 if (!os::numa_has_static_binding()) {
355 for (int i = 0; i < lgrp_spaces()->length(); i++) {
356 if (!lgrp_spaces()->at(i)->invalid_region().is_empty()) {
357 should_initialize = true;
358 break;
359 }
360 }
361 }
362
363 if (should_initialize ||
364 (UseAdaptiveNUMAChunkSizing && adaptation_cycles() < samples_count())) {
365 // A NUMA space is never mangled
366 initialize(region(),
367 SpaceDecorator::Clear,
368 SpaceDecorator::DontMangle);
369 }
370 }
371
372 if (NUMAStats) {
373 for (int i = 0; i < lgrp_spaces()->length(); i++) {
374 lgrp_spaces()->at(i)->accumulate_statistics(page_size());
375 }
376 }
377
378 scan_pages(NUMAPageScanRate);
379 }
380
381 // Scan pages. Free pages that have smaller size or wrong placement.
382 void MutableNUMASpace::scan_pages(size_t page_count)
383 {
384 size_t pages_per_chunk = page_count / lgrp_spaces()->length();
385 if (pages_per_chunk > 0) {
386 for (int i = 0; i < lgrp_spaces()->length(); i++) {
387 LGRPSpace *ls = lgrp_spaces()->at(i);
388 ls->scan_pages(page_size(), pages_per_chunk);
389 }
390 }
391 }
392
393 // Accumulate statistics about the allocation rate of each lgrp.
394 void MutableNUMASpace::accumulate_statistics() {
395 if (UseAdaptiveNUMAChunkSizing) {
396 for (int i = 0; i < lgrp_spaces()->length(); i++) {
397 lgrp_spaces()->at(i)->sample();
398 }
399 increment_samples_count();
400 }
401
402 if (NUMAStats) {
403 for (int i = 0; i < lgrp_spaces()->length(); i++) {
404 lgrp_spaces()->at(i)->accumulate_statistics(page_size());
405 }
406 }
407 }
408
409 // Get the current size of a chunk.
410 // This function computes the size of the chunk based on the
411 // difference between chunk ends. This allows it to work correctly in
412 // case the whole space is resized and during the process of adaptive
413 // chunk resizing.
414 size_t MutableNUMASpace::current_chunk_size(int i) {
415 HeapWord *cur_end, *prev_end;
416 if (i == 0) {
417 prev_end = bottom();
418 } else {
419 prev_end = lgrp_spaces()->at(i - 1)->space()->end();
420 }
421 if (i == lgrp_spaces()->length() - 1) {
422 cur_end = end();
423 } else {
424 cur_end = lgrp_spaces()->at(i)->space()->end();
425 }
426 if (cur_end > prev_end) {
427 return pointer_delta(cur_end, prev_end, sizeof(char));
428 }
429 return 0;
430 }
431
432 // Return the default chunk size by equally diving the space.
433 // page_size() aligned.
434 size_t MutableNUMASpace::default_chunk_size() {
435 return base_space_size() / lgrp_spaces()->length() * page_size();
436 }
437
438 // Produce a new chunk size. page_size() aligned.
439 // This function is expected to be called on sequence of i's from 0 to
440 // lgrp_spaces()->length().
441 size_t MutableNUMASpace::adaptive_chunk_size(int i, size_t limit) {
442 size_t pages_available = base_space_size();
443 for (int j = 0; j < i; j++) {
444 pages_available -= align_down(current_chunk_size(j), page_size()) / page_size();
445 }
446 pages_available -= lgrp_spaces()->length() - i - 1;
447 assert(pages_available > 0, "No pages left");
448 float alloc_rate = 0;
449 for (int j = i; j < lgrp_spaces()->length(); j++) {
450 alloc_rate += lgrp_spaces()->at(j)->alloc_rate()->average();
451 }
452 size_t chunk_size = 0;
453 if (alloc_rate > 0) {
454 LGRPSpace *ls = lgrp_spaces()->at(i);
455 chunk_size = (size_t)(ls->alloc_rate()->average() / alloc_rate * pages_available) * page_size();
456 }
457 chunk_size = MAX2(chunk_size, page_size());
458
459 if (limit > 0) {
460 limit = align_down(limit, page_size());
461 if (chunk_size > current_chunk_size(i)) {
462 size_t upper_bound = pages_available * page_size();
463 if (upper_bound > limit &&
464 current_chunk_size(i) < upper_bound - limit) {
465 // The resulting upper bound should not exceed the available
466 // amount of memory (pages_available * page_size()).
467 upper_bound = current_chunk_size(i) + limit;
468 }
469 chunk_size = MIN2(chunk_size, upper_bound);
470 } else {
471 size_t lower_bound = page_size();
472 if (current_chunk_size(i) > limit) { // lower_bound shouldn't underflow.
473 lower_bound = current_chunk_size(i) - limit;
474 }
475 chunk_size = MAX2(chunk_size, lower_bound);
476 }
477 }
478 assert(chunk_size <= pages_available * page_size(), "Chunk size out of range");
479 return chunk_size;
480 }
481
482
483 // Return the bottom_region and the top_region. Align them to page_size() boundary.
484 // |------------------new_region---------------------------------|
485 // |----bottom_region--|---intersection---|------top_region------|
486 void MutableNUMASpace::select_tails(MemRegion new_region, MemRegion intersection,
487 MemRegion* bottom_region, MemRegion *top_region) {
488 // Is there bottom?
489 if (new_region.start() < intersection.start()) { // Yes
490 // Try to coalesce small pages into a large one.
491 if (UseLargePages && page_size() >= alignment()) {
492 HeapWord* p = align_up(intersection.start(), alignment());
493 if (new_region.contains(p)
494 && pointer_delta(p, new_region.start(), sizeof(char)) >= alignment()) {
495 if (intersection.contains(p)) {
496 intersection = MemRegion(p, intersection.end());
497 } else {
498 intersection = MemRegion(p, p);
499 }
500 }
501 }
502 *bottom_region = MemRegion(new_region.start(), intersection.start());
503 } else {
504 *bottom_region = MemRegion();
505 }
506
507 // Is there top?
508 if (intersection.end() < new_region.end()) { // Yes
509 // Try to coalesce small pages into a large one.
510 if (UseLargePages && page_size() >= alignment()) {
511 HeapWord* p = align_down(intersection.end(), alignment());
512 if (new_region.contains(p)
513 && pointer_delta(new_region.end(), p, sizeof(char)) >= alignment()) {
514 if (intersection.contains(p)) {
515 intersection = MemRegion(intersection.start(), p);
516 } else {
517 intersection = MemRegion(p, p);
518 }
519 }
520 }
521 *top_region = MemRegion(intersection.end(), new_region.end());
522 } else {
523 *top_region = MemRegion();
524 }
525 }
526
527 // Try to merge the invalid region with the bottom or top region by decreasing
528 // the intersection area. Return the invalid_region aligned to the page_size()
529 // boundary if it's inside the intersection. Return non-empty invalid_region
530 // if it lies inside the intersection (also page-aligned).
531 // |------------------new_region---------------------------------|
532 // |----------------|-------invalid---|--------------------------|
533 // |----bottom_region--|---intersection---|------top_region------|
534 void MutableNUMASpace::merge_regions(MemRegion new_region, MemRegion* intersection,
535 MemRegion *invalid_region) {
536 if (intersection->start() >= invalid_region->start() && intersection->contains(invalid_region->end())) {
537 *intersection = MemRegion(invalid_region->end(), intersection->end());
538 *invalid_region = MemRegion();
539 } else
540 if (intersection->end() <= invalid_region->end() && intersection->contains(invalid_region->start())) {
541 *intersection = MemRegion(intersection->start(), invalid_region->start());
542 *invalid_region = MemRegion();
543 } else
544 if (intersection->equals(*invalid_region) || invalid_region->contains(*intersection)) {
545 *intersection = MemRegion(new_region.start(), new_region.start());
546 *invalid_region = MemRegion();
547 } else
548 if (intersection->contains(invalid_region)) {
549 // That's the only case we have to make an additional bias_region() call.
550 HeapWord* start = invalid_region->start();
551 HeapWord* end = invalid_region->end();
552 if (UseLargePages && page_size() >= alignment()) {
553 HeapWord *p = align_down(start, alignment());
554 if (new_region.contains(p)) {
555 start = p;
556 }
557 p = align_up(end, alignment());
558 if (new_region.contains(end)) {
559 end = p;
560 }
561 }
562 if (intersection->start() > start) {
563 *intersection = MemRegion(start, intersection->end());
564 }
565 if (intersection->end() < end) {
566 *intersection = MemRegion(intersection->start(), end);
567 }
568 *invalid_region = MemRegion(start, end);
569 }
570 }
571
572 void MutableNUMASpace::initialize(MemRegion mr,
573 bool clear_space,
574 bool mangle_space,
575 bool setup_pages) {
576 assert(clear_space, "Reallocation will destroy data!");
577 assert(lgrp_spaces()->length() > 0, "There should be at least one space");
578
579 MemRegion old_region = region(), new_region;
580 set_bottom(mr.start());
581 set_end(mr.end());
582 // Must always clear the space
583 clear(SpaceDecorator::DontMangle);
584
585 // Compute chunk sizes
586 size_t prev_page_size = page_size();
587 set_page_size(UseLargePages ? alignment() : os::vm_page_size());
588 HeapWord* rounded_bottom = align_up(bottom(), page_size());
589 HeapWord* rounded_end = align_down(end(), page_size());
590 size_t base_space_size_pages = pointer_delta(rounded_end, rounded_bottom, sizeof(char)) / page_size();
591
592 // Try small pages if the chunk size is too small
593 if (base_space_size_pages / lgrp_spaces()->length() == 0
594 && page_size() > (size_t)os::vm_page_size()) {
595 // Changing the page size below can lead to freeing of memory. So we fail initialization.
596 if (_must_use_large_pages) {
597 vm_exit_during_initialization("Failed initializing NUMA with large pages. Too small heap size");
598 }
599 set_page_size(os::vm_page_size());
600 rounded_bottom = align_up(bottom(), page_size());
601 rounded_end = align_down(end(), page_size());
602 base_space_size_pages = pointer_delta(rounded_end, rounded_bottom, sizeof(char)) / page_size();
603 }
604 guarantee(base_space_size_pages / lgrp_spaces()->length() > 0, "Space too small");
605 set_base_space_size(base_space_size_pages);
606
607 // Handle space resize
608 MemRegion top_region, bottom_region;
609 if (!old_region.equals(region())) {
610 new_region = MemRegion(rounded_bottom, rounded_end);
611 MemRegion intersection = new_region.intersection(old_region);
612 if (intersection.start() == NULL ||
613 intersection.end() == NULL ||
614 prev_page_size > page_size()) { // If the page size got smaller we have to change
615 // the page size preference for the whole space.
616 intersection = MemRegion(new_region.start(), new_region.start());
617 }
618 select_tails(new_region, intersection, &bottom_region, &top_region);
619 bias_region(bottom_region, lgrp_spaces()->at(0)->lgrp_id());
620 bias_region(top_region, lgrp_spaces()->at(lgrp_spaces()->length() - 1)->lgrp_id());
621 }
622
623 // Check if the space layout has changed significantly?
624 // This happens when the space has been resized so that either head or tail
625 // chunk became less than a page.
626 bool layout_valid = UseAdaptiveNUMAChunkSizing &&
627 current_chunk_size(0) > page_size() &&
628 current_chunk_size(lgrp_spaces()->length() - 1) > page_size();
629
630
631 for (int i = 0; i < lgrp_spaces()->length(); i++) {
632 LGRPSpace *ls = lgrp_spaces()->at(i);
633 MutableSpace *s = ls->space();
634 old_region = s->region();
635
636 size_t chunk_byte_size = 0, old_chunk_byte_size = 0;
637 if (i < lgrp_spaces()->length() - 1) {
638 if (!UseAdaptiveNUMAChunkSizing ||
639 (UseAdaptiveNUMAChunkSizing && NUMAChunkResizeWeight == 0) ||
640 samples_count() < AdaptiveSizePolicyReadyThreshold) {
641 // No adaptation. Divide the space equally.
642 chunk_byte_size = default_chunk_size();
643 } else
644 if (!layout_valid || NUMASpaceResizeRate == 0) {
645 // Fast adaptation. If no space resize rate is set, resize
646 // the chunks instantly.
647 chunk_byte_size = adaptive_chunk_size(i, 0);
648 } else {
649 // Slow adaptation. Resize the chunks moving no more than
650 // NUMASpaceResizeRate bytes per collection.
651 size_t limit = NUMASpaceResizeRate /
652 (lgrp_spaces()->length() * (lgrp_spaces()->length() + 1) / 2);
653 chunk_byte_size = adaptive_chunk_size(i, MAX2(limit * (i + 1), page_size()));
654 }
655
656 assert(chunk_byte_size >= page_size(), "Chunk size too small");
657 assert(chunk_byte_size <= capacity_in_bytes(), "Sanity check");
658 }
659
660 if (i == 0) { // Bottom chunk
661 if (i != lgrp_spaces()->length() - 1) {
662 new_region = MemRegion(bottom(), rounded_bottom + (chunk_byte_size >> LogHeapWordSize));
663 } else {
664 new_region = MemRegion(bottom(), end());
665 }
666 } else
667 if (i < lgrp_spaces()->length() - 1) { // Middle chunks
668 MutableSpace *ps = lgrp_spaces()->at(i - 1)->space();
669 new_region = MemRegion(ps->end(),
670 ps->end() + (chunk_byte_size >> LogHeapWordSize));
671 } else { // Top chunk
672 MutableSpace *ps = lgrp_spaces()->at(i - 1)->space();
673 new_region = MemRegion(ps->end(), end());
674 }
675 guarantee(region().contains(new_region), "Region invariant");
676
677
678 // The general case:
679 // |---------------------|--invalid---|--------------------------|
680 // |------------------new_region---------------------------------|
681 // |----bottom_region--|---intersection---|------top_region------|
682 // |----old_region----|
683 // The intersection part has all pages in place we don't need to migrate them.
684 // Pages for the top and bottom part should be freed and then reallocated.
685
686 MemRegion intersection = old_region.intersection(new_region);
687
688 if (intersection.start() == NULL || intersection.end() == NULL) {
689 intersection = MemRegion(new_region.start(), new_region.start());
690 }
691
692 if (!os::numa_has_static_binding()) {
693 MemRegion invalid_region = ls->invalid_region().intersection(new_region);
694 // Invalid region is a range of memory that could've possibly
695 // been allocated on the other node. That's relevant only on Solaris where
696 // there is no static memory binding.
697 if (!invalid_region.is_empty()) {
698 merge_regions(new_region, &intersection, &invalid_region);
699 free_region(invalid_region);
700 ls->set_invalid_region(MemRegion());
701 }
702 }
703
704 select_tails(new_region, intersection, &bottom_region, &top_region);
705
706 if (!os::numa_has_static_binding()) {
707 // If that's a system with the first-touch policy then it's enough
708 // to free the pages.
709 free_region(bottom_region);
710 free_region(top_region);
711 } else {
712 // In a system with static binding we have to change the bias whenever
713 // we reshape the heap.
714 bias_region(bottom_region, ls->lgrp_id());
715 bias_region(top_region, ls->lgrp_id());
716 }
717
718 // Clear space (set top = bottom) but never mangle.
719 s->initialize(new_region, SpaceDecorator::Clear, SpaceDecorator::DontMangle, MutableSpace::DontSetupPages);
720
721 set_adaptation_cycles(samples_count());
722 }
723 }
724
725 // Set the top of the whole space.
726 // Mark the the holes in chunks below the top() as invalid.
727 void MutableNUMASpace::set_top(HeapWord* value) {
728 bool found_top = false;
729 for (int i = 0; i < lgrp_spaces()->length();) {
730 LGRPSpace *ls = lgrp_spaces()->at(i);
731 MutableSpace *s = ls->space();
732 HeapWord *top = MAX2(align_down(s->top(), page_size()), s->bottom());
733
734 if (s->contains(value)) {
735 // Check if setting the chunk's top to a given value would create a hole less than
736 // a minimal object; assuming that's not the last chunk in which case we don't care.
737 if (i < lgrp_spaces()->length() - 1) {
738 size_t remainder = pointer_delta(s->end(), value);
739 const size_t min_fill_size = CollectedHeap::min_fill_size();
740 if (remainder < min_fill_size && remainder > 0) {
741 // Add a minimum size filler object; it will cross the chunk boundary.
742 CollectedHeap::fill_with_object(value, min_fill_size);
743 value += min_fill_size;
744 assert(!s->contains(value), "Should be in the next chunk");
745 // Restart the loop from the same chunk, since the value has moved
746 // to the next one.
747 continue;
748 }
749 }
750
751 if (!os::numa_has_static_binding() && top < value && top < s->end()) {
752 ls->add_invalid_region(MemRegion(top, value));
753 }
754 s->set_top(value);
755 found_top = true;
756 } else {
757 if (found_top) {
758 s->set_top(s->bottom());
759 } else {
760 if (!os::numa_has_static_binding() && top < s->end()) {
761 ls->add_invalid_region(MemRegion(top, s->end()));
762 }
763 s->set_top(s->end());
764 }
765 }
766 i++;
767 }
768 MutableSpace::set_top(value);
769 }
770
771 void MutableNUMASpace::clear(bool mangle_space) {
772 MutableSpace::set_top(bottom());
773 for (int i = 0; i < lgrp_spaces()->length(); i++) {
774 // Never mangle NUMA spaces because the mangling will
775 // bind the memory to a possibly unwanted lgroup.
776 lgrp_spaces()->at(i)->space()->clear(SpaceDecorator::DontMangle);
777 }
778 }
779
780 /*
781 Linux supports static memory binding, therefore the most part of the
782 logic dealing with the possible invalid page allocation is effectively
783 disabled. Besides there is no notion of the home node in Linux. A
784 thread is allowed to migrate freely. Although the scheduler is rather
785 reluctant to move threads between the nodes. We check for the current
786 node every allocation. And with a high probability a thread stays on
787 the same node for some time allowing local access to recently allocated
788 objects.
789 */
790
791 HeapWord* MutableNUMASpace::allocate(size_t size) {
792 Thread* thr = Thread::current();
793 int lgrp_id = thr->lgrp_id();
794 if (lgrp_id == -1 || !os::numa_has_group_homing()) {
795 lgrp_id = os::numa_get_group_id();
796 thr->set_lgrp_id(lgrp_id);
797 }
798
799 int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals);
800
801 // It is possible that a new CPU has been hotplugged and
802 // we haven't reshaped the space accordingly.
803 if (i == -1) {
804 i = os::random() % lgrp_spaces()->length();
805 }
806
807 LGRPSpace* ls = lgrp_spaces()->at(i);
808 MutableSpace *s = ls->space();
809 HeapWord *p = s->allocate(size);
810
811 if (p != NULL) {
812 size_t remainder = s->free_in_words();
813 if (remainder < CollectedHeap::min_fill_size() && remainder > 0) {
814 s->set_top(s->top() - size);
815 p = NULL;
816 }
817 }
818 if (p != NULL) {
819 if (top() < s->top()) { // Keep _top updated.
820 MutableSpace::set_top(s->top());
821 }
822 }
823 // Make the page allocation happen here if there is no static binding..
824 if (p != NULL && !os::numa_has_static_binding()) {
825 for (HeapWord *i = p; i < p + size; i += os::vm_page_size() >> LogHeapWordSize) {
826 *(int*)i = 0;
827 }
828 }
829 if (p == NULL) {
830 ls->set_allocation_failed();
831 }
832 return p;
833 }
834
835 // This version is lock-free.
836 HeapWord* MutableNUMASpace::cas_allocate(size_t size) {
837 Thread* thr = Thread::current();
838 int lgrp_id = thr->lgrp_id();
839 if (lgrp_id == -1 || !os::numa_has_group_homing()) {
840 lgrp_id = os::numa_get_group_id();
841 thr->set_lgrp_id(lgrp_id);
842 }
843
844 int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals);
845 // It is possible that a new CPU has been hotplugged and
846 // we haven't reshaped the space accordingly.
847 if (i == -1) {
848 i = os::random() % lgrp_spaces()->length();
849 }
850 LGRPSpace *ls = lgrp_spaces()->at(i);
851 MutableSpace *s = ls->space();
852 HeapWord *p = s->cas_allocate(size);
853 if (p != NULL) {
854 size_t remainder = pointer_delta(s->end(), p + size);
855 if (remainder < CollectedHeap::min_fill_size() && remainder > 0) {
856 if (s->cas_deallocate(p, size)) {
857 // We were the last to allocate and created a fragment less than
858 // a minimal object.
859 p = NULL;
860 } else {
861 guarantee(false, "Deallocation should always succeed");
862 }
863 }
864 }
865 if (p != NULL) {
866 HeapWord* cur_top, *cur_chunk_top = p + size;
867 while ((cur_top = top()) < cur_chunk_top) { // Keep _top updated.
868 if (Atomic::cmpxchg_ptr(cur_chunk_top, top_addr(), cur_top) == cur_top) {
869 break;
870 }
871 }
872 }
873
874 // Make the page allocation happen here if there is no static binding.
875 if (p != NULL && !os::numa_has_static_binding() ) {
876 for (HeapWord *i = p; i < p + size; i += os::vm_page_size() >> LogHeapWordSize) {
877 *(int*)i = 0;
878 }
879 }
880 if (p == NULL) {
881 ls->set_allocation_failed();
882 }
883 return p;
884 }
885
886 void MutableNUMASpace::print_short_on(outputStream* st) const {
887 MutableSpace::print_short_on(st);
888 st->print(" (");
889 for (int i = 0; i < lgrp_spaces()->length(); i++) {
890 st->print("lgrp %d: ", lgrp_spaces()->at(i)->lgrp_id());
891 lgrp_spaces()->at(i)->space()->print_short_on(st);
892 if (i < lgrp_spaces()->length() - 1) {
893 st->print(", ");
894 }
895 }
896 st->print(")");
897 }
898
899 void MutableNUMASpace::print_on(outputStream* st) const {
900 MutableSpace::print_on(st);
901 for (int i = 0; i < lgrp_spaces()->length(); i++) {
902 LGRPSpace *ls = lgrp_spaces()->at(i);
903 st->print(" lgrp %d", ls->lgrp_id());
904 ls->space()->print_on(st);
905 if (NUMAStats) {
906 for (int i = 0; i < lgrp_spaces()->length(); i++) {
907 lgrp_spaces()->at(i)->accumulate_statistics(page_size());
908 }
909 st->print(" local/remote/unbiased/uncommitted: " SIZE_FORMAT "K/"
910 SIZE_FORMAT "K/" SIZE_FORMAT "K/" SIZE_FORMAT
911 "K, large/small pages: " SIZE_FORMAT "/" SIZE_FORMAT "\n",
912 ls->space_stats()->_local_space / K,
913 ls->space_stats()->_remote_space / K,
914 ls->space_stats()->_unbiased_space / K,
915 ls->space_stats()->_uncommited_space / K,
916 ls->space_stats()->_large_pages,
917 ls->space_stats()->_small_pages);
918 }
919 }
920 }
921
922 void MutableNUMASpace::verify() {
923 // This can be called after setting an arbitrary value to the space's top,
924 // so an object can cross the chunk boundary. We ensure the parsability
925 // of the space and just walk the objects in linear fashion.
926 ensure_parsability();
927 MutableSpace::verify();
928 }
929
930 // Scan pages and gather stats about page placement and size.
931 void MutableNUMASpace::LGRPSpace::accumulate_statistics(size_t page_size) {
932 clear_space_stats();
933 char *start = (char*)align_up(space()->bottom(), page_size);
934 char* end = (char*)align_down(space()->end(), page_size);
935 if (start < end) {
936 for (char *p = start; p < end;) {
937 os::page_info info;
938 if (os::get_page_info(p, &info)) {
939 if (info.size > 0) {
940 if (info.size > (size_t)os::vm_page_size()) {
941 space_stats()->_large_pages++;
942 } else {
943 space_stats()->_small_pages++;
944 }
945 if (info.lgrp_id == lgrp_id()) {
946 space_stats()->_local_space += info.size;
947 } else {
948 space_stats()->_remote_space += info.size;
949 }
950 p += info.size;
951 } else {
952 p += os::vm_page_size();
953 space_stats()->_uncommited_space += os::vm_page_size();
954 }
955 } else {
956 return;
957 }
958 }
959 }
960 space_stats()->_unbiased_space = pointer_delta(start, space()->bottom(), sizeof(char)) +
961 pointer_delta(space()->end(), end, sizeof(char));
962
963 }
964
965 // Scan page_count pages and verify if they have the right size and right placement.
966 // If invalid pages are found they are freed in hope that subsequent reallocation
967 // will be more successful.
968 void MutableNUMASpace::LGRPSpace::scan_pages(size_t page_size, size_t page_count)
969 {
970 char* range_start = (char*)align_up(space()->bottom(), page_size);
971 char* range_end = (char*)align_down(space()->end(), page_size);
972
973 if (range_start > last_page_scanned() || last_page_scanned() >= range_end) {
974 set_last_page_scanned(range_start);
975 }
976
977 char *scan_start = last_page_scanned();
978 char* scan_end = MIN2(scan_start + page_size * page_count, range_end);
979
980 os::page_info page_expected, page_found;
981 page_expected.size = page_size;
982 page_expected.lgrp_id = lgrp_id();
983
984 char *s = scan_start;
985 while (s < scan_end) {
986 char *e = os::scan_pages(s, (char*)scan_end, &page_expected, &page_found);
987 if (e == NULL) {
988 break;
989 }
990 if (e != scan_end) {
991 assert(e < scan_end, "e: " PTR_FORMAT " scan_end: " PTR_FORMAT, p2i(e), p2i(scan_end));
992
993 if ((page_expected.size != page_size || page_expected.lgrp_id != lgrp_id())
994 && page_expected.size != 0) {
995 os::free_memory(s, pointer_delta(e, s, sizeof(char)), page_size);
996 }
997 page_expected = page_found;
998 }
999 s = e;
1000 }
1001
1002 set_last_page_scanned(scan_end);
1003 }