1 /*
2 * Copyright (c) 2006, 2017, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "gc/parallel/mutableNUMASpace.hpp"
27 #include "gc/shared/collectedHeap.hpp"
28 #include "gc/shared/spaceDecorator.hpp"
29 #include "oops/oop.inline.hpp"
30 #include "runtime/atomic.hpp"
31 #include "runtime/thread.inline.hpp"
32 #include "runtime/threadSMR.hpp"
33 #include "utilities/align.hpp"
34
35 MutableNUMASpace::MutableNUMASpace(size_t alignment) : MutableSpace(alignment), _must_use_large_pages(false) {
36 _lgrp_spaces = new (ResourceObj::C_HEAP, mtGC) GrowableArray<LGRPSpace*>(0, true);
37 _page_size = os::vm_page_size();
38 _adaptation_cycles = 0;
39 _samples_count = 0;
40
41 #ifdef LINUX
42 // Changing the page size can lead to freeing of memory. When using large pages
43 // and the memory has been both reserved and committed, Linux does not support
44 // freeing parts of it.
45 if (UseLargePages && !os::can_commit_large_page_memory()) {
46 _must_use_large_pages = true;
47 }
48 #endif // LINUX
49
50 update_layout(true);
51 }
52
53 MutableNUMASpace::~MutableNUMASpace() {
54 for (int i = 0; i < lgrp_spaces()->length(); i++) {
55 delete lgrp_spaces()->at(i);
56 }
57 delete lgrp_spaces();
58 }
59
60 #ifndef PRODUCT
61 void MutableNUMASpace::mangle_unused_area() {
62 // This method should do nothing.
63 // It can be called on a numa space during a full compaction.
64 }
65 void MutableNUMASpace::mangle_unused_area_complete() {
66 // This method should do nothing.
67 // It can be called on a numa space during a full compaction.
68 }
69 void MutableNUMASpace::mangle_region(MemRegion mr) {
70 // This method should do nothing because numa spaces are not mangled.
71 }
72 void MutableNUMASpace::set_top_for_allocations(HeapWord* v) {
73 assert(false, "Do not mangle MutableNUMASpace's");
74 }
75 void MutableNUMASpace::set_top_for_allocations() {
76 // This method should do nothing.
77 }
78 void MutableNUMASpace::check_mangled_unused_area(HeapWord* limit) {
79 // This method should do nothing.
80 }
81 void MutableNUMASpace::check_mangled_unused_area_complete() {
82 // This method should do nothing.
83 }
84 #endif // NOT_PRODUCT
85
86 // There may be unallocated holes in the middle chunks
87 // that should be filled with dead objects to ensure parsability.
88 void MutableNUMASpace::ensure_parsability() {
89 for (int i = 0; i < lgrp_spaces()->length(); i++) {
90 LGRPSpace *ls = lgrp_spaces()->at(i);
91 MutableSpace *s = ls->space();
92 if (s->top() < top()) { // For all spaces preceding the one containing top()
93 if (s->free_in_words() > 0) {
94 intptr_t cur_top = (intptr_t)s->top();
95 size_t words_left_to_fill = pointer_delta(s->end(), s->top());;
96 while (words_left_to_fill > 0) {
97 size_t words_to_fill = MIN2(words_left_to_fill, CollectedHeap::filler_array_max_size());
98 assert(words_to_fill >= CollectedHeap::min_fill_size(),
99 "Remaining size (" SIZE_FORMAT ") is too small to fill (based on " SIZE_FORMAT " and " SIZE_FORMAT ")",
100 words_to_fill, words_left_to_fill, CollectedHeap::filler_array_max_size());
101 CollectedHeap::fill_with_object((HeapWord*)cur_top, words_to_fill);
102 if (!os::numa_has_static_binding()) {
103 size_t touched_words = words_to_fill;
104 #ifndef ASSERT
105 if (!ZapUnusedHeapArea) {
106 touched_words = MIN2((size_t)align_object_size(typeArrayOopDesc::header_size(T_INT)),
107 touched_words);
108 }
109 #endif
110 MemRegion invalid;
111 HeapWord *crossing_start = align_up((HeapWord*)cur_top, os::vm_page_size());
112 HeapWord *crossing_end = align_down((HeapWord*)(cur_top + touched_words), os::vm_page_size());
113 if (crossing_start != crossing_end) {
114 // If object header crossed a small page boundary we mark the area
115 // as invalid rounding it to a page_size().
116 HeapWord *start = MAX2(align_down((HeapWord*)cur_top, page_size()), s->bottom());
117 HeapWord *end = MIN2(align_up((HeapWord*)(cur_top + touched_words), page_size()), s->end());
118 invalid = MemRegion(start, end);
119 }
120
121 ls->add_invalid_region(invalid);
122 }
123 cur_top = cur_top + (words_to_fill * HeapWordSize);
124 words_left_to_fill -= words_to_fill;
125 }
126 }
127 } else {
128 if (!os::numa_has_static_binding()) {
129 #ifdef ASSERT
130 MemRegion invalid(s->top(), s->end());
131 ls->add_invalid_region(invalid);
132 #else
133 if (ZapUnusedHeapArea) {
134 MemRegion invalid(s->top(), s->end());
135 ls->add_invalid_region(invalid);
136 } else {
137 return;
138 }
139 #endif
140 } else {
141 return;
142 }
143 }
144 }
145 }
146
147 size_t MutableNUMASpace::used_in_words() const {
148 size_t s = 0;
149 for (int i = 0; i < lgrp_spaces()->length(); i++) {
150 s += lgrp_spaces()->at(i)->space()->used_in_words();
151 }
152 return s;
153 }
154
155 size_t MutableNUMASpace::free_in_words() const {
156 size_t s = 0;
157 for (int i = 0; i < lgrp_spaces()->length(); i++) {
158 s += lgrp_spaces()->at(i)->space()->free_in_words();
159 }
160 return s;
161 }
162
163
164 size_t MutableNUMASpace::tlab_capacity(Thread *thr) const {
165 guarantee(thr != NULL, "No thread");
166 int lgrp_id = thr->lgrp_id();
167 if (lgrp_id == -1) {
168 // This case can occur after the topology of the system has
169 // changed. Thread can change their location, the new home
170 // group will be determined during the first allocation
171 // attempt. For now we can safely assume that all spaces
172 // have equal size because the whole space will be reinitialized.
173 if (lgrp_spaces()->length() > 0) {
174 return capacity_in_bytes() / lgrp_spaces()->length();
175 } else {
176 assert(false, "There should be at least one locality group");
177 return 0;
178 }
179 }
180 // That's the normal case, where we know the locality group of the thread.
181 int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals);
182 if (i == -1) {
183 return 0;
184 }
185 return lgrp_spaces()->at(i)->space()->capacity_in_bytes();
186 }
187
188 size_t MutableNUMASpace::tlab_used(Thread *thr) const {
189 // Please see the comments for tlab_capacity().
190 guarantee(thr != NULL, "No thread");
191 int lgrp_id = thr->lgrp_id();
192 if (lgrp_id == -1) {
193 if (lgrp_spaces()->length() > 0) {
194 return (used_in_bytes()) / lgrp_spaces()->length();
195 } else {
196 assert(false, "There should be at least one locality group");
197 return 0;
198 }
199 }
200 int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals);
201 if (i == -1) {
202 return 0;
203 }
204 return lgrp_spaces()->at(i)->space()->used_in_bytes();
205 }
206
207
208 size_t MutableNUMASpace::unsafe_max_tlab_alloc(Thread *thr) const {
209 // Please see the comments for tlab_capacity().
210 guarantee(thr != NULL, "No thread");
211 int lgrp_id = thr->lgrp_id();
212 if (lgrp_id == -1) {
213 if (lgrp_spaces()->length() > 0) {
214 return free_in_bytes() / lgrp_spaces()->length();
215 } else {
216 assert(false, "There should be at least one locality group");
217 return 0;
218 }
219 }
220 int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals);
221 if (i == -1) {
222 return 0;
223 }
224 return lgrp_spaces()->at(i)->space()->free_in_bytes();
225 }
226
227
228 size_t MutableNUMASpace::capacity_in_words(Thread* thr) const {
229 guarantee(thr != NULL, "No thread");
230 int lgrp_id = thr->lgrp_id();
231 if (lgrp_id == -1) {
232 if (lgrp_spaces()->length() > 0) {
233 return capacity_in_words() / lgrp_spaces()->length();
234 } else {
235 assert(false, "There should be at least one locality group");
236 return 0;
237 }
238 }
239 int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals);
240 if (i == -1) {
241 return 0;
242 }
243 return lgrp_spaces()->at(i)->space()->capacity_in_words();
244 }
245
246 // Check if the NUMA topology has changed. Add and remove spaces if needed.
247 // The update can be forced by setting the force parameter equal to true.
248 bool MutableNUMASpace::update_layout(bool force) {
249 // Check if the topology had changed.
250 bool changed = os::numa_topology_changed();
251 if (force || changed) {
252 // Compute lgrp intersection. Add/remove spaces.
253 int lgrp_limit = (int)os::numa_get_groups_num();
254 int *lgrp_ids = NEW_C_HEAP_ARRAY(int, lgrp_limit, mtGC);
255 int lgrp_num = (int)os::numa_get_leaf_groups(lgrp_ids, lgrp_limit);
256 assert(lgrp_num > 0, "There should be at least one locality group");
257 // Add new spaces for the new nodes
258 for (int i = 0; i < lgrp_num; i++) {
259 bool found = false;
260 for (int j = 0; j < lgrp_spaces()->length(); j++) {
261 if (lgrp_spaces()->at(j)->lgrp_id() == lgrp_ids[i]) {
262 found = true;
263 break;
264 }
265 }
266 if (!found) {
267 lgrp_spaces()->append(new LGRPSpace(lgrp_ids[i], alignment()));
268 }
269 }
270
271 // Remove spaces for the removed nodes.
272 for (int i = 0; i < lgrp_spaces()->length();) {
273 bool found = false;
274 for (int j = 0; j < lgrp_num; j++) {
275 if (lgrp_spaces()->at(i)->lgrp_id() == lgrp_ids[j]) {
276 found = true;
277 break;
278 }
279 }
280 if (!found) {
281 delete lgrp_spaces()->at(i);
282 lgrp_spaces()->remove_at(i);
283 } else {
284 i++;
285 }
286 }
287
288 FREE_C_HEAP_ARRAY(int, lgrp_ids);
289
290 if (changed) {
291 for (JavaThreadIteratorWithHandle jtiwh; JavaThread *thread = jtiwh.next(); ) {
292 thread->set_lgrp_id(-1);
293 }
294 }
295 return true;
296 }
297 return false;
298 }
299
300 // Bias region towards the first-touching lgrp. Set the right page sizes.
301 void MutableNUMASpace::bias_region(MemRegion mr, int lgrp_id) {
302 HeapWord *start = align_up(mr.start(), page_size());
303 HeapWord *end = align_down(mr.end(), page_size());
304 if (end > start) {
305 MemRegion aligned_region(start, end);
306 assert((intptr_t)aligned_region.start() % page_size() == 0 &&
307 (intptr_t)aligned_region.byte_size() % page_size() == 0, "Bad alignment");
308 assert(region().contains(aligned_region), "Sanity");
309 // First we tell the OS which page size we want in the given range. The underlying
310 // large page can be broken down if we require small pages.
311 os::realign_memory((char*)aligned_region.start(), aligned_region.byte_size(), page_size());
312 // Then we uncommit the pages in the range.
313 os::free_memory((char*)aligned_region.start(), aligned_region.byte_size(), page_size());
314 // And make them local/first-touch biased.
315 os::numa_make_local((char*)aligned_region.start(), aligned_region.byte_size(), lgrp_id);
316 }
317 }
318
319 // Free all pages in the region.
320 void MutableNUMASpace::free_region(MemRegion mr) {
321 HeapWord *start = align_up(mr.start(), page_size());
322 HeapWord *end = align_down(mr.end(), page_size());
323 if (end > start) {
324 MemRegion aligned_region(start, end);
325 assert((intptr_t)aligned_region.start() % page_size() == 0 &&
326 (intptr_t)aligned_region.byte_size() % page_size() == 0, "Bad alignment");
327 assert(region().contains(aligned_region), "Sanity");
328 os::free_memory((char*)aligned_region.start(), aligned_region.byte_size(), page_size());
329 }
330 }
331
332 // Update space layout. Perform adaptation.
333 void MutableNUMASpace::update() {
334 if (update_layout(false)) {
335 // If the topology has changed, make all chunks zero-sized.
336 // And clear the alloc-rate statistics.
337 // In future we may want to handle this more gracefully in order
338 // to avoid the reallocation of the pages as much as possible.
339 for (int i = 0; i < lgrp_spaces()->length(); i++) {
340 LGRPSpace *ls = lgrp_spaces()->at(i);
341 MutableSpace *s = ls->space();
342 s->set_end(s->bottom());
343 s->set_top(s->bottom());
344 ls->clear_alloc_rate();
345 }
346 // A NUMA space is never mangled
347 initialize(region(),
348 SpaceDecorator::Clear,
349 SpaceDecorator::DontMangle);
350 } else {
351 bool should_initialize = false;
352 if (!os::numa_has_static_binding()) {
353 for (int i = 0; i < lgrp_spaces()->length(); i++) {
354 if (!lgrp_spaces()->at(i)->invalid_region().is_empty()) {
355 should_initialize = true;
356 break;
357 }
358 }
359 }
360
361 if (should_initialize ||
362 (UseAdaptiveNUMAChunkSizing && adaptation_cycles() < samples_count())) {
363 // A NUMA space is never mangled
364 initialize(region(),
365 SpaceDecorator::Clear,
366 SpaceDecorator::DontMangle);
367 }
368 }
369
370 if (NUMAStats) {
371 for (int i = 0; i < lgrp_spaces()->length(); i++) {
372 lgrp_spaces()->at(i)->accumulate_statistics(page_size());
373 }
374 }
375
376 scan_pages(NUMAPageScanRate);
377 }
378
379 // Scan pages. Free pages that have smaller size or wrong placement.
380 void MutableNUMASpace::scan_pages(size_t page_count)
381 {
382 size_t pages_per_chunk = page_count / lgrp_spaces()->length();
383 if (pages_per_chunk > 0) {
384 for (int i = 0; i < lgrp_spaces()->length(); i++) {
385 LGRPSpace *ls = lgrp_spaces()->at(i);
386 ls->scan_pages(page_size(), pages_per_chunk);
387 }
388 }
389 }
390
391 // Accumulate statistics about the allocation rate of each lgrp.
392 void MutableNUMASpace::accumulate_statistics() {
393 if (UseAdaptiveNUMAChunkSizing) {
394 for (int i = 0; i < lgrp_spaces()->length(); i++) {
395 lgrp_spaces()->at(i)->sample();
396 }
397 increment_samples_count();
398 }
399
400 if (NUMAStats) {
401 for (int i = 0; i < lgrp_spaces()->length(); i++) {
402 lgrp_spaces()->at(i)->accumulate_statistics(page_size());
403 }
404 }
405 }
406
407 // Get the current size of a chunk.
408 // This function computes the size of the chunk based on the
409 // difference between chunk ends. This allows it to work correctly in
410 // case the whole space is resized and during the process of adaptive
411 // chunk resizing.
412 size_t MutableNUMASpace::current_chunk_size(int i) {
413 HeapWord *cur_end, *prev_end;
414 if (i == 0) {
415 prev_end = bottom();
416 } else {
417 prev_end = lgrp_spaces()->at(i - 1)->space()->end();
418 }
419 if (i == lgrp_spaces()->length() - 1) {
420 cur_end = end();
421 } else {
422 cur_end = lgrp_spaces()->at(i)->space()->end();
423 }
424 if (cur_end > prev_end) {
425 return pointer_delta(cur_end, prev_end, sizeof(char));
426 }
427 return 0;
428 }
429
430 // Return the default chunk size by equally diving the space.
431 // page_size() aligned.
432 size_t MutableNUMASpace::default_chunk_size() {
433 return base_space_size() / lgrp_spaces()->length() * page_size();
434 }
435
436 // Produce a new chunk size. page_size() aligned.
437 // This function is expected to be called on sequence of i's from 0 to
438 // lgrp_spaces()->length().
439 size_t MutableNUMASpace::adaptive_chunk_size(int i, size_t limit) {
440 size_t pages_available = base_space_size();
441 for (int j = 0; j < i; j++) {
442 pages_available -= align_down(current_chunk_size(j), page_size()) / page_size();
443 }
444 pages_available -= lgrp_spaces()->length() - i - 1;
445 assert(pages_available > 0, "No pages left");
446 float alloc_rate = 0;
447 for (int j = i; j < lgrp_spaces()->length(); j++) {
448 alloc_rate += lgrp_spaces()->at(j)->alloc_rate()->average();
449 }
450 size_t chunk_size = 0;
451 if (alloc_rate > 0) {
452 LGRPSpace *ls = lgrp_spaces()->at(i);
453 chunk_size = (size_t)(ls->alloc_rate()->average() / alloc_rate * pages_available) * page_size();
454 }
455 chunk_size = MAX2(chunk_size, page_size());
456
457 if (limit > 0) {
458 limit = align_down(limit, page_size());
459 if (chunk_size > current_chunk_size(i)) {
460 size_t upper_bound = pages_available * page_size();
461 if (upper_bound > limit &&
462 current_chunk_size(i) < upper_bound - limit) {
463 // The resulting upper bound should not exceed the available
464 // amount of memory (pages_available * page_size()).
465 upper_bound = current_chunk_size(i) + limit;
466 }
467 chunk_size = MIN2(chunk_size, upper_bound);
468 } else {
469 size_t lower_bound = page_size();
470 if (current_chunk_size(i) > limit) { // lower_bound shouldn't underflow.
471 lower_bound = current_chunk_size(i) - limit;
472 }
473 chunk_size = MAX2(chunk_size, lower_bound);
474 }
475 }
476 assert(chunk_size <= pages_available * page_size(), "Chunk size out of range");
477 return chunk_size;
478 }
479
480
481 // Return the bottom_region and the top_region. Align them to page_size() boundary.
482 // |------------------new_region---------------------------------|
483 // |----bottom_region--|---intersection---|------top_region------|
484 void MutableNUMASpace::select_tails(MemRegion new_region, MemRegion intersection,
485 MemRegion* bottom_region, MemRegion *top_region) {
486 // Is there bottom?
487 if (new_region.start() < intersection.start()) { // Yes
488 // Try to coalesce small pages into a large one.
489 if (UseLargePages && page_size() >= alignment()) {
490 HeapWord* p = align_up(intersection.start(), alignment());
491 if (new_region.contains(p)
492 && pointer_delta(p, new_region.start(), sizeof(char)) >= alignment()) {
493 if (intersection.contains(p)) {
494 intersection = MemRegion(p, intersection.end());
495 } else {
496 intersection = MemRegion(p, p);
497 }
498 }
499 }
500 *bottom_region = MemRegion(new_region.start(), intersection.start());
501 } else {
502 *bottom_region = MemRegion();
503 }
504
505 // Is there top?
506 if (intersection.end() < new_region.end()) { // Yes
507 // Try to coalesce small pages into a large one.
508 if (UseLargePages && page_size() >= alignment()) {
509 HeapWord* p = align_down(intersection.end(), alignment());
510 if (new_region.contains(p)
511 && pointer_delta(new_region.end(), p, sizeof(char)) >= alignment()) {
512 if (intersection.contains(p)) {
513 intersection = MemRegion(intersection.start(), p);
514 } else {
515 intersection = MemRegion(p, p);
516 }
517 }
518 }
519 *top_region = MemRegion(intersection.end(), new_region.end());
520 } else {
521 *top_region = MemRegion();
522 }
523 }
524
525 // Try to merge the invalid region with the bottom or top region by decreasing
526 // the intersection area. Return the invalid_region aligned to the page_size()
527 // boundary if it's inside the intersection. Return non-empty invalid_region
528 // if it lies inside the intersection (also page-aligned).
529 // |------------------new_region---------------------------------|
530 // |----------------|-------invalid---|--------------------------|
531 // |----bottom_region--|---intersection---|------top_region------|
532 void MutableNUMASpace::merge_regions(MemRegion new_region, MemRegion* intersection,
533 MemRegion *invalid_region) {
534 if (intersection->start() >= invalid_region->start() && intersection->contains(invalid_region->end())) {
535 *intersection = MemRegion(invalid_region->end(), intersection->end());
536 *invalid_region = MemRegion();
537 } else
538 if (intersection->end() <= invalid_region->end() && intersection->contains(invalid_region->start())) {
539 *intersection = MemRegion(intersection->start(), invalid_region->start());
540 *invalid_region = MemRegion();
541 } else
542 if (intersection->equals(*invalid_region) || invalid_region->contains(*intersection)) {
543 *intersection = MemRegion(new_region.start(), new_region.start());
544 *invalid_region = MemRegion();
545 } else
546 if (intersection->contains(invalid_region)) {
547 // That's the only case we have to make an additional bias_region() call.
548 HeapWord* start = invalid_region->start();
549 HeapWord* end = invalid_region->end();
550 if (UseLargePages && page_size() >= alignment()) {
551 HeapWord *p = align_down(start, alignment());
552 if (new_region.contains(p)) {
553 start = p;
554 }
555 p = align_up(end, alignment());
556 if (new_region.contains(end)) {
557 end = p;
558 }
559 }
560 if (intersection->start() > start) {
561 *intersection = MemRegion(start, intersection->end());
562 }
563 if (intersection->end() < end) {
564 *intersection = MemRegion(intersection->start(), end);
565 }
566 *invalid_region = MemRegion(start, end);
567 }
568 }
569
570 void MutableNUMASpace::initialize(MemRegion mr,
571 bool clear_space,
572 bool mangle_space,
573 bool setup_pages) {
574 assert(clear_space, "Reallocation will destroy data!");
575 assert(lgrp_spaces()->length() > 0, "There should be at least one space");
576
577 MemRegion old_region = region(), new_region;
578 set_bottom(mr.start());
579 set_end(mr.end());
580 // Must always clear the space
581 clear(SpaceDecorator::DontMangle);
582
583 // Compute chunk sizes
584 size_t prev_page_size = page_size();
585 set_page_size(UseLargePages ? alignment() : os::vm_page_size());
586 HeapWord* rounded_bottom = align_up(bottom(), page_size());
587 HeapWord* rounded_end = align_down(end(), page_size());
588 size_t base_space_size_pages = pointer_delta(rounded_end, rounded_bottom, sizeof(char)) / page_size();
589
590 // Try small pages if the chunk size is too small
591 if (base_space_size_pages / lgrp_spaces()->length() == 0
592 && page_size() > (size_t)os::vm_page_size()) {
593 // Changing the page size below can lead to freeing of memory. So we fail initialization.
594 if (_must_use_large_pages) {
595 vm_exit_during_initialization("Failed initializing NUMA with large pages. Too small heap size");
596 }
597 set_page_size(os::vm_page_size());
598 rounded_bottom = align_up(bottom(), page_size());
599 rounded_end = align_down(end(), page_size());
600 base_space_size_pages = pointer_delta(rounded_end, rounded_bottom, sizeof(char)) / page_size();
601 }
602 guarantee(base_space_size_pages / lgrp_spaces()->length() > 0, "Space too small");
603 set_base_space_size(base_space_size_pages);
604
605 // Handle space resize
606 MemRegion top_region, bottom_region;
607 if (!old_region.equals(region())) {
608 new_region = MemRegion(rounded_bottom, rounded_end);
609 MemRegion intersection = new_region.intersection(old_region);
610 if (intersection.start() == NULL ||
611 intersection.end() == NULL ||
612 prev_page_size > page_size()) { // If the page size got smaller we have to change
613 // the page size preference for the whole space.
614 intersection = MemRegion(new_region.start(), new_region.start());
615 }
616 select_tails(new_region, intersection, &bottom_region, &top_region);
617 bias_region(bottom_region, lgrp_spaces()->at(0)->lgrp_id());
618 bias_region(top_region, lgrp_spaces()->at(lgrp_spaces()->length() - 1)->lgrp_id());
619 }
620
621 // Check if the space layout has changed significantly?
622 // This happens when the space has been resized so that either head or tail
623 // chunk became less than a page.
624 bool layout_valid = UseAdaptiveNUMAChunkSizing &&
625 current_chunk_size(0) > page_size() &&
626 current_chunk_size(lgrp_spaces()->length() - 1) > page_size();
627
628
629 for (int i = 0; i < lgrp_spaces()->length(); i++) {
630 LGRPSpace *ls = lgrp_spaces()->at(i);
631 MutableSpace *s = ls->space();
632 old_region = s->region();
633
634 size_t chunk_byte_size = 0, old_chunk_byte_size = 0;
635 if (i < lgrp_spaces()->length() - 1) {
636 if (!UseAdaptiveNUMAChunkSizing ||
637 (UseAdaptiveNUMAChunkSizing && NUMAChunkResizeWeight == 0) ||
638 samples_count() < AdaptiveSizePolicyReadyThreshold) {
639 // No adaptation. Divide the space equally.
640 chunk_byte_size = default_chunk_size();
641 } else
642 if (!layout_valid || NUMASpaceResizeRate == 0) {
643 // Fast adaptation. If no space resize rate is set, resize
644 // the chunks instantly.
645 chunk_byte_size = adaptive_chunk_size(i, 0);
646 } else {
647 // Slow adaptation. Resize the chunks moving no more than
648 // NUMASpaceResizeRate bytes per collection.
649 size_t limit = NUMASpaceResizeRate /
650 (lgrp_spaces()->length() * (lgrp_spaces()->length() + 1) / 2);
651 chunk_byte_size = adaptive_chunk_size(i, MAX2(limit * (i + 1), page_size()));
652 }
653
654 assert(chunk_byte_size >= page_size(), "Chunk size too small");
655 assert(chunk_byte_size <= capacity_in_bytes(), "Sanity check");
656 }
657
658 if (i == 0) { // Bottom chunk
659 if (i != lgrp_spaces()->length() - 1) {
660 new_region = MemRegion(bottom(), rounded_bottom + (chunk_byte_size >> LogHeapWordSize));
661 } else {
662 new_region = MemRegion(bottom(), end());
663 }
664 } else
665 if (i < lgrp_spaces()->length() - 1) { // Middle chunks
666 MutableSpace *ps = lgrp_spaces()->at(i - 1)->space();
667 new_region = MemRegion(ps->end(),
668 ps->end() + (chunk_byte_size >> LogHeapWordSize));
669 } else { // Top chunk
670 MutableSpace *ps = lgrp_spaces()->at(i - 1)->space();
671 new_region = MemRegion(ps->end(), end());
672 }
673 guarantee(region().contains(new_region), "Region invariant");
674
675
676 // The general case:
677 // |---------------------|--invalid---|--------------------------|
678 // |------------------new_region---------------------------------|
679 // |----bottom_region--|---intersection---|------top_region------|
680 // |----old_region----|
681 // The intersection part has all pages in place we don't need to migrate them.
682 // Pages for the top and bottom part should be freed and then reallocated.
683
684 MemRegion intersection = old_region.intersection(new_region);
685
686 if (intersection.start() == NULL || intersection.end() == NULL) {
687 intersection = MemRegion(new_region.start(), new_region.start());
688 }
689
690 if (!os::numa_has_static_binding()) {
691 MemRegion invalid_region = ls->invalid_region().intersection(new_region);
692 // Invalid region is a range of memory that could've possibly
693 // been allocated on the other node. That's relevant only on Solaris where
694 // there is no static memory binding.
695 if (!invalid_region.is_empty()) {
696 merge_regions(new_region, &intersection, &invalid_region);
697 free_region(invalid_region);
698 ls->set_invalid_region(MemRegion());
699 }
700 }
701
702 select_tails(new_region, intersection, &bottom_region, &top_region);
703
704 if (!os::numa_has_static_binding()) {
705 // If that's a system with the first-touch policy then it's enough
706 // to free the pages.
707 free_region(bottom_region);
708 free_region(top_region);
709 } else {
710 // In a system with static binding we have to change the bias whenever
711 // we reshape the heap.
712 bias_region(bottom_region, ls->lgrp_id());
713 bias_region(top_region, ls->lgrp_id());
714 }
715
716 // Clear space (set top = bottom) but never mangle.
717 s->initialize(new_region, SpaceDecorator::Clear, SpaceDecorator::DontMangle, MutableSpace::DontSetupPages);
718
719 set_adaptation_cycles(samples_count());
720 }
721 }
722
723 // Set the top of the whole space.
724 // Mark the the holes in chunks below the top() as invalid.
725 void MutableNUMASpace::set_top(HeapWord* value) {
726 bool found_top = false;
727 for (int i = 0; i < lgrp_spaces()->length();) {
728 LGRPSpace *ls = lgrp_spaces()->at(i);
729 MutableSpace *s = ls->space();
730 HeapWord *top = MAX2(align_down(s->top(), page_size()), s->bottom());
731
732 if (s->contains(value)) {
733 // Check if setting the chunk's top to a given value would create a hole less than
734 // a minimal object; assuming that's not the last chunk in which case we don't care.
735 if (i < lgrp_spaces()->length() - 1) {
736 size_t remainder = pointer_delta(s->end(), value);
737 const size_t min_fill_size = CollectedHeap::min_fill_size();
738 if (remainder < min_fill_size && remainder > 0) {
739 // Add a minimum size filler object; it will cross the chunk boundary.
740 CollectedHeap::fill_with_object(value, min_fill_size);
741 value += min_fill_size;
742 assert(!s->contains(value), "Should be in the next chunk");
743 // Restart the loop from the same chunk, since the value has moved
744 // to the next one.
745 continue;
746 }
747 }
748
749 if (!os::numa_has_static_binding() && top < value && top < s->end()) {
750 ls->add_invalid_region(MemRegion(top, value));
751 }
752 s->set_top(value);
753 found_top = true;
754 } else {
755 if (found_top) {
756 s->set_top(s->bottom());
757 } else {
758 if (!os::numa_has_static_binding() && top < s->end()) {
759 ls->add_invalid_region(MemRegion(top, s->end()));
760 }
761 s->set_top(s->end());
762 }
763 }
764 i++;
765 }
766 MutableSpace::set_top(value);
767 }
768
769 void MutableNUMASpace::clear(bool mangle_space) {
770 MutableSpace::set_top(bottom());
771 for (int i = 0; i < lgrp_spaces()->length(); i++) {
772 // Never mangle NUMA spaces because the mangling will
773 // bind the memory to a possibly unwanted lgroup.
774 lgrp_spaces()->at(i)->space()->clear(SpaceDecorator::DontMangle);
775 }
776 }
777
778 /*
779 Linux supports static memory binding, therefore the most part of the
780 logic dealing with the possible invalid page allocation is effectively
781 disabled. Besides there is no notion of the home node in Linux. A
782 thread is allowed to migrate freely. Although the scheduler is rather
783 reluctant to move threads between the nodes. We check for the current
784 node every allocation. And with a high probability a thread stays on
785 the same node for some time allowing local access to recently allocated
786 objects.
787 */
788
789 HeapWord* MutableNUMASpace::allocate(size_t size) {
790 Thread* thr = Thread::current();
791 int lgrp_id = thr->lgrp_id();
792 if (lgrp_id == -1 || !os::numa_has_group_homing()) {
793 lgrp_id = os::numa_get_group_id();
794 thr->set_lgrp_id(lgrp_id);
795 }
796
797 int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals);
798
799 // It is possible that a new CPU has been hotplugged and
800 // we haven't reshaped the space accordingly.
801 if (i == -1) {
802 i = os::random() % lgrp_spaces()->length();
803 }
804
805 LGRPSpace* ls = lgrp_spaces()->at(i);
806 MutableSpace *s = ls->space();
807 HeapWord *p = s->allocate(size);
808
809 if (p != NULL) {
810 size_t remainder = s->free_in_words();
811 if (remainder < CollectedHeap::min_fill_size() && remainder > 0) {
812 s->set_top(s->top() - size);
813 p = NULL;
814 }
815 }
816 if (p != NULL) {
817 if (top() < s->top()) { // Keep _top updated.
818 MutableSpace::set_top(s->top());
819 }
820 }
821 // Make the page allocation happen here if there is no static binding..
822 if (p != NULL && !os::numa_has_static_binding()) {
823 for (HeapWord *i = p; i < p + size; i += os::vm_page_size() >> LogHeapWordSize) {
824 *(int*)i = 0;
825 }
826 }
827 if (p == NULL) {
828 ls->set_allocation_failed();
829 }
830 return p;
831 }
832
833 // This version is lock-free.
834 HeapWord* MutableNUMASpace::cas_allocate(size_t size) {
835 Thread* thr = Thread::current();
836 int lgrp_id = thr->lgrp_id();
837 if (lgrp_id == -1 || !os::numa_has_group_homing()) {
838 lgrp_id = os::numa_get_group_id();
839 thr->set_lgrp_id(lgrp_id);
840 }
841
842 int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals);
843 // It is possible that a new CPU has been hotplugged and
844 // we haven't reshaped the space accordingly.
845 if (i == -1) {
846 i = os::random() % lgrp_spaces()->length();
847 }
848 LGRPSpace *ls = lgrp_spaces()->at(i);
849 MutableSpace *s = ls->space();
850 HeapWord *p = s->cas_allocate(size);
851 if (p != NULL) {
852 size_t remainder = pointer_delta(s->end(), p + size);
853 if (remainder < CollectedHeap::min_fill_size() && remainder > 0) {
854 if (s->cas_deallocate(p, size)) {
855 // We were the last to allocate and created a fragment less than
856 // a minimal object.
857 p = NULL;
858 } else {
859 guarantee(false, "Deallocation should always succeed");
860 }
861 }
862 }
863 if (p != NULL) {
864 HeapWord* cur_top, *cur_chunk_top = p + size;
865 while ((cur_top = top()) < cur_chunk_top) { // Keep _top updated.
866 if (Atomic::cmpxchg(cur_chunk_top, top_addr(), cur_top) == cur_top) {
867 break;
868 }
869 }
870 }
871
872 // Make the page allocation happen here if there is no static binding.
873 if (p != NULL && !os::numa_has_static_binding() ) {
874 for (HeapWord *i = p; i < p + size; i += os::vm_page_size() >> LogHeapWordSize) {
875 *(int*)i = 0;
876 }
877 }
878 if (p == NULL) {
879 ls->set_allocation_failed();
880 }
881 return p;
882 }
883
884 void MutableNUMASpace::print_short_on(outputStream* st) const {
885 MutableSpace::print_short_on(st);
886 st->print(" (");
887 for (int i = 0; i < lgrp_spaces()->length(); i++) {
888 st->print("lgrp %d: ", lgrp_spaces()->at(i)->lgrp_id());
889 lgrp_spaces()->at(i)->space()->print_short_on(st);
890 if (i < lgrp_spaces()->length() - 1) {
891 st->print(", ");
892 }
893 }
894 st->print(")");
895 }
896
897 void MutableNUMASpace::print_on(outputStream* st) const {
898 MutableSpace::print_on(st);
899 for (int i = 0; i < lgrp_spaces()->length(); i++) {
900 LGRPSpace *ls = lgrp_spaces()->at(i);
901 st->print(" lgrp %d", ls->lgrp_id());
902 ls->space()->print_on(st);
903 if (NUMAStats) {
904 for (int i = 0; i < lgrp_spaces()->length(); i++) {
905 lgrp_spaces()->at(i)->accumulate_statistics(page_size());
906 }
907 st->print(" local/remote/unbiased/uncommitted: " SIZE_FORMAT "K/"
908 SIZE_FORMAT "K/" SIZE_FORMAT "K/" SIZE_FORMAT
909 "K, large/small pages: " SIZE_FORMAT "/" SIZE_FORMAT "\n",
910 ls->space_stats()->_local_space / K,
911 ls->space_stats()->_remote_space / K,
912 ls->space_stats()->_unbiased_space / K,
913 ls->space_stats()->_uncommited_space / K,
914 ls->space_stats()->_large_pages,
915 ls->space_stats()->_small_pages);
916 }
917 }
918 }
919
920 void MutableNUMASpace::verify() {
921 // This can be called after setting an arbitrary value to the space's top,
922 // so an object can cross the chunk boundary. We ensure the parsability
923 // of the space and just walk the objects in linear fashion.
924 ensure_parsability();
925 MutableSpace::verify();
926 }
927
928 // Scan pages and gather stats about page placement and size.
929 void MutableNUMASpace::LGRPSpace::accumulate_statistics(size_t page_size) {
930 clear_space_stats();
931 char *start = (char*)align_up(space()->bottom(), page_size);
932 char* end = (char*)align_down(space()->end(), page_size);
933 if (start < end) {
934 for (char *p = start; p < end;) {
935 os::page_info info;
936 if (os::get_page_info(p, &info)) {
937 if (info.size > 0) {
938 if (info.size > (size_t)os::vm_page_size()) {
939 space_stats()->_large_pages++;
940 } else {
941 space_stats()->_small_pages++;
942 }
943 if (info.lgrp_id == lgrp_id()) {
944 space_stats()->_local_space += info.size;
945 } else {
946 space_stats()->_remote_space += info.size;
947 }
948 p += info.size;
949 } else {
950 p += os::vm_page_size();
951 space_stats()->_uncommited_space += os::vm_page_size();
952 }
953 } else {
954 return;
955 }
956 }
957 }
958 space_stats()->_unbiased_space = pointer_delta(start, space()->bottom(), sizeof(char)) +
959 pointer_delta(space()->end(), end, sizeof(char));
960
961 }
962
963 // Scan page_count pages and verify if they have the right size and right placement.
964 // If invalid pages are found they are freed in hope that subsequent reallocation
965 // will be more successful.
966 void MutableNUMASpace::LGRPSpace::scan_pages(size_t page_size, size_t page_count)
967 {
968 char* range_start = (char*)align_up(space()->bottom(), page_size);
969 char* range_end = (char*)align_down(space()->end(), page_size);
970
971 if (range_start > last_page_scanned() || last_page_scanned() >= range_end) {
972 set_last_page_scanned(range_start);
973 }
974
975 char *scan_start = last_page_scanned();
976 char* scan_end = MIN2(scan_start + page_size * page_count, range_end);
977
978 os::page_info page_expected, page_found;
979 page_expected.size = page_size;
980 page_expected.lgrp_id = lgrp_id();
981
982 char *s = scan_start;
983 while (s < scan_end) {
984 char *e = os::scan_pages(s, (char*)scan_end, &page_expected, &page_found);
985 if (e == NULL) {
986 break;
987 }
988 if (e != scan_end) {
989 assert(e < scan_end, "e: " PTR_FORMAT " scan_end: " PTR_FORMAT, p2i(e), p2i(scan_end));
990
991 if ((page_expected.size != page_size || page_expected.lgrp_id != lgrp_id())
992 && page_expected.size != 0) {
993 os::free_memory(s, pointer_delta(e, s, sizeof(char)), page_size);
994 }
995 page_expected = page_found;
996 }
997 s = e;
998 }
999
1000 set_last_page_scanned(scan_end);
1001 }