1
2 /*
3 * Copyright (c) 2006, 2013, Oracle and/or its affiliates. All rights reserved.
4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 *
6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "precompiled.hpp"
27 #include "gc_implementation/shared/mutableNUMASpace.hpp"
28 #include "gc_implementation/shared/spaceDecorator.hpp"
29 #include "memory/sharedHeap.hpp"
30 #include "oops/oop.inline.hpp"
31 #include "runtime/thread.inline.hpp"
32
33 MutableNUMASpace::MutableNUMASpace(size_t alignment) : MutableSpace(alignment) {
34 _lgrp_spaces = new (ResourceObj::C_HEAP, mtGC) GrowableArray<LGRPSpace*>(0, true);
35 _page_size = os::vm_page_size();
36 _adaptation_cycles = 0;
37 _samples_count = 0;
38 update_layout(true);
39 }
40
41 MutableNUMASpace::~MutableNUMASpace() {
42 for (int i = 0; i < lgrp_spaces()->length(); i++) {
43 delete lgrp_spaces()->at(i);
44 }
45 delete lgrp_spaces();
46 }
47
48 #ifndef PRODUCT
49 void MutableNUMASpace::mangle_unused_area() {
50 // This method should do nothing.
51 // It can be called on a numa space during a full compaction.
52 }
53 void MutableNUMASpace::mangle_unused_area_complete() {
54 // This method should do nothing.
55 // It can be called on a numa space during a full compaction.
56 }
57 void MutableNUMASpace::mangle_region(MemRegion mr) {
58 // This method should do nothing because numa spaces are not mangled.
59 }
60 void MutableNUMASpace::set_top_for_allocations(HeapWord* v) {
61 assert(false, "Do not mangle MutableNUMASpace's");
62 }
63 void MutableNUMASpace::set_top_for_allocations() {
64 // This method should do nothing.
65 }
66 void MutableNUMASpace::check_mangled_unused_area(HeapWord* limit) {
67 // This method should do nothing.
68 }
69 void MutableNUMASpace::check_mangled_unused_area_complete() {
70 // This method should do nothing.
71 }
72 #endif // NOT_PRODUCT
73
74 // There may be unallocated holes in the middle chunks
75 // that should be filled with dead objects to ensure parsability.
76 void MutableNUMASpace::ensure_parsability() {
77 for (int i = 0; i < lgrp_spaces()->length(); i++) {
78 LGRPSpace *ls = lgrp_spaces()->at(i);
79 MutableSpace *s = ls->space();
80 if (s->top() < top()) { // For all spaces preceding the one containing top()
81 if (s->free_in_words() > 0) {
82 intptr_t cur_top = (intptr_t)s->top();
83 size_t words_left_to_fill = pointer_delta(s->end(), s->top());;
84 while (words_left_to_fill > 0) {
85 size_t words_to_fill = MIN2(words_left_to_fill, CollectedHeap::filler_array_max_size());
86 assert(words_to_fill >= CollectedHeap::min_fill_size(),
87 err_msg("Remaining size ("SIZE_FORMAT ") is too small to fill (based on " SIZE_FORMAT " and " SIZE_FORMAT ")",
88 words_to_fill, words_left_to_fill, CollectedHeap::filler_array_max_size()));
89 CollectedHeap::fill_with_object((HeapWord*)cur_top, words_to_fill);
90 if (!os::numa_has_static_binding()) {
91 size_t touched_words = words_to_fill;
92 #ifndef ASSERT
93 if (!ZapUnusedHeapArea) {
94 touched_words = MIN2((size_t)align_object_size(typeArrayOopDesc::header_size(T_INT)),
95 touched_words);
96 }
97 #endif
98 MemRegion invalid;
99 HeapWord *crossing_start = (HeapWord*)round_to(cur_top, os::vm_page_size());
100 HeapWord *crossing_end = (HeapWord*)round_to(cur_top + touched_words, os::vm_page_size());
101 if (crossing_start != crossing_end) {
102 // If object header crossed a small page boundary we mark the area
103 // as invalid rounding it to a page_size().
104 HeapWord *start = MAX2((HeapWord*)round_down(cur_top, page_size()), s->bottom());
105 HeapWord *end = MIN2((HeapWord*)round_to(cur_top + touched_words, page_size()), s->end());
106 invalid = MemRegion(start, end);
107 }
108
109 ls->add_invalid_region(invalid);
110 }
111 cur_top = cur_top + (words_to_fill * HeapWordSize);
112 words_left_to_fill -= words_to_fill;
113 }
114 }
115 } else {
116 if (!os::numa_has_static_binding()) {
117 #ifdef ASSERT
118 MemRegion invalid(s->top(), s->end());
119 ls->add_invalid_region(invalid);
120 #else
121 if (ZapUnusedHeapArea) {
122 MemRegion invalid(s->top(), s->end());
123 ls->add_invalid_region(invalid);
124 } else {
125 return;
126 }
127 #endif
128 } else {
129 return;
130 }
131 }
132 }
133 }
134
135 size_t MutableNUMASpace::used_in_words() const {
136 size_t s = 0;
137 for (int i = 0; i < lgrp_spaces()->length(); i++) {
138 s += lgrp_spaces()->at(i)->space()->used_in_words();
139 }
140 return s;
141 }
142
143 size_t MutableNUMASpace::free_in_words() const {
144 size_t s = 0;
145 for (int i = 0; i < lgrp_spaces()->length(); i++) {
146 s += lgrp_spaces()->at(i)->space()->free_in_words();
147 }
148 return s;
149 }
150
151
152 size_t MutableNUMASpace::tlab_capacity(Thread *thr) const {
153 guarantee(thr != NULL, "No thread");
154 int lgrp_id = thr->lgrp_id();
155 if (lgrp_id == -1) {
156 // This case can occur after the topology of the system has
157 // changed. Thread can change their location, the new home
158 // group will be determined during the first allocation
159 // attempt. For now we can safely assume that all spaces
160 // have equal size because the whole space will be reinitialized.
161 if (lgrp_spaces()->length() > 0) {
162 return capacity_in_bytes() / lgrp_spaces()->length();
163 } else {
164 assert(false, "There should be at least one locality group");
165 return 0;
166 }
167 }
168 // That's the normal case, where we know the locality group of the thread.
169 int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals);
170 if (i == -1) {
171 return 0;
172 }
173 return lgrp_spaces()->at(i)->space()->capacity_in_bytes();
174 }
175
176 size_t MutableNUMASpace::tlab_used(Thread *thr) const {
177 // Please see the comments for tlab_capacity().
178 guarantee(thr != NULL, "No thread");
179 int lgrp_id = thr->lgrp_id();
180 if (lgrp_id == -1) {
181 if (lgrp_spaces()->length() > 0) {
182 return (used_in_bytes()) / lgrp_spaces()->length();
183 } else {
184 assert(false, "There should be at least one locality group");
185 return 0;
186 }
187 }
188 int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals);
189 if (i == -1) {
190 return 0;
191 }
192 return lgrp_spaces()->at(i)->space()->used_in_bytes();
193 }
194
195
196 size_t MutableNUMASpace::unsafe_max_tlab_alloc(Thread *thr) const {
197 // Please see the comments for tlab_capacity().
198 guarantee(thr != NULL, "No thread");
199 int lgrp_id = thr->lgrp_id();
200 if (lgrp_id == -1) {
201 if (lgrp_spaces()->length() > 0) {
202 return free_in_bytes() / lgrp_spaces()->length();
203 } else {
204 assert(false, "There should be at least one locality group");
205 return 0;
206 }
207 }
208 int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals);
209 if (i == -1) {
210 return 0;
211 }
212 return lgrp_spaces()->at(i)->space()->free_in_bytes();
213 }
214
215
216 size_t MutableNUMASpace::capacity_in_words(Thread* thr) const {
217 guarantee(thr != NULL, "No thread");
218 int lgrp_id = thr->lgrp_id();
219 if (lgrp_id == -1) {
220 if (lgrp_spaces()->length() > 0) {
221 return capacity_in_words() / lgrp_spaces()->length();
222 } else {
223 assert(false, "There should be at least one locality group");
224 return 0;
225 }
226 }
227 int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals);
228 if (i == -1) {
229 return 0;
230 }
231 return lgrp_spaces()->at(i)->space()->capacity_in_words();
232 }
233
234 // Check if the NUMA topology has changed. Add and remove spaces if needed.
235 // The update can be forced by setting the force parameter equal to true.
236 bool MutableNUMASpace::update_layout(bool force) {
237 // Check if the topology had changed.
238 bool changed = os::numa_topology_changed();
239 if (force || changed) {
240 // Compute lgrp intersection. Add/remove spaces.
241 int lgrp_limit = (int)os::numa_get_groups_num();
242 int *lgrp_ids = NEW_C_HEAP_ARRAY(int, lgrp_limit, mtGC);
243 int lgrp_num = (int)os::numa_get_leaf_groups(lgrp_ids, lgrp_limit);
244 assert(lgrp_num > 0, "There should be at least one locality group");
245 // Add new spaces for the new nodes
246 for (int i = 0; i < lgrp_num; i++) {
247 bool found = false;
248 for (int j = 0; j < lgrp_spaces()->length(); j++) {
249 if (lgrp_spaces()->at(j)->lgrp_id() == lgrp_ids[i]) {
250 found = true;
251 break;
252 }
253 }
254 if (!found) {
255 lgrp_spaces()->append(new LGRPSpace(lgrp_ids[i], alignment()));
256 }
257 }
258
259 // Remove spaces for the removed nodes.
260 for (int i = 0; i < lgrp_spaces()->length();) {
261 bool found = false;
262 for (int j = 0; j < lgrp_num; j++) {
263 if (lgrp_spaces()->at(i)->lgrp_id() == lgrp_ids[j]) {
264 found = true;
265 break;
266 }
267 }
268 if (!found) {
269 delete lgrp_spaces()->at(i);
270 lgrp_spaces()->remove_at(i);
271 } else {
272 i++;
273 }
274 }
275
276 FREE_C_HEAP_ARRAY(int, lgrp_ids, mtGC);
277
278 if (changed) {
279 for (JavaThread *thread = Threads::first(); thread; thread = thread->next()) {
280 thread->set_lgrp_id(-1);
281 }
282 }
283 return true;
284 }
285 return false;
286 }
287
288 // Bias region towards the first-touching lgrp. Set the right page sizes.
289 void MutableNUMASpace::bias_region(MemRegion mr, int lgrp_id) {
290 HeapWord *start = (HeapWord*)round_to((intptr_t)mr.start(), page_size());
291 HeapWord *end = (HeapWord*)round_down((intptr_t)mr.end(), page_size());
292 if (end > start) {
293 MemRegion aligned_region(start, end);
294 assert((intptr_t)aligned_region.start() % page_size() == 0 &&
295 (intptr_t)aligned_region.byte_size() % page_size() == 0, "Bad alignment");
296 assert(region().contains(aligned_region), "Sanity");
297 // First we tell the OS which page size we want in the given range. The underlying
298 // large page can be broken down if we require small pages.
299 os::realign_memory((char*)aligned_region.start(), aligned_region.byte_size(), page_size());
300 // Then we uncommit the pages in the range.
301 os::free_memory((char*)aligned_region.start(), aligned_region.byte_size(), page_size());
302 // And make them local/first-touch biased.
303 os::numa_make_local((char*)aligned_region.start(), aligned_region.byte_size(), lgrp_id);
304 }
305 }
306
307 // Free all pages in the region.
308 void MutableNUMASpace::free_region(MemRegion mr) {
309 HeapWord *start = (HeapWord*)round_to((intptr_t)mr.start(), page_size());
310 HeapWord *end = (HeapWord*)round_down((intptr_t)mr.end(), page_size());
311 if (end > start) {
312 MemRegion aligned_region(start, end);
313 assert((intptr_t)aligned_region.start() % page_size() == 0 &&
314 (intptr_t)aligned_region.byte_size() % page_size() == 0, "Bad alignment");
315 assert(region().contains(aligned_region), "Sanity");
316 os::free_memory((char*)aligned_region.start(), aligned_region.byte_size(), page_size());
317 }
318 }
319
320 // Update space layout. Perform adaptation.
321 void MutableNUMASpace::update() {
322 if (update_layout(false)) {
323 // If the topology has changed, make all chunks zero-sized.
324 // And clear the alloc-rate statistics.
325 // In future we may want to handle this more gracefully in order
326 // to avoid the reallocation of the pages as much as possible.
327 for (int i = 0; i < lgrp_spaces()->length(); i++) {
328 LGRPSpace *ls = lgrp_spaces()->at(i);
329 MutableSpace *s = ls->space();
330 s->set_end(s->bottom());
331 s->set_top(s->bottom());
332 ls->clear_alloc_rate();
333 }
334 // A NUMA space is never mangled
335 initialize(region(),
336 SpaceDecorator::Clear,
337 SpaceDecorator::DontMangle);
338 } else {
339 bool should_initialize = false;
340 if (!os::numa_has_static_binding()) {
341 for (int i = 0; i < lgrp_spaces()->length(); i++) {
342 if (!lgrp_spaces()->at(i)->invalid_region().is_empty()) {
343 should_initialize = true;
344 break;
345 }
346 }
347 }
348
349 if (should_initialize ||
350 (UseAdaptiveNUMAChunkSizing && adaptation_cycles() < samples_count())) {
351 // A NUMA space is never mangled
352 initialize(region(),
353 SpaceDecorator::Clear,
354 SpaceDecorator::DontMangle);
355 }
356 }
357
358 if (NUMAStats) {
359 for (int i = 0; i < lgrp_spaces()->length(); i++) {
360 lgrp_spaces()->at(i)->accumulate_statistics(page_size());
361 }
362 }
363
364 scan_pages(NUMAPageScanRate);
365 }
366
367 // Scan pages. Free pages that have smaller size or wrong placement.
368 void MutableNUMASpace::scan_pages(size_t page_count)
369 {
370 size_t pages_per_chunk = page_count / lgrp_spaces()->length();
371 if (pages_per_chunk > 0) {
372 for (int i = 0; i < lgrp_spaces()->length(); i++) {
373 LGRPSpace *ls = lgrp_spaces()->at(i);
374 ls->scan_pages(page_size(), pages_per_chunk);
375 }
376 }
377 }
378
379 // Accumulate statistics about the allocation rate of each lgrp.
380 void MutableNUMASpace::accumulate_statistics() {
381 if (UseAdaptiveNUMAChunkSizing) {
382 for (int i = 0; i < lgrp_spaces()->length(); i++) {
383 lgrp_spaces()->at(i)->sample();
384 }
385 increment_samples_count();
386 }
387
388 if (NUMAStats) {
389 for (int i = 0; i < lgrp_spaces()->length(); i++) {
390 lgrp_spaces()->at(i)->accumulate_statistics(page_size());
391 }
392 }
393 }
394
395 // Get the current size of a chunk.
396 // This function computes the size of the chunk based on the
397 // difference between chunk ends. This allows it to work correctly in
398 // case the whole space is resized and during the process of adaptive
399 // chunk resizing.
400 size_t MutableNUMASpace::current_chunk_size(int i) {
401 HeapWord *cur_end, *prev_end;
402 if (i == 0) {
403 prev_end = bottom();
404 } else {
405 prev_end = lgrp_spaces()->at(i - 1)->space()->end();
406 }
407 if (i == lgrp_spaces()->length() - 1) {
408 cur_end = end();
409 } else {
410 cur_end = lgrp_spaces()->at(i)->space()->end();
411 }
412 if (cur_end > prev_end) {
413 return pointer_delta(cur_end, prev_end, sizeof(char));
414 }
415 return 0;
416 }
417
418 // Return the default chunk size by equally diving the space.
419 // page_size() aligned.
420 size_t MutableNUMASpace::default_chunk_size() {
421 return base_space_size() / lgrp_spaces()->length() * page_size();
422 }
423
424 // Produce a new chunk size. page_size() aligned.
425 // This function is expected to be called on sequence of i's from 0 to
426 // lgrp_spaces()->length().
427 size_t MutableNUMASpace::adaptive_chunk_size(int i, size_t limit) {
428 size_t pages_available = base_space_size();
429 for (int j = 0; j < i; j++) {
430 pages_available -= round_down(current_chunk_size(j), page_size()) / page_size();
431 }
432 pages_available -= lgrp_spaces()->length() - i - 1;
433 assert(pages_available > 0, "No pages left");
434 float alloc_rate = 0;
435 for (int j = i; j < lgrp_spaces()->length(); j++) {
436 alloc_rate += lgrp_spaces()->at(j)->alloc_rate()->average();
437 }
438 size_t chunk_size = 0;
439 if (alloc_rate > 0) {
440 LGRPSpace *ls = lgrp_spaces()->at(i);
441 chunk_size = (size_t)(ls->alloc_rate()->average() / alloc_rate * pages_available) * page_size();
442 }
443 chunk_size = MAX2(chunk_size, page_size());
444
445 if (limit > 0) {
446 limit = round_down(limit, page_size());
447 if (chunk_size > current_chunk_size(i)) {
448 size_t upper_bound = pages_available * page_size();
449 if (upper_bound > limit &&
450 current_chunk_size(i) < upper_bound - limit) {
451 // The resulting upper bound should not exceed the available
452 // amount of memory (pages_available * page_size()).
453 upper_bound = current_chunk_size(i) + limit;
454 }
455 chunk_size = MIN2(chunk_size, upper_bound);
456 } else {
457 size_t lower_bound = page_size();
458 if (current_chunk_size(i) > limit) { // lower_bound shouldn't underflow.
459 lower_bound = current_chunk_size(i) - limit;
460 }
461 chunk_size = MAX2(chunk_size, lower_bound);
462 }
463 }
464 assert(chunk_size <= pages_available * page_size(), "Chunk size out of range");
465 return chunk_size;
466 }
467
468
469 // Return the bottom_region and the top_region. Align them to page_size() boundary.
470 // |------------------new_region---------------------------------|
471 // |----bottom_region--|---intersection---|------top_region------|
472 void MutableNUMASpace::select_tails(MemRegion new_region, MemRegion intersection,
473 MemRegion* bottom_region, MemRegion *top_region) {
474 // Is there bottom?
475 if (new_region.start() < intersection.start()) { // Yes
476 // Try to coalesce small pages into a large one.
477 if (UseLargePages && page_size() >= alignment()) {
478 HeapWord* p = (HeapWord*)round_to((intptr_t) intersection.start(), alignment());
479 if (new_region.contains(p)
480 && pointer_delta(p, new_region.start(), sizeof(char)) >= alignment()) {
481 if (intersection.contains(p)) {
482 intersection = MemRegion(p, intersection.end());
483 } else {
484 intersection = MemRegion(p, p);
485 }
486 }
487 }
488 *bottom_region = MemRegion(new_region.start(), intersection.start());
489 } else {
490 *bottom_region = MemRegion();
491 }
492
493 // Is there top?
494 if (intersection.end() < new_region.end()) { // Yes
495 // Try to coalesce small pages into a large one.
496 if (UseLargePages && page_size() >= alignment()) {
497 HeapWord* p = (HeapWord*)round_down((intptr_t) intersection.end(), alignment());
498 if (new_region.contains(p)
499 && pointer_delta(new_region.end(), p, sizeof(char)) >= alignment()) {
500 if (intersection.contains(p)) {
501 intersection = MemRegion(intersection.start(), p);
502 } else {
503 intersection = MemRegion(p, p);
504 }
505 }
506 }
507 *top_region = MemRegion(intersection.end(), new_region.end());
508 } else {
509 *top_region = MemRegion();
510 }
511 }
512
513 // Try to merge the invalid region with the bottom or top region by decreasing
514 // the intersection area. Return the invalid_region aligned to the page_size()
515 // boundary if it's inside the intersection. Return non-empty invalid_region
516 // if it lies inside the intersection (also page-aligned).
517 // |------------------new_region---------------------------------|
518 // |----------------|-------invalid---|--------------------------|
519 // |----bottom_region--|---intersection---|------top_region------|
520 void MutableNUMASpace::merge_regions(MemRegion new_region, MemRegion* intersection,
521 MemRegion *invalid_region) {
522 if (intersection->start() >= invalid_region->start() && intersection->contains(invalid_region->end())) {
523 *intersection = MemRegion(invalid_region->end(), intersection->end());
524 *invalid_region = MemRegion();
525 } else
526 if (intersection->end() <= invalid_region->end() && intersection->contains(invalid_region->start())) {
527 *intersection = MemRegion(intersection->start(), invalid_region->start());
528 *invalid_region = MemRegion();
529 } else
530 if (intersection->equals(*invalid_region) || invalid_region->contains(*intersection)) {
531 *intersection = MemRegion(new_region.start(), new_region.start());
532 *invalid_region = MemRegion();
533 } else
534 if (intersection->contains(invalid_region)) {
535 // That's the only case we have to make an additional bias_region() call.
536 HeapWord* start = invalid_region->start();
537 HeapWord* end = invalid_region->end();
538 if (UseLargePages && page_size() >= alignment()) {
539 HeapWord *p = (HeapWord*)round_down((intptr_t) start, alignment());
540 if (new_region.contains(p)) {
541 start = p;
542 }
543 p = (HeapWord*)round_to((intptr_t) end, alignment());
544 if (new_region.contains(end)) {
545 end = p;
546 }
547 }
548 if (intersection->start() > start) {
549 *intersection = MemRegion(start, intersection->end());
550 }
551 if (intersection->end() < end) {
552 *intersection = MemRegion(intersection->start(), end);
553 }
554 *invalid_region = MemRegion(start, end);
555 }
556 }
557
558 void MutableNUMASpace::initialize(MemRegion mr,
559 bool clear_space,
560 bool mangle_space,
561 bool setup_pages) {
562 assert(clear_space, "Reallocation will destroy data!");
563 assert(lgrp_spaces()->length() > 0, "There should be at least one space");
564
565 MemRegion old_region = region(), new_region;
566 set_bottom(mr.start());
567 set_end(mr.end());
568 // Must always clear the space
569 clear(SpaceDecorator::DontMangle);
570
571 // Compute chunk sizes
572 size_t prev_page_size = page_size();
573 set_page_size(UseLargePages ? alignment() : os::vm_page_size());
574 HeapWord* rounded_bottom = (HeapWord*)round_to((intptr_t) bottom(), page_size());
575 HeapWord* rounded_end = (HeapWord*)round_down((intptr_t) end(), page_size());
576 size_t base_space_size_pages = pointer_delta(rounded_end, rounded_bottom, sizeof(char)) / page_size();
577
578 // Try small pages if the chunk size is too small
579 if (base_space_size_pages / lgrp_spaces()->length() == 0
580 && page_size() > (size_t)os::vm_page_size()) {
581 set_page_size(os::vm_page_size());
582 rounded_bottom = (HeapWord*)round_to((intptr_t) bottom(), page_size());
583 rounded_end = (HeapWord*)round_down((intptr_t) end(), page_size());
584 base_space_size_pages = pointer_delta(rounded_end, rounded_bottom, sizeof(char)) / page_size();
585 }
586 guarantee(base_space_size_pages / lgrp_spaces()->length() > 0, "Space too small");
587 set_base_space_size(base_space_size_pages);
588
589 // Handle space resize
590 MemRegion top_region, bottom_region;
591 if (!old_region.equals(region())) {
592 new_region = MemRegion(rounded_bottom, rounded_end);
593 MemRegion intersection = new_region.intersection(old_region);
594 if (intersection.start() == NULL ||
595 intersection.end() == NULL ||
596 prev_page_size > page_size()) { // If the page size got smaller we have to change
597 // the page size preference for the whole space.
598 intersection = MemRegion(new_region.start(), new_region.start());
599 }
600 select_tails(new_region, intersection, &bottom_region, &top_region);
601 bias_region(bottom_region, lgrp_spaces()->at(0)->lgrp_id());
602 bias_region(top_region, lgrp_spaces()->at(lgrp_spaces()->length() - 1)->lgrp_id());
603 }
604
605 // Check if the space layout has changed significantly?
606 // This happens when the space has been resized so that either head or tail
607 // chunk became less than a page.
608 bool layout_valid = UseAdaptiveNUMAChunkSizing &&
609 current_chunk_size(0) > page_size() &&
610 current_chunk_size(lgrp_spaces()->length() - 1) > page_size();
611
612
613 for (int i = 0; i < lgrp_spaces()->length(); i++) {
614 LGRPSpace *ls = lgrp_spaces()->at(i);
615 MutableSpace *s = ls->space();
616 old_region = s->region();
617
618 size_t chunk_byte_size = 0, old_chunk_byte_size = 0;
619 if (i < lgrp_spaces()->length() - 1) {
620 if (!UseAdaptiveNUMAChunkSizing ||
621 (UseAdaptiveNUMAChunkSizing && NUMAChunkResizeWeight == 0) ||
622 samples_count() < AdaptiveSizePolicyReadyThreshold) {
623 // No adaptation. Divide the space equally.
624 chunk_byte_size = default_chunk_size();
625 } else
626 if (!layout_valid || NUMASpaceResizeRate == 0) {
627 // Fast adaptation. If no space resize rate is set, resize
628 // the chunks instantly.
629 chunk_byte_size = adaptive_chunk_size(i, 0);
630 } else {
631 // Slow adaptation. Resize the chunks moving no more than
632 // NUMASpaceResizeRate bytes per collection.
633 size_t limit = NUMASpaceResizeRate /
634 (lgrp_spaces()->length() * (lgrp_spaces()->length() + 1) / 2);
635 chunk_byte_size = adaptive_chunk_size(i, MAX2(limit * (i + 1), page_size()));
636 }
637
638 assert(chunk_byte_size >= page_size(), "Chunk size too small");
639 assert(chunk_byte_size <= capacity_in_bytes(), "Sanity check");
640 }
641
642 if (i == 0) { // Bottom chunk
643 if (i != lgrp_spaces()->length() - 1) {
644 new_region = MemRegion(bottom(), rounded_bottom + (chunk_byte_size >> LogHeapWordSize));
645 } else {
646 new_region = MemRegion(bottom(), end());
647 }
648 } else
649 if (i < lgrp_spaces()->length() - 1) { // Middle chunks
650 MutableSpace *ps = lgrp_spaces()->at(i - 1)->space();
651 new_region = MemRegion(ps->end(),
652 ps->end() + (chunk_byte_size >> LogHeapWordSize));
653 } else { // Top chunk
654 MutableSpace *ps = lgrp_spaces()->at(i - 1)->space();
655 new_region = MemRegion(ps->end(), end());
656 }
657 guarantee(region().contains(new_region), "Region invariant");
658
659
660 // The general case:
661 // |---------------------|--invalid---|--------------------------|
662 // |------------------new_region---------------------------------|
663 // |----bottom_region--|---intersection---|------top_region------|
664 // |----old_region----|
665 // The intersection part has all pages in place we don't need to migrate them.
666 // Pages for the top and bottom part should be freed and then reallocated.
667
668 MemRegion intersection = old_region.intersection(new_region);
669
670 if (intersection.start() == NULL || intersection.end() == NULL) {
671 intersection = MemRegion(new_region.start(), new_region.start());
672 }
673
674 if (!os::numa_has_static_binding()) {
675 MemRegion invalid_region = ls->invalid_region().intersection(new_region);
676 // Invalid region is a range of memory that could've possibly
677 // been allocated on the other node. That's relevant only on Solaris where
678 // there is no static memory binding.
679 if (!invalid_region.is_empty()) {
680 merge_regions(new_region, &intersection, &invalid_region);
681 free_region(invalid_region);
682 ls->set_invalid_region(MemRegion());
683 }
684 }
685
686 select_tails(new_region, intersection, &bottom_region, &top_region);
687
688 if (!os::numa_has_static_binding()) {
689 // If that's a system with the first-touch policy then it's enough
690 // to free the pages.
691 free_region(bottom_region);
692 free_region(top_region);
693 } else {
694 // In a system with static binding we have to change the bias whenever
695 // we reshape the heap.
696 bias_region(bottom_region, ls->lgrp_id());
697 bias_region(top_region, ls->lgrp_id());
698 }
699
700 // Clear space (set top = bottom) but never mangle.
701 s->initialize(new_region, SpaceDecorator::Clear, SpaceDecorator::DontMangle, MutableSpace::DontSetupPages);
702
703 set_adaptation_cycles(samples_count());
704 }
705 }
706
707 // Set the top of the whole space.
708 // Mark the the holes in chunks below the top() as invalid.
709 void MutableNUMASpace::set_top(HeapWord* value) {
710 bool found_top = false;
711 for (int i = 0; i < lgrp_spaces()->length();) {
712 LGRPSpace *ls = lgrp_spaces()->at(i);
713 MutableSpace *s = ls->space();
714 HeapWord *top = MAX2((HeapWord*)round_down((intptr_t)s->top(), page_size()), s->bottom());
715
716 if (s->contains(value)) {
717 // Check if setting the chunk's top to a given value would create a hole less than
718 // a minimal object; assuming that's not the last chunk in which case we don't care.
719 if (i < lgrp_spaces()->length() - 1) {
720 size_t remainder = pointer_delta(s->end(), value);
721 const size_t min_fill_size = CollectedHeap::min_fill_size();
722 if (remainder < min_fill_size && remainder > 0) {
723 // Add a minimum size filler object; it will cross the chunk boundary.
724 CollectedHeap::fill_with_object(value, min_fill_size);
725 value += min_fill_size;
726 assert(!s->contains(value), "Should be in the next chunk");
727 // Restart the loop from the same chunk, since the value has moved
728 // to the next one.
729 continue;
730 }
731 }
732
733 if (!os::numa_has_static_binding() && top < value && top < s->end()) {
734 ls->add_invalid_region(MemRegion(top, value));
735 }
736 s->set_top(value);
737 found_top = true;
738 } else {
739 if (found_top) {
740 s->set_top(s->bottom());
741 } else {
742 if (!os::numa_has_static_binding() && top < s->end()) {
743 ls->add_invalid_region(MemRegion(top, s->end()));
744 }
745 s->set_top(s->end());
746 }
747 }
748 i++;
749 }
750 MutableSpace::set_top(value);
751 }
752
753 void MutableNUMASpace::clear(bool mangle_space) {
754 MutableSpace::set_top(bottom());
755 for (int i = 0; i < lgrp_spaces()->length(); i++) {
756 // Never mangle NUMA spaces because the mangling will
757 // bind the memory to a possibly unwanted lgroup.
758 lgrp_spaces()->at(i)->space()->clear(SpaceDecorator::DontMangle);
759 }
760 }
761
762 /*
763 Linux supports static memory binding, therefore the most part of the
764 logic dealing with the possible invalid page allocation is effectively
765 disabled. Besides there is no notion of the home node in Linux. A
766 thread is allowed to migrate freely. Although the scheduler is rather
767 reluctant to move threads between the nodes. We check for the current
768 node every allocation. And with a high probability a thread stays on
769 the same node for some time allowing local access to recently allocated
770 objects.
771 */
772
773 HeapWord* MutableNUMASpace::allocate(size_t size) {
774 Thread* thr = Thread::current();
775 int lgrp_id = thr->lgrp_id();
776 if (lgrp_id == -1 || !os::numa_has_group_homing()) {
777 lgrp_id = os::numa_get_group_id();
778 thr->set_lgrp_id(lgrp_id);
779 }
780
781 int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals);
782
783 // It is possible that a new CPU has been hotplugged and
784 // we haven't reshaped the space accordingly.
785 if (i == -1) {
786 i = os::random() % lgrp_spaces()->length();
787 }
788
789 LGRPSpace* ls = lgrp_spaces()->at(i);
790 MutableSpace *s = ls->space();
791 HeapWord *p = s->allocate(size);
792
793 if (p != NULL) {
794 size_t remainder = s->free_in_words();
795 if (remainder < CollectedHeap::min_fill_size() && remainder > 0) {
796 s->set_top(s->top() - size);
797 p = NULL;
798 }
799 }
800 if (p != NULL) {
801 if (top() < s->top()) { // Keep _top updated.
802 MutableSpace::set_top(s->top());
803 }
804 }
805 // Make the page allocation happen here if there is no static binding..
806 if (p != NULL && !os::numa_has_static_binding()) {
807 for (HeapWord *i = p; i < p + size; i += os::vm_page_size() >> LogHeapWordSize) {
808 *(int*)i = 0;
809 }
810 }
811 if (p == NULL) {
812 ls->set_allocation_failed();
813 }
814 return p;
815 }
816
817 // This version is lock-free.
818 HeapWord* MutableNUMASpace::cas_allocate(size_t size) {
819 Thread* thr = Thread::current();
820 int lgrp_id = thr->lgrp_id();
821 if (lgrp_id == -1 || !os::numa_has_group_homing()) {
822 lgrp_id = os::numa_get_group_id();
823 thr->set_lgrp_id(lgrp_id);
824 }
825
826 int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals);
827 // It is possible that a new CPU has been hotplugged and
828 // we haven't reshaped the space accordingly.
829 if (i == -1) {
830 i = os::random() % lgrp_spaces()->length();
831 }
832 LGRPSpace *ls = lgrp_spaces()->at(i);
833 MutableSpace *s = ls->space();
834 HeapWord *p = s->cas_allocate(size);
835 if (p != NULL) {
836 size_t remainder = pointer_delta(s->end(), p + size);
837 if (remainder < CollectedHeap::min_fill_size() && remainder > 0) {
838 if (s->cas_deallocate(p, size)) {
839 // We were the last to allocate and created a fragment less than
840 // a minimal object.
841 p = NULL;
842 } else {
843 guarantee(false, "Deallocation should always succeed");
844 }
845 }
846 }
847 if (p != NULL) {
848 HeapWord* cur_top, *cur_chunk_top = p + size;
849 while ((cur_top = top()) < cur_chunk_top) { // Keep _top updated.
850 if (Atomic::cmpxchg_ptr(cur_chunk_top, top_addr(), cur_top) == cur_top) {
851 break;
852 }
853 }
854 }
855
856 // Make the page allocation happen here if there is no static binding.
857 if (p != NULL && !os::numa_has_static_binding() ) {
858 for (HeapWord *i = p; i < p + size; i += os::vm_page_size() >> LogHeapWordSize) {
859 *(int*)i = 0;
860 }
861 }
862 if (p == NULL) {
863 ls->set_allocation_failed();
864 }
865 return p;
866 }
867
868 void MutableNUMASpace::print_short_on(outputStream* st) const {
869 MutableSpace::print_short_on(st);
870 st->print(" (");
871 for (int i = 0; i < lgrp_spaces()->length(); i++) {
872 st->print("lgrp %d: ", lgrp_spaces()->at(i)->lgrp_id());
873 lgrp_spaces()->at(i)->space()->print_short_on(st);
874 if (i < lgrp_spaces()->length() - 1) {
875 st->print(", ");
876 }
877 }
878 st->print(")");
879 }
880
881 void MutableNUMASpace::print_on(outputStream* st) const {
882 MutableSpace::print_on(st);
883 for (int i = 0; i < lgrp_spaces()->length(); i++) {
884 LGRPSpace *ls = lgrp_spaces()->at(i);
885 st->print(" lgrp %d", ls->lgrp_id());
886 ls->space()->print_on(st);
887 if (NUMAStats) {
888 for (int i = 0; i < lgrp_spaces()->length(); i++) {
889 lgrp_spaces()->at(i)->accumulate_statistics(page_size());
890 }
891 st->print(" local/remote/unbiased/uncommitted: " SIZE_FORMAT "K/"
892 SIZE_FORMAT "K/" SIZE_FORMAT "K/" SIZE_FORMAT
893 "K, large/small pages: " SIZE_FORMAT "/" SIZE_FORMAT "\n",
894 ls->space_stats()->_local_space / K,
895 ls->space_stats()->_remote_space / K,
896 ls->space_stats()->_unbiased_space / K,
897 ls->space_stats()->_uncommited_space / K,
898 ls->space_stats()->_large_pages,
899 ls->space_stats()->_small_pages);
900 }
901 }
902 }
903
904 void MutableNUMASpace::verify() {
905 // This can be called after setting an arbitrary value to the space's top,
906 // so an object can cross the chunk boundary. We ensure the parsability
907 // of the space and just walk the objects in linear fashion.
908 ensure_parsability();
909 MutableSpace::verify();
910 }
911
912 // Scan pages and gather stats about page placement and size.
913 void MutableNUMASpace::LGRPSpace::accumulate_statistics(size_t page_size) {
914 clear_space_stats();
915 char *start = (char*)round_to((intptr_t) space()->bottom(), page_size);
916 char* end = (char*)round_down((intptr_t) space()->end(), page_size);
917 if (start < end) {
918 for (char *p = start; p < end;) {
919 os::page_info info;
920 if (os::get_page_info(p, &info)) {
921 if (info.size > 0) {
922 if (info.size > (size_t)os::vm_page_size()) {
923 space_stats()->_large_pages++;
924 } else {
925 space_stats()->_small_pages++;
926 }
927 if (info.lgrp_id == lgrp_id()) {
928 space_stats()->_local_space += info.size;
929 } else {
930 space_stats()->_remote_space += info.size;
931 }
932 p += info.size;
933 } else {
934 p += os::vm_page_size();
935 space_stats()->_uncommited_space += os::vm_page_size();
936 }
937 } else {
938 return;
939 }
940 }
941 }
942 space_stats()->_unbiased_space = pointer_delta(start, space()->bottom(), sizeof(char)) +
943 pointer_delta(space()->end(), end, sizeof(char));
944
945 }
946
947 // Scan page_count pages and verify if they have the right size and right placement.
948 // If invalid pages are found they are freed in hope that subsequent reallocation
949 // will be more successful.
950 void MutableNUMASpace::LGRPSpace::scan_pages(size_t page_size, size_t page_count)
951 {
952 char* range_start = (char*)round_to((intptr_t) space()->bottom(), page_size);
953 char* range_end = (char*)round_down((intptr_t) space()->end(), page_size);
954
955 if (range_start > last_page_scanned() || last_page_scanned() >= range_end) {
956 set_last_page_scanned(range_start);
957 }
958
959 char *scan_start = last_page_scanned();
960 char* scan_end = MIN2(scan_start + page_size * page_count, range_end);
961
962 os::page_info page_expected, page_found;
963 page_expected.size = page_size;
964 page_expected.lgrp_id = lgrp_id();
965
966 char *s = scan_start;
967 while (s < scan_end) {
968 char *e = os::scan_pages(s, (char*)scan_end, &page_expected, &page_found);
969 if (e == NULL) {
970 break;
971 }
972 if (e != scan_end) {
973 assert(e < scan_end, err_msg("e: " PTR_FORMAT " scan_end: " PTR_FORMAT, e, scan_end));
974
975 if ((page_expected.size != page_size || page_expected.lgrp_id != lgrp_id())
976 && page_expected.size != 0) {
977 os::free_memory(s, pointer_delta(e, s, sizeof(char)), page_size);
978 }
979 page_expected = page_found;
980 }
981 s = e;
982 }
983
984 set_last_page_scanned(scan_end);
985 }