1
2 /*
3 * Copyright (c) 2006, 2012, Oracle and/or its affiliates. All rights reserved.
4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 *
6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "precompiled.hpp"
27 #include "gc_implementation/shared/mutableNUMASpace.hpp"
28 #include "gc_implementation/shared/spaceDecorator.hpp"
29 #include "memory/sharedHeap.hpp"
30 #include "oops/oop.inline.hpp"
31 #ifdef TARGET_OS_FAMILY_linux
32 # include "thread_linux.inline.hpp"
33 #endif
34 #ifdef TARGET_OS_FAMILY_solaris
35 # include "thread_solaris.inline.hpp"
36 #endif
37 #ifdef TARGET_OS_FAMILY_windows
38 # include "thread_windows.inline.hpp"
39 #endif
40 #ifdef TARGET_OS_FAMILY_bsd
41 # include "thread_bsd.inline.hpp"
42 #endif
43
44
45 MutableNUMASpace::MutableNUMASpace(size_t alignment) : MutableSpace(alignment) {
46 _lgrp_spaces = new (ResourceObj::C_HEAP, mtGC) GrowableArray<LGRPSpace*>(0, true);
47 _page_size = os::vm_page_size();
48 _adaptation_cycles = 0;
49 _samples_count = 0;
50 update_layout(true);
51 }
52
53 MutableNUMASpace::~MutableNUMASpace() {
54 for (int i = 0; i < lgrp_spaces()->length(); i++) {
55 delete lgrp_spaces()->at(i);
56 }
57 delete lgrp_spaces();
58 }
59
60 #ifndef PRODUCT
61 void MutableNUMASpace::mangle_unused_area() {
62 // This method should do nothing.
63 // It can be called on a numa space during a full compaction.
64 }
65 void MutableNUMASpace::mangle_unused_area_complete() {
66 // This method should do nothing.
67 // It can be called on a numa space during a full compaction.
68 }
69 void MutableNUMASpace::mangle_region(MemRegion mr) {
70 // This method should do nothing because numa spaces are not mangled.
71 }
72 void MutableNUMASpace::set_top_for_allocations(HeapWord* v) {
73 assert(false, "Do not mangle MutableNUMASpace's");
74 }
75 void MutableNUMASpace::set_top_for_allocations() {
76 // This method should do nothing.
77 }
78 void MutableNUMASpace::check_mangled_unused_area(HeapWord* limit) {
79 // This method should do nothing.
80 }
81 void MutableNUMASpace::check_mangled_unused_area_complete() {
82 // This method should do nothing.
83 }
84 #endif // NOT_PRODUCT
85
86 // There may be unallocated holes in the middle chunks
87 // that should be filled with dead objects to ensure parseability.
88 void MutableNUMASpace::ensure_parsability() {
89 for (int i = 0; i < lgrp_spaces()->length(); i++) {
90 LGRPSpace *ls = lgrp_spaces()->at(i);
91 MutableSpace *s = ls->space();
92 if (s->top() < top()) { // For all spaces preceding the one containing top()
93 if (s->free_in_words() > 0) {
94 intptr_t cur_top = (intptr_t)s->top();
95 size_t words_left_to_fill = pointer_delta(s->end(), s->top());;
96 while (words_left_to_fill > 0) {
97 size_t words_to_fill = MIN2(words_left_to_fill, CollectedHeap::filler_array_max_size());
98 assert(words_to_fill >= CollectedHeap::min_fill_size(),
99 err_msg("Remaining size ("SIZE_FORMAT ") is too small to fill (based on " SIZE_FORMAT " and " SIZE_FORMAT ")",
100 words_to_fill, words_left_to_fill, CollectedHeap::filler_array_max_size()));
101 CollectedHeap::fill_with_object((HeapWord*)cur_top, words_to_fill);
102 if (!os::numa_has_static_binding()) {
103 size_t touched_words = words_to_fill;
104 #ifndef ASSERT
105 if (!ZapUnusedHeapArea) {
106 touched_words = MIN2((size_t)align_object_size(typeArrayOopDesc::header_size(T_INT)),
107 touched_words);
108 }
109 #endif
110 MemRegion invalid;
111 HeapWord *crossing_start = (HeapWord*)round_to(cur_top, os::vm_page_size());
112 HeapWord *crossing_end = (HeapWord*)round_to(cur_top + touched_words, os::vm_page_size());
113 if (crossing_start != crossing_end) {
114 // If object header crossed a small page boundary we mark the area
115 // as invalid rounding it to a page_size().
116 HeapWord *start = MAX2((HeapWord*)round_down(cur_top, page_size()), s->bottom());
117 HeapWord *end = MIN2((HeapWord*)round_to(cur_top + touched_words, page_size()), s->end());
118 invalid = MemRegion(start, end);
119 }
120
121 ls->add_invalid_region(invalid);
122 }
123 cur_top = cur_top + (words_to_fill * HeapWordSize);
124 words_left_to_fill -= words_to_fill;
125 }
126 }
127 } else {
128 if (!os::numa_has_static_binding()) {
129 #ifdef ASSERT
130 MemRegion invalid(s->top(), s->end());
131 ls->add_invalid_region(invalid);
132 #else
133 if (ZapUnusedHeapArea) {
134 MemRegion invalid(s->top(), s->end());
135 ls->add_invalid_region(invalid);
136 } else {
137 return;
138 }
139 #endif
140 } else {
141 return;
142 }
143 }
144 }
145 }
146
147 size_t MutableNUMASpace::used_in_words() const {
148 size_t s = 0;
149 for (int i = 0; i < lgrp_spaces()->length(); i++) {
150 s += lgrp_spaces()->at(i)->space()->used_in_words();
151 }
152 return s;
153 }
154
155 size_t MutableNUMASpace::free_in_words() const {
156 size_t s = 0;
157 for (int i = 0; i < lgrp_spaces()->length(); i++) {
158 s += lgrp_spaces()->at(i)->space()->free_in_words();
159 }
160 return s;
161 }
162
163
164 size_t MutableNUMASpace::tlab_capacity(Thread *thr) const {
165 guarantee(thr != NULL, "No thread");
166 int lgrp_id = thr->lgrp_id();
167 if (lgrp_id == -1) {
168 // This case can occur after the topology of the system has
169 // changed. Thread can change their location, the new home
170 // group will be determined during the first allocation
171 // attempt. For now we can safely assume that all spaces
172 // have equal size because the whole space will be reinitialized.
173 if (lgrp_spaces()->length() > 0) {
174 return capacity_in_bytes() / lgrp_spaces()->length();
175 } else {
176 assert(false, "There should be at least one locality group");
177 return 0;
178 }
179 }
180 // That's the normal case, where we know the locality group of the thread.
181 int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals);
182 if (i == -1) {
183 return 0;
184 }
185 return lgrp_spaces()->at(i)->space()->capacity_in_bytes();
186 }
187
188 size_t MutableNUMASpace::unsafe_max_tlab_alloc(Thread *thr) const {
189 // Please see the comments for tlab_capacity().
190 guarantee(thr != NULL, "No thread");
191 int lgrp_id = thr->lgrp_id();
192 if (lgrp_id == -1) {
193 if (lgrp_spaces()->length() > 0) {
194 return free_in_bytes() / lgrp_spaces()->length();
195 } else {
196 assert(false, "There should be at least one locality group");
197 return 0;
198 }
199 }
200 int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals);
201 if (i == -1) {
202 return 0;
203 }
204 return lgrp_spaces()->at(i)->space()->free_in_bytes();
205 }
206
207
208 size_t MutableNUMASpace::capacity_in_words(Thread* thr) const {
209 guarantee(thr != NULL, "No thread");
210 int lgrp_id = thr->lgrp_id();
211 if (lgrp_id == -1) {
212 if (lgrp_spaces()->length() > 0) {
213 return capacity_in_words() / lgrp_spaces()->length();
214 } else {
215 assert(false, "There should be at least one locality group");
216 return 0;
217 }
218 }
219 int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals);
220 if (i == -1) {
221 return 0;
222 }
223 return lgrp_spaces()->at(i)->space()->capacity_in_words();
224 }
225
226 // Check if the NUMA topology has changed. Add and remove spaces if needed.
227 // The update can be forced by setting the force parameter equal to true.
228 bool MutableNUMASpace::update_layout(bool force) {
229 // Check if the topology had changed.
230 bool changed = os::numa_topology_changed();
231 if (force || changed) {
232 // Compute lgrp intersection. Add/remove spaces.
233 int lgrp_limit = (int)os::numa_get_groups_num();
234 int *lgrp_ids = NEW_C_HEAP_ARRAY(int, lgrp_limit, mtGC);
235 int lgrp_num = (int)os::numa_get_leaf_groups(lgrp_ids, lgrp_limit);
236 assert(lgrp_num > 0, "There should be at least one locality group");
237 // Add new spaces for the new nodes
238 for (int i = 0; i < lgrp_num; i++) {
239 bool found = false;
240 for (int j = 0; j < lgrp_spaces()->length(); j++) {
241 if (lgrp_spaces()->at(j)->lgrp_id() == lgrp_ids[i]) {
242 found = true;
243 break;
244 }
245 }
246 if (!found) {
247 lgrp_spaces()->append(new LGRPSpace(lgrp_ids[i], alignment()));
248 }
249 }
250
251 // Remove spaces for the removed nodes.
252 for (int i = 0; i < lgrp_spaces()->length();) {
253 bool found = false;
254 for (int j = 0; j < lgrp_num; j++) {
255 if (lgrp_spaces()->at(i)->lgrp_id() == lgrp_ids[j]) {
256 found = true;
257 break;
258 }
259 }
260 if (!found) {
261 delete lgrp_spaces()->at(i);
262 lgrp_spaces()->remove_at(i);
263 } else {
264 i++;
265 }
266 }
267
268 FREE_C_HEAP_ARRAY(int, lgrp_ids, mtGC);
269
270 if (changed) {
271 for (JavaThread *thread = Threads::first(); thread; thread = thread->next()) {
272 thread->set_lgrp_id(-1);
273 }
274 }
275 return true;
276 }
277 return false;
278 }
279
280 // Bias region towards the first-touching lgrp. Set the right page sizes.
281 void MutableNUMASpace::bias_region(MemRegion mr, int lgrp_id) {
282 HeapWord *start = (HeapWord*)round_to((intptr_t)mr.start(), page_size());
283 HeapWord *end = (HeapWord*)round_down((intptr_t)mr.end(), page_size());
284 if (end > start) {
285 MemRegion aligned_region(start, end);
286 assert((intptr_t)aligned_region.start() % page_size() == 0 &&
287 (intptr_t)aligned_region.byte_size() % page_size() == 0, "Bad alignment");
288 assert(region().contains(aligned_region), "Sanity");
289 // First we tell the OS which page size we want in the given range. The underlying
290 // large page can be broken down if we require small pages.
291 os::realign_memory((char*)aligned_region.start(), aligned_region.byte_size(), page_size());
292 // Then we uncommit the pages in the range.
293 os::free_memory((char*)aligned_region.start(), aligned_region.byte_size(), page_size());
294 // And make them local/first-touch biased.
295 os::numa_make_local((char*)aligned_region.start(), aligned_region.byte_size(), lgrp_id);
296 }
297 }
298
299 // Free all pages in the region.
300 void MutableNUMASpace::free_region(MemRegion mr) {
301 HeapWord *start = (HeapWord*)round_to((intptr_t)mr.start(), page_size());
302 HeapWord *end = (HeapWord*)round_down((intptr_t)mr.end(), page_size());
303 if (end > start) {
304 MemRegion aligned_region(start, end);
305 assert((intptr_t)aligned_region.start() % page_size() == 0 &&
306 (intptr_t)aligned_region.byte_size() % page_size() == 0, "Bad alignment");
307 assert(region().contains(aligned_region), "Sanity");
308 os::free_memory((char*)aligned_region.start(), aligned_region.byte_size(), page_size());
309 }
310 }
311
312 // Update space layout. Perform adaptation.
313 void MutableNUMASpace::update() {
314 if (update_layout(false)) {
315 // If the topology has changed, make all chunks zero-sized.
316 // And clear the alloc-rate statistics.
317 // In future we may want to handle this more gracefully in order
318 // to avoid the reallocation of the pages as much as possible.
319 for (int i = 0; i < lgrp_spaces()->length(); i++) {
320 LGRPSpace *ls = lgrp_spaces()->at(i);
321 MutableSpace *s = ls->space();
322 s->set_end(s->bottom());
323 s->set_top(s->bottom());
324 ls->clear_alloc_rate();
325 }
326 // A NUMA space is never mangled
327 initialize(region(),
328 SpaceDecorator::Clear,
329 SpaceDecorator::DontMangle);
330 } else {
331 bool should_initialize = false;
332 if (!os::numa_has_static_binding()) {
333 for (int i = 0; i < lgrp_spaces()->length(); i++) {
334 if (!lgrp_spaces()->at(i)->invalid_region().is_empty()) {
335 should_initialize = true;
336 break;
337 }
338 }
339 }
340
341 if (should_initialize ||
342 (UseAdaptiveNUMAChunkSizing && adaptation_cycles() < samples_count())) {
343 // A NUMA space is never mangled
344 initialize(region(),
345 SpaceDecorator::Clear,
346 SpaceDecorator::DontMangle);
347 }
348 }
349
350 if (NUMAStats) {
351 for (int i = 0; i < lgrp_spaces()->length(); i++) {
352 lgrp_spaces()->at(i)->accumulate_statistics(page_size());
353 }
354 }
355
356 scan_pages(NUMAPageScanRate);
357 }
358
359 // Scan pages. Free pages that have smaller size or wrong placement.
360 void MutableNUMASpace::scan_pages(size_t page_count)
361 {
362 size_t pages_per_chunk = page_count / lgrp_spaces()->length();
363 if (pages_per_chunk > 0) {
364 for (int i = 0; i < lgrp_spaces()->length(); i++) {
365 LGRPSpace *ls = lgrp_spaces()->at(i);
366 ls->scan_pages(page_size(), pages_per_chunk);
367 }
368 }
369 }
370
371 // Accumulate statistics about the allocation rate of each lgrp.
372 void MutableNUMASpace::accumulate_statistics() {
373 if (UseAdaptiveNUMAChunkSizing) {
374 for (int i = 0; i < lgrp_spaces()->length(); i++) {
375 lgrp_spaces()->at(i)->sample();
376 }
377 increment_samples_count();
378 }
379
380 if (NUMAStats) {
381 for (int i = 0; i < lgrp_spaces()->length(); i++) {
382 lgrp_spaces()->at(i)->accumulate_statistics(page_size());
383 }
384 }
385 }
386
387 // Get the current size of a chunk.
388 // This function computes the size of the chunk based on the
389 // difference between chunk ends. This allows it to work correctly in
390 // case the whole space is resized and during the process of adaptive
391 // chunk resizing.
392 size_t MutableNUMASpace::current_chunk_size(int i) {
393 HeapWord *cur_end, *prev_end;
394 if (i == 0) {
395 prev_end = bottom();
396 } else {
397 prev_end = lgrp_spaces()->at(i - 1)->space()->end();
398 }
399 if (i == lgrp_spaces()->length() - 1) {
400 cur_end = end();
401 } else {
402 cur_end = lgrp_spaces()->at(i)->space()->end();
403 }
404 if (cur_end > prev_end) {
405 return pointer_delta(cur_end, prev_end, sizeof(char));
406 }
407 return 0;
408 }
409
410 // Return the default chunk size by equally diving the space.
411 // page_size() aligned.
412 size_t MutableNUMASpace::default_chunk_size() {
413 return base_space_size() / lgrp_spaces()->length() * page_size();
414 }
415
416 // Produce a new chunk size. page_size() aligned.
417 // This function is expected to be called on sequence of i's from 0 to
418 // lgrp_spaces()->length().
419 size_t MutableNUMASpace::adaptive_chunk_size(int i, size_t limit) {
420 size_t pages_available = base_space_size();
421 for (int j = 0; j < i; j++) {
422 pages_available -= round_down(current_chunk_size(j), page_size()) / page_size();
423 }
424 pages_available -= lgrp_spaces()->length() - i - 1;
425 assert(pages_available > 0, "No pages left");
426 float alloc_rate = 0;
427 for (int j = i; j < lgrp_spaces()->length(); j++) {
428 alloc_rate += lgrp_spaces()->at(j)->alloc_rate()->average();
429 }
430 size_t chunk_size = 0;
431 if (alloc_rate > 0) {
432 LGRPSpace *ls = lgrp_spaces()->at(i);
433 chunk_size = (size_t)(ls->alloc_rate()->average() / alloc_rate * pages_available) * page_size();
434 }
435 chunk_size = MAX2(chunk_size, page_size());
436
437 if (limit > 0) {
438 limit = round_down(limit, page_size());
439 if (chunk_size > current_chunk_size(i)) {
440 size_t upper_bound = pages_available * page_size();
441 if (upper_bound > limit &&
442 current_chunk_size(i) < upper_bound - limit) {
443 // The resulting upper bound should not exceed the available
444 // amount of memory (pages_available * page_size()).
445 upper_bound = current_chunk_size(i) + limit;
446 }
447 chunk_size = MIN2(chunk_size, upper_bound);
448 } else {
449 size_t lower_bound = page_size();
450 if (current_chunk_size(i) > limit) { // lower_bound shouldn't underflow.
451 lower_bound = current_chunk_size(i) - limit;
452 }
453 chunk_size = MAX2(chunk_size, lower_bound);
454 }
455 }
456 assert(chunk_size <= pages_available * page_size(), "Chunk size out of range");
457 return chunk_size;
458 }
459
460
461 // Return the bottom_region and the top_region. Align them to page_size() boundary.
462 // |------------------new_region---------------------------------|
463 // |----bottom_region--|---intersection---|------top_region------|
464 void MutableNUMASpace::select_tails(MemRegion new_region, MemRegion intersection,
465 MemRegion* bottom_region, MemRegion *top_region) {
466 // Is there bottom?
467 if (new_region.start() < intersection.start()) { // Yes
468 // Try to coalesce small pages into a large one.
469 if (UseLargePages && page_size() >= alignment()) {
470 HeapWord* p = (HeapWord*)round_to((intptr_t) intersection.start(), alignment());
471 if (new_region.contains(p)
472 && pointer_delta(p, new_region.start(), sizeof(char)) >= alignment()) {
473 if (intersection.contains(p)) {
474 intersection = MemRegion(p, intersection.end());
475 } else {
476 intersection = MemRegion(p, p);
477 }
478 }
479 }
480 *bottom_region = MemRegion(new_region.start(), intersection.start());
481 } else {
482 *bottom_region = MemRegion();
483 }
484
485 // Is there top?
486 if (intersection.end() < new_region.end()) { // Yes
487 // Try to coalesce small pages into a large one.
488 if (UseLargePages && page_size() >= alignment()) {
489 HeapWord* p = (HeapWord*)round_down((intptr_t) intersection.end(), alignment());
490 if (new_region.contains(p)
491 && pointer_delta(new_region.end(), p, sizeof(char)) >= alignment()) {
492 if (intersection.contains(p)) {
493 intersection = MemRegion(intersection.start(), p);
494 } else {
495 intersection = MemRegion(p, p);
496 }
497 }
498 }
499 *top_region = MemRegion(intersection.end(), new_region.end());
500 } else {
501 *top_region = MemRegion();
502 }
503 }
504
505 // Try to merge the invalid region with the bottom or top region by decreasing
506 // the intersection area. Return the invalid_region aligned to the page_size()
507 // boundary if it's inside the intersection. Return non-empty invalid_region
508 // if it lies inside the intersection (also page-aligned).
509 // |------------------new_region---------------------------------|
510 // |----------------|-------invalid---|--------------------------|
511 // |----bottom_region--|---intersection---|------top_region------|
512 void MutableNUMASpace::merge_regions(MemRegion new_region, MemRegion* intersection,
513 MemRegion *invalid_region) {
514 if (intersection->start() >= invalid_region->start() && intersection->contains(invalid_region->end())) {
515 *intersection = MemRegion(invalid_region->end(), intersection->end());
516 *invalid_region = MemRegion();
517 } else
518 if (intersection->end() <= invalid_region->end() && intersection->contains(invalid_region->start())) {
519 *intersection = MemRegion(intersection->start(), invalid_region->start());
520 *invalid_region = MemRegion();
521 } else
522 if (intersection->equals(*invalid_region) || invalid_region->contains(*intersection)) {
523 *intersection = MemRegion(new_region.start(), new_region.start());
524 *invalid_region = MemRegion();
525 } else
526 if (intersection->contains(invalid_region)) {
527 // That's the only case we have to make an additional bias_region() call.
528 HeapWord* start = invalid_region->start();
529 HeapWord* end = invalid_region->end();
530 if (UseLargePages && page_size() >= alignment()) {
531 HeapWord *p = (HeapWord*)round_down((intptr_t) start, alignment());
532 if (new_region.contains(p)) {
533 start = p;
534 }
535 p = (HeapWord*)round_to((intptr_t) end, alignment());
536 if (new_region.contains(end)) {
537 end = p;
538 }
539 }
540 if (intersection->start() > start) {
541 *intersection = MemRegion(start, intersection->end());
542 }
543 if (intersection->end() < end) {
544 *intersection = MemRegion(intersection->start(), end);
545 }
546 *invalid_region = MemRegion(start, end);
547 }
548 }
549
550 void MutableNUMASpace::initialize(MemRegion mr,
551 bool clear_space,
552 bool mangle_space,
553 bool setup_pages) {
554 assert(clear_space, "Reallocation will destory data!");
555 assert(lgrp_spaces()->length() > 0, "There should be at least one space");
556
557 MemRegion old_region = region(), new_region;
558 set_bottom(mr.start());
559 set_end(mr.end());
560 // Must always clear the space
561 clear(SpaceDecorator::DontMangle);
562
563 // Compute chunk sizes
564 size_t prev_page_size = page_size();
565 set_page_size(UseLargePages ? alignment() : os::vm_page_size());
566 HeapWord* rounded_bottom = (HeapWord*)round_to((intptr_t) bottom(), page_size());
567 HeapWord* rounded_end = (HeapWord*)round_down((intptr_t) end(), page_size());
568 size_t base_space_size_pages = pointer_delta(rounded_end, rounded_bottom, sizeof(char)) / page_size();
569
570 // Try small pages if the chunk size is too small
571 if (base_space_size_pages / lgrp_spaces()->length() == 0
572 && page_size() > (size_t)os::vm_page_size()) {
573 set_page_size(os::vm_page_size());
574 rounded_bottom = (HeapWord*)round_to((intptr_t) bottom(), page_size());
575 rounded_end = (HeapWord*)round_down((intptr_t) end(), page_size());
576 base_space_size_pages = pointer_delta(rounded_end, rounded_bottom, sizeof(char)) / page_size();
577 }
578 guarantee(base_space_size_pages / lgrp_spaces()->length() > 0, "Space too small");
579 set_base_space_size(base_space_size_pages);
580
581 // Handle space resize
582 MemRegion top_region, bottom_region;
583 if (!old_region.equals(region())) {
584 new_region = MemRegion(rounded_bottom, rounded_end);
585 MemRegion intersection = new_region.intersection(old_region);
586 if (intersection.start() == NULL ||
587 intersection.end() == NULL ||
588 prev_page_size > page_size()) { // If the page size got smaller we have to change
589 // the page size preference for the whole space.
590 intersection = MemRegion(new_region.start(), new_region.start());
591 }
592 select_tails(new_region, intersection, &bottom_region, &top_region);
593 bias_region(bottom_region, lgrp_spaces()->at(0)->lgrp_id());
594 bias_region(top_region, lgrp_spaces()->at(lgrp_spaces()->length() - 1)->lgrp_id());
595 }
596
597 // Check if the space layout has changed significantly?
598 // This happens when the space has been resized so that either head or tail
599 // chunk became less than a page.
600 bool layout_valid = UseAdaptiveNUMAChunkSizing &&
601 current_chunk_size(0) > page_size() &&
602 current_chunk_size(lgrp_spaces()->length() - 1) > page_size();
603
604
605 for (int i = 0; i < lgrp_spaces()->length(); i++) {
606 LGRPSpace *ls = lgrp_spaces()->at(i);
607 MutableSpace *s = ls->space();
608 old_region = s->region();
609
610 size_t chunk_byte_size = 0, old_chunk_byte_size = 0;
611 if (i < lgrp_spaces()->length() - 1) {
612 if (!UseAdaptiveNUMAChunkSizing ||
613 (UseAdaptiveNUMAChunkSizing && NUMAChunkResizeWeight == 0) ||
614 samples_count() < AdaptiveSizePolicyReadyThreshold) {
615 // No adaptation. Divide the space equally.
616 chunk_byte_size = default_chunk_size();
617 } else
618 if (!layout_valid || NUMASpaceResizeRate == 0) {
619 // Fast adaptation. If no space resize rate is set, resize
620 // the chunks instantly.
621 chunk_byte_size = adaptive_chunk_size(i, 0);
622 } else {
623 // Slow adaptation. Resize the chunks moving no more than
624 // NUMASpaceResizeRate bytes per collection.
625 size_t limit = NUMASpaceResizeRate /
626 (lgrp_spaces()->length() * (lgrp_spaces()->length() + 1) / 2);
627 chunk_byte_size = adaptive_chunk_size(i, MAX2(limit * (i + 1), page_size()));
628 }
629
630 assert(chunk_byte_size >= page_size(), "Chunk size too small");
631 assert(chunk_byte_size <= capacity_in_bytes(), "Sanity check");
632 }
633
634 if (i == 0) { // Bottom chunk
635 if (i != lgrp_spaces()->length() - 1) {
636 new_region = MemRegion(bottom(), rounded_bottom + (chunk_byte_size >> LogHeapWordSize));
637 } else {
638 new_region = MemRegion(bottom(), end());
639 }
640 } else
641 if (i < lgrp_spaces()->length() - 1) { // Middle chunks
642 MutableSpace *ps = lgrp_spaces()->at(i - 1)->space();
643 new_region = MemRegion(ps->end(),
644 ps->end() + (chunk_byte_size >> LogHeapWordSize));
645 } else { // Top chunk
646 MutableSpace *ps = lgrp_spaces()->at(i - 1)->space();
647 new_region = MemRegion(ps->end(), end());
648 }
649 guarantee(region().contains(new_region), "Region invariant");
650
651
652 // The general case:
653 // |---------------------|--invalid---|--------------------------|
654 // |------------------new_region---------------------------------|
655 // |----bottom_region--|---intersection---|------top_region------|
656 // |----old_region----|
657 // The intersection part has all pages in place we don't need to migrate them.
658 // Pages for the top and bottom part should be freed and then reallocated.
659
660 MemRegion intersection = old_region.intersection(new_region);
661
662 if (intersection.start() == NULL || intersection.end() == NULL) {
663 intersection = MemRegion(new_region.start(), new_region.start());
664 }
665
666 if (!os::numa_has_static_binding()) {
667 MemRegion invalid_region = ls->invalid_region().intersection(new_region);
668 // Invalid region is a range of memory that could've possibly
669 // been allocated on the other node. That's relevant only on Solaris where
670 // there is no static memory binding.
671 if (!invalid_region.is_empty()) {
672 merge_regions(new_region, &intersection, &invalid_region);
673 free_region(invalid_region);
674 ls->set_invalid_region(MemRegion());
675 }
676 }
677
678 select_tails(new_region, intersection, &bottom_region, &top_region);
679
680 if (!os::numa_has_static_binding()) {
681 // If that's a system with the first-touch policy then it's enough
682 // to free the pages.
683 free_region(bottom_region);
684 free_region(top_region);
685 } else {
686 // In a system with static binding we have to change the bias whenever
687 // we reshape the heap.
688 bias_region(bottom_region, ls->lgrp_id());
689 bias_region(top_region, ls->lgrp_id());
690 }
691
692 // Clear space (set top = bottom) but never mangle.
693 s->initialize(new_region, SpaceDecorator::Clear, SpaceDecorator::DontMangle, MutableSpace::DontSetupPages);
694
695 set_adaptation_cycles(samples_count());
696 }
697 }
698
699 // Set the top of the whole space.
700 // Mark the the holes in chunks below the top() as invalid.
701 void MutableNUMASpace::set_top(HeapWord* value) {
702 bool found_top = false;
703 for (int i = 0; i < lgrp_spaces()->length();) {
704 LGRPSpace *ls = lgrp_spaces()->at(i);
705 MutableSpace *s = ls->space();
706 HeapWord *top = MAX2((HeapWord*)round_down((intptr_t)s->top(), page_size()), s->bottom());
707
708 if (s->contains(value)) {
709 // Check if setting the chunk's top to a given value would create a hole less than
710 // a minimal object; assuming that's not the last chunk in which case we don't care.
711 if (i < lgrp_spaces()->length() - 1) {
712 size_t remainder = pointer_delta(s->end(), value);
713 const size_t min_fill_size = CollectedHeap::min_fill_size();
714 if (remainder < min_fill_size && remainder > 0) {
715 // Add a minimum size filler object; it will cross the chunk boundary.
716 CollectedHeap::fill_with_object(value, min_fill_size);
717 value += min_fill_size;
718 assert(!s->contains(value), "Should be in the next chunk");
719 // Restart the loop from the same chunk, since the value has moved
720 // to the next one.
721 continue;
722 }
723 }
724
725 if (!os::numa_has_static_binding() && top < value && top < s->end()) {
726 ls->add_invalid_region(MemRegion(top, value));
727 }
728 s->set_top(value);
729 found_top = true;
730 } else {
731 if (found_top) {
732 s->set_top(s->bottom());
733 } else {
734 if (!os::numa_has_static_binding() && top < s->end()) {
735 ls->add_invalid_region(MemRegion(top, s->end()));
736 }
737 s->set_top(s->end());
738 }
739 }
740 i++;
741 }
742 MutableSpace::set_top(value);
743 }
744
745 void MutableNUMASpace::clear(bool mangle_space) {
746 MutableSpace::set_top(bottom());
747 for (int i = 0; i < lgrp_spaces()->length(); i++) {
748 // Never mangle NUMA spaces because the mangling will
749 // bind the memory to a possibly unwanted lgroup.
750 lgrp_spaces()->at(i)->space()->clear(SpaceDecorator::DontMangle);
751 }
752 }
753
754 /*
755 Linux supports static memory binding, therefore the most part of the
756 logic dealing with the possible invalid page allocation is effectively
757 disabled. Besides there is no notion of the home node in Linux. A
758 thread is allowed to migrate freely. Although the scheduler is rather
759 reluctant to move threads between the nodes. We check for the current
760 node every allocation. And with a high probability a thread stays on
761 the same node for some time allowing local access to recently allocated
762 objects.
763 */
764
765 HeapWord* MutableNUMASpace::allocate(size_t size) {
766 Thread* thr = Thread::current();
767 int lgrp_id = thr->lgrp_id();
768 if (lgrp_id == -1 || !os::numa_has_group_homing()) {
769 lgrp_id = os::numa_get_group_id();
770 thr->set_lgrp_id(lgrp_id);
771 }
772
773 int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals);
774
775 // It is possible that a new CPU has been hotplugged and
776 // we haven't reshaped the space accordingly.
777 if (i == -1) {
778 i = os::random() % lgrp_spaces()->length();
779 }
780
781 LGRPSpace* ls = lgrp_spaces()->at(i);
782 MutableSpace *s = ls->space();
783 HeapWord *p = s->allocate(size);
784
785 if (p != NULL) {
786 size_t remainder = s->free_in_words();
787 if (remainder < CollectedHeap::min_fill_size() && remainder > 0) {
788 s->set_top(s->top() - size);
789 p = NULL;
790 }
791 }
792 if (p != NULL) {
793 if (top() < s->top()) { // Keep _top updated.
794 MutableSpace::set_top(s->top());
795 }
796 }
797 // Make the page allocation happen here if there is no static binding..
798 if (p != NULL && !os::numa_has_static_binding()) {
799 for (HeapWord *i = p; i < p + size; i += os::vm_page_size() >> LogHeapWordSize) {
800 *(int*)i = 0;
801 }
802 }
803 if (p == NULL) {
804 ls->set_allocation_failed();
805 }
806 return p;
807 }
808
809 // This version is lock-free.
810 HeapWord* MutableNUMASpace::cas_allocate(size_t size) {
811 Thread* thr = Thread::current();
812 int lgrp_id = thr->lgrp_id();
813 if (lgrp_id == -1 || !os::numa_has_group_homing()) {
814 lgrp_id = os::numa_get_group_id();
815 thr->set_lgrp_id(lgrp_id);
816 }
817
818 int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals);
819 // It is possible that a new CPU has been hotplugged and
820 // we haven't reshaped the space accordingly.
821 if (i == -1) {
822 i = os::random() % lgrp_spaces()->length();
823 }
824 LGRPSpace *ls = lgrp_spaces()->at(i);
825 MutableSpace *s = ls->space();
826 HeapWord *p = s->cas_allocate(size);
827 if (p != NULL) {
828 size_t remainder = pointer_delta(s->end(), p + size);
829 if (remainder < CollectedHeap::min_fill_size() && remainder > 0) {
830 if (s->cas_deallocate(p, size)) {
831 // We were the last to allocate and created a fragment less than
832 // a minimal object.
833 p = NULL;
834 } else {
835 guarantee(false, "Deallocation should always succeed");
836 }
837 }
838 }
839 if (p != NULL) {
840 HeapWord* cur_top, *cur_chunk_top = p + size;
841 while ((cur_top = top()) < cur_chunk_top) { // Keep _top updated.
842 if (Atomic::cmpxchg_ptr(cur_chunk_top, top_addr(), cur_top) == cur_top) {
843 break;
844 }
845 }
846 }
847
848 // Make the page allocation happen here if there is no static binding.
849 if (p != NULL && !os::numa_has_static_binding() ) {
850 for (HeapWord *i = p; i < p + size; i += os::vm_page_size() >> LogHeapWordSize) {
851 *(int*)i = 0;
852 }
853 }
854 if (p == NULL) {
855 ls->set_allocation_failed();
856 }
857 return p;
858 }
859
860 void MutableNUMASpace::print_short_on(outputStream* st) const {
861 MutableSpace::print_short_on(st);
862 st->print(" (");
863 for (int i = 0; i < lgrp_spaces()->length(); i++) {
864 st->print("lgrp %d: ", lgrp_spaces()->at(i)->lgrp_id());
865 lgrp_spaces()->at(i)->space()->print_short_on(st);
866 if (i < lgrp_spaces()->length() - 1) {
867 st->print(", ");
868 }
869 }
870 st->print(")");
871 }
872
873 void MutableNUMASpace::print_on(outputStream* st) const {
874 MutableSpace::print_on(st);
875 for (int i = 0; i < lgrp_spaces()->length(); i++) {
876 LGRPSpace *ls = lgrp_spaces()->at(i);
877 st->print(" lgrp %d", ls->lgrp_id());
878 ls->space()->print_on(st);
879 if (NUMAStats) {
880 for (int i = 0; i < lgrp_spaces()->length(); i++) {
881 lgrp_spaces()->at(i)->accumulate_statistics(page_size());
882 }
883 st->print(" local/remote/unbiased/uncommitted: %dK/%dK/%dK/%dK, large/small pages: %d/%d\n",
884 ls->space_stats()->_local_space / K,
885 ls->space_stats()->_remote_space / K,
886 ls->space_stats()->_unbiased_space / K,
887 ls->space_stats()->_uncommited_space / K,
888 ls->space_stats()->_large_pages,
889 ls->space_stats()->_small_pages);
890 }
891 }
892 }
893
894 void MutableNUMASpace::verify() {
895 // This can be called after setting an arbitary value to the space's top,
896 // so an object can cross the chunk boundary. We ensure the parsablity
897 // of the space and just walk the objects in linear fashion.
898 ensure_parsability();
899 MutableSpace::verify();
900 }
901
902 // Scan pages and gather stats about page placement and size.
903 void MutableNUMASpace::LGRPSpace::accumulate_statistics(size_t page_size) {
904 clear_space_stats();
905 char *start = (char*)round_to((intptr_t) space()->bottom(), page_size);
906 char* end = (char*)round_down((intptr_t) space()->end(), page_size);
907 if (start < end) {
908 for (char *p = start; p < end;) {
909 os::page_info info;
910 if (os::get_page_info(p, &info)) {
911 if (info.size > 0) {
912 if (info.size > (size_t)os::vm_page_size()) {
913 space_stats()->_large_pages++;
914 } else {
915 space_stats()->_small_pages++;
916 }
917 if (info.lgrp_id == lgrp_id()) {
918 space_stats()->_local_space += info.size;
919 } else {
920 space_stats()->_remote_space += info.size;
921 }
922 p += info.size;
923 } else {
924 p += os::vm_page_size();
925 space_stats()->_uncommited_space += os::vm_page_size();
926 }
927 } else {
928 return;
929 }
930 }
931 }
932 space_stats()->_unbiased_space = pointer_delta(start, space()->bottom(), sizeof(char)) +
933 pointer_delta(space()->end(), end, sizeof(char));
934
935 }
936
937 // Scan page_count pages and verify if they have the right size and right placement.
938 // If invalid pages are found they are freed in hope that subsequent reallocation
939 // will be more successful.
940 void MutableNUMASpace::LGRPSpace::scan_pages(size_t page_size, size_t page_count)
941 {
942 char* range_start = (char*)round_to((intptr_t) space()->bottom(), page_size);
943 char* range_end = (char*)round_down((intptr_t) space()->end(), page_size);
944
945 if (range_start > last_page_scanned() || last_page_scanned() >= range_end) {
946 set_last_page_scanned(range_start);
947 }
948
949 char *scan_start = last_page_scanned();
950 char* scan_end = MIN2(scan_start + page_size * page_count, range_end);
951
952 os::page_info page_expected, page_found;
953 page_expected.size = page_size;
954 page_expected.lgrp_id = lgrp_id();
955
956 char *s = scan_start;
957 while (s < scan_end) {
958 char *e = os::scan_pages(s, (char*)scan_end, &page_expected, &page_found);
959 if (e == NULL) {
960 break;
961 }
962 if (e != scan_end) {
963 if ((page_expected.size != page_size || page_expected.lgrp_id != lgrp_id())
964 && page_expected.size != 0) {
965 os::free_memory(s, pointer_delta(e, s, sizeof(char)), page_size);
966 }
967 page_expected = page_found;
968 }
969 s = e;
970 }
971
972 set_last_page_scanned(scan_end);
973 }