1
2 /*
3 * Copyright (c) 2006, 2014, Oracle and/or its affiliates. All rights reserved.
4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 *
6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "precompiled.hpp"
27 #include "gc_implementation/shared/mutableNUMASpace.hpp"
28 #include "gc_implementation/shared/spaceDecorator.hpp"
29 #include "memory/sharedHeap.hpp"
30 #include "oops/oop.inline.hpp"
31 #include "runtime/atomic.inline.hpp"
32 #include "runtime/thread.inline.hpp"
33
34 PRAGMA_FORMAT_MUTE_WARNINGS_FOR_GCC
35
36 MutableNUMASpace::MutableNUMASpace(size_t alignment) : MutableSpace(alignment) {
37 _lgrp_spaces = new (ResourceObj::C_HEAP, mtGC) GrowableArray<LGRPSpace*>(0, true);
38 _page_size = os::vm_page_size();
39 _adaptation_cycles = 0;
40 _samples_count = 0;
41 update_layout(true);
42 }
43
44 MutableNUMASpace::~MutableNUMASpace() {
45 for (int i = 0; i < lgrp_spaces()->length(); i++) {
46 delete lgrp_spaces()->at(i);
47 }
48 delete lgrp_spaces();
49 }
50
51 #ifndef PRODUCT
52 void MutableNUMASpace::mangle_unused_area() {
53 // This method should do nothing.
54 // It can be called on a numa space during a full compaction.
55 }
56 void MutableNUMASpace::mangle_unused_area_complete() {
57 // This method should do nothing.
58 // It can be called on a numa space during a full compaction.
59 }
60 void MutableNUMASpace::mangle_region(MemRegion mr) {
61 // This method should do nothing because numa spaces are not mangled.
62 }
63 void MutableNUMASpace::set_top_for_allocations(HeapWord* v) {
64 assert(false, "Do not mangle MutableNUMASpace's");
65 }
66 void MutableNUMASpace::set_top_for_allocations() {
67 // This method should do nothing.
68 }
69 void MutableNUMASpace::check_mangled_unused_area(HeapWord* limit) {
70 // This method should do nothing.
71 }
72 void MutableNUMASpace::check_mangled_unused_area_complete() {
73 // This method should do nothing.
74 }
75 #endif // NOT_PRODUCT
76
77 // There may be unallocated holes in the middle chunks
78 // that should be filled with dead objects to ensure parsability.
79 void MutableNUMASpace::ensure_parsability() {
80 for (int i = 0; i < lgrp_spaces()->length(); i++) {
81 LGRPSpace *ls = lgrp_spaces()->at(i);
82 MutableSpace *s = ls->space();
83 if (s->top() < top()) { // For all spaces preceding the one containing top()
84 if (s->free_in_words() > 0) {
85 intptr_t cur_top = (intptr_t)s->top();
86 size_t words_left_to_fill = pointer_delta(s->end(), s->top());;
87 while (words_left_to_fill > 0) {
88 size_t words_to_fill = MIN2(words_left_to_fill, CollectedHeap::filler_array_max_size());
89 assert(words_to_fill >= CollectedHeap::min_fill_size(),
90 err_msg("Remaining size ("SIZE_FORMAT ") is too small to fill (based on " SIZE_FORMAT " and " SIZE_FORMAT ")",
91 words_to_fill, words_left_to_fill, CollectedHeap::filler_array_max_size()));
92 CollectedHeap::fill_with_object((HeapWord*)cur_top, words_to_fill);
93 if (!os::numa_has_static_binding()) {
94 size_t touched_words = words_to_fill;
95 #ifndef ASSERT
96 if (!ZapUnusedHeapArea) {
97 touched_words = MIN2((size_t)align_object_size(typeArrayOopDesc::header_size(T_INT)),
98 touched_words);
99 }
100 #endif
101 MemRegion invalid;
102 HeapWord *crossing_start = (HeapWord*)round_to(cur_top, os::vm_page_size());
103 HeapWord *crossing_end = (HeapWord*)round_to(cur_top + touched_words, os::vm_page_size());
104 if (crossing_start != crossing_end) {
105 // If object header crossed a small page boundary we mark the area
106 // as invalid rounding it to a page_size().
107 HeapWord *start = MAX2((HeapWord*)round_down(cur_top, page_size()), s->bottom());
108 HeapWord *end = MIN2((HeapWord*)round_to(cur_top + touched_words, page_size()), s->end());
109 invalid = MemRegion(start, end);
110 }
111
112 ls->add_invalid_region(invalid);
113 }
114 cur_top = cur_top + (words_to_fill * HeapWordSize);
115 words_left_to_fill -= words_to_fill;
116 }
117 }
118 } else {
119 if (!os::numa_has_static_binding()) {
120 #ifdef ASSERT
121 MemRegion invalid(s->top(), s->end());
122 ls->add_invalid_region(invalid);
123 #else
124 if (ZapUnusedHeapArea) {
125 MemRegion invalid(s->top(), s->end());
126 ls->add_invalid_region(invalid);
127 } else {
128 return;
129 }
130 #endif
131 } else {
132 return;
133 }
134 }
135 }
136 }
137
138 size_t MutableNUMASpace::used_in_words() const {
139 size_t s = 0;
140 for (int i = 0; i < lgrp_spaces()->length(); i++) {
141 s += lgrp_spaces()->at(i)->space()->used_in_words();
142 }
143 return s;
144 }
145
146 size_t MutableNUMASpace::free_in_words() const {
147 size_t s = 0;
148 for (int i = 0; i < lgrp_spaces()->length(); i++) {
149 s += lgrp_spaces()->at(i)->space()->free_in_words();
150 }
151 return s;
152 }
153
154
155 size_t MutableNUMASpace::tlab_capacity(Thread *thr) const {
156 guarantee(thr != NULL, "No thread");
157 int lgrp_id = thr->lgrp_id();
158 if (lgrp_id == -1) {
159 // This case can occur after the topology of the system has
160 // changed. Thread can change their location, the new home
161 // group will be determined during the first allocation
162 // attempt. For now we can safely assume that all spaces
163 // have equal size because the whole space will be reinitialized.
164 if (lgrp_spaces()->length() > 0) {
165 return capacity_in_bytes() / lgrp_spaces()->length();
166 } else {
167 assert(false, "There should be at least one locality group");
168 return 0;
169 }
170 }
171 // That's the normal case, where we know the locality group of the thread.
172 int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals);
173 if (i == -1) {
174 return 0;
175 }
176 return lgrp_spaces()->at(i)->space()->capacity_in_bytes();
177 }
178
179 size_t MutableNUMASpace::tlab_used(Thread *thr) const {
180 // Please see the comments for tlab_capacity().
181 guarantee(thr != NULL, "No thread");
182 int lgrp_id = thr->lgrp_id();
183 if (lgrp_id == -1) {
184 if (lgrp_spaces()->length() > 0) {
185 return (used_in_bytes()) / lgrp_spaces()->length();
186 } else {
187 assert(false, "There should be at least one locality group");
188 return 0;
189 }
190 }
191 int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals);
192 if (i == -1) {
193 return 0;
194 }
195 return lgrp_spaces()->at(i)->space()->used_in_bytes();
196 }
197
198
199 size_t MutableNUMASpace::unsafe_max_tlab_alloc(Thread *thr) const {
200 // Please see the comments for tlab_capacity().
201 guarantee(thr != NULL, "No thread");
202 int lgrp_id = thr->lgrp_id();
203 if (lgrp_id == -1) {
204 if (lgrp_spaces()->length() > 0) {
205 return free_in_bytes() / lgrp_spaces()->length();
206 } else {
207 assert(false, "There should be at least one locality group");
208 return 0;
209 }
210 }
211 int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals);
212 if (i == -1) {
213 return 0;
214 }
215 return lgrp_spaces()->at(i)->space()->free_in_bytes();
216 }
217
218
219 size_t MutableNUMASpace::capacity_in_words(Thread* thr) const {
220 guarantee(thr != NULL, "No thread");
221 int lgrp_id = thr->lgrp_id();
222 if (lgrp_id == -1) {
223 if (lgrp_spaces()->length() > 0) {
224 return capacity_in_words() / lgrp_spaces()->length();
225 } else {
226 assert(false, "There should be at least one locality group");
227 return 0;
228 }
229 }
230 int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals);
231 if (i == -1) {
232 return 0;
233 }
234 return lgrp_spaces()->at(i)->space()->capacity_in_words();
235 }
236
237 // Check if the NUMA topology has changed. Add and remove spaces if needed.
238 // The update can be forced by setting the force parameter equal to true.
239 bool MutableNUMASpace::update_layout(bool force) {
240 // Check if the topology had changed.
241 bool changed = os::numa_topology_changed();
242 if (force || changed) {
243 // Compute lgrp intersection. Add/remove spaces.
244 int lgrp_limit = (int)os::numa_get_groups_num();
245 int *lgrp_ids = NEW_C_HEAP_ARRAY(int, lgrp_limit, mtGC);
246 int lgrp_num = (int)os::numa_get_leaf_groups(lgrp_ids, lgrp_limit);
247 assert(lgrp_num > 0, "There should be at least one locality group");
248 // Add new spaces for the new nodes
249 for (int i = 0; i < lgrp_num; i++) {
250 bool found = false;
251 for (int j = 0; j < lgrp_spaces()->length(); j++) {
252 if (lgrp_spaces()->at(j)->lgrp_id() == lgrp_ids[i]) {
253 found = true;
254 break;
255 }
256 }
257 if (!found) {
258 lgrp_spaces()->append(new LGRPSpace(lgrp_ids[i], alignment()));
259 }
260 }
261
262 // Remove spaces for the removed nodes.
263 for (int i = 0; i < lgrp_spaces()->length();) {
264 bool found = false;
265 for (int j = 0; j < lgrp_num; j++) {
266 if (lgrp_spaces()->at(i)->lgrp_id() == lgrp_ids[j]) {
267 found = true;
268 break;
269 }
270 }
271 if (!found) {
272 delete lgrp_spaces()->at(i);
273 lgrp_spaces()->remove_at(i);
274 } else {
275 i++;
276 }
277 }
278
279 FREE_C_HEAP_ARRAY(int, lgrp_ids, mtGC);
280
281 if (changed) {
282 for (JavaThread *thread = Threads::first(); thread; thread = thread->next()) {
283 thread->set_lgrp_id(-1);
284 }
285 }
286 return true;
287 }
288 return false;
289 }
290
291 // Bias region towards the first-touching lgrp. Set the right page sizes.
292 void MutableNUMASpace::bias_region(MemRegion mr, int lgrp_id) {
293 HeapWord *start = (HeapWord*)round_to((intptr_t)mr.start(), page_size());
294 HeapWord *end = (HeapWord*)round_down((intptr_t)mr.end(), page_size());
295 if (end > start) {
296 MemRegion aligned_region(start, end);
297 assert((intptr_t)aligned_region.start() % page_size() == 0 &&
298 (intptr_t)aligned_region.byte_size() % page_size() == 0, "Bad alignment");
299 assert(region().contains(aligned_region), "Sanity");
300 // First we tell the OS which page size we want in the given range. The underlying
301 // large page can be broken down if we require small pages.
302 os::realign_memory((char*)aligned_region.start(), aligned_region.byte_size(), page_size());
303 // Then we uncommit the pages in the range.
304 os::free_memory((char*)aligned_region.start(), aligned_region.byte_size(), page_size());
305 // And make them local/first-touch biased.
306 os::numa_make_local((char*)aligned_region.start(), aligned_region.byte_size(), lgrp_id);
307 }
308 }
309
310 // Free all pages in the region.
311 void MutableNUMASpace::free_region(MemRegion mr) {
312 HeapWord *start = (HeapWord*)round_to((intptr_t)mr.start(), page_size());
313 HeapWord *end = (HeapWord*)round_down((intptr_t)mr.end(), page_size());
314 if (end > start) {
315 MemRegion aligned_region(start, end);
316 assert((intptr_t)aligned_region.start() % page_size() == 0 &&
317 (intptr_t)aligned_region.byte_size() % page_size() == 0, "Bad alignment");
318 assert(region().contains(aligned_region), "Sanity");
319 os::free_memory((char*)aligned_region.start(), aligned_region.byte_size(), page_size());
320 }
321 }
322
323 // Update space layout. Perform adaptation.
324 void MutableNUMASpace::update() {
325 if (update_layout(false)) {
326 // If the topology has changed, make all chunks zero-sized.
327 // And clear the alloc-rate statistics.
328 // In future we may want to handle this more gracefully in order
329 // to avoid the reallocation of the pages as much as possible.
330 for (int i = 0; i < lgrp_spaces()->length(); i++) {
331 LGRPSpace *ls = lgrp_spaces()->at(i);
332 MutableSpace *s = ls->space();
333 s->set_end(s->bottom());
334 s->set_top(s->bottom());
335 ls->clear_alloc_rate();
336 }
337 // A NUMA space is never mangled
338 initialize(region(),
339 SpaceDecorator::Clear,
340 SpaceDecorator::DontMangle);
341 } else {
342 bool should_initialize = false;
343 if (!os::numa_has_static_binding()) {
344 for (int i = 0; i < lgrp_spaces()->length(); i++) {
345 if (!lgrp_spaces()->at(i)->invalid_region().is_empty()) {
346 should_initialize = true;
347 break;
348 }
349 }
350 }
351
352 if (should_initialize ||
353 (UseAdaptiveNUMAChunkSizing && adaptation_cycles() < samples_count())) {
354 // A NUMA space is never mangled
355 initialize(region(),
356 SpaceDecorator::Clear,
357 SpaceDecorator::DontMangle);
358 }
359 }
360
361 if (NUMAStats) {
362 for (int i = 0; i < lgrp_spaces()->length(); i++) {
363 lgrp_spaces()->at(i)->accumulate_statistics(page_size());
364 }
365 }
366
367 scan_pages(NUMAPageScanRate);
368 }
369
370 // Scan pages. Free pages that have smaller size or wrong placement.
371 void MutableNUMASpace::scan_pages(size_t page_count)
372 {
373 size_t pages_per_chunk = page_count / lgrp_spaces()->length();
374 if (pages_per_chunk > 0) {
375 for (int i = 0; i < lgrp_spaces()->length(); i++) {
376 LGRPSpace *ls = lgrp_spaces()->at(i);
377 ls->scan_pages(page_size(), pages_per_chunk);
378 }
379 }
380 }
381
382 // Accumulate statistics about the allocation rate of each lgrp.
383 void MutableNUMASpace::accumulate_statistics() {
384 if (UseAdaptiveNUMAChunkSizing) {
385 for (int i = 0; i < lgrp_spaces()->length(); i++) {
386 lgrp_spaces()->at(i)->sample();
387 }
388 increment_samples_count();
389 }
390
391 if (NUMAStats) {
392 for (int i = 0; i < lgrp_spaces()->length(); i++) {
393 lgrp_spaces()->at(i)->accumulate_statistics(page_size());
394 }
395 }
396 }
397
398 // Get the current size of a chunk.
399 // This function computes the size of the chunk based on the
400 // difference between chunk ends. This allows it to work correctly in
401 // case the whole space is resized and during the process of adaptive
402 // chunk resizing.
403 size_t MutableNUMASpace::current_chunk_size(int i) {
404 HeapWord *cur_end, *prev_end;
405 if (i == 0) {
406 prev_end = bottom();
407 } else {
408 prev_end = lgrp_spaces()->at(i - 1)->space()->end();
409 }
410 if (i == lgrp_spaces()->length() - 1) {
411 cur_end = end();
412 } else {
413 cur_end = lgrp_spaces()->at(i)->space()->end();
414 }
415 if (cur_end > prev_end) {
416 return pointer_delta(cur_end, prev_end, sizeof(char));
417 }
418 return 0;
419 }
420
421 // Return the default chunk size by equally diving the space.
422 // page_size() aligned.
423 size_t MutableNUMASpace::default_chunk_size() {
424 return base_space_size() / lgrp_spaces()->length() * page_size();
425 }
426
427 // Produce a new chunk size. page_size() aligned.
428 // This function is expected to be called on sequence of i's from 0 to
429 // lgrp_spaces()->length().
430 size_t MutableNUMASpace::adaptive_chunk_size(int i, size_t limit) {
431 size_t pages_available = base_space_size();
432 for (int j = 0; j < i; j++) {
433 pages_available -= round_down(current_chunk_size(j), page_size()) / page_size();
434 }
435 pages_available -= lgrp_spaces()->length() - i - 1;
436 assert(pages_available > 0, "No pages left");
437 float alloc_rate = 0;
438 for (int j = i; j < lgrp_spaces()->length(); j++) {
439 alloc_rate += lgrp_spaces()->at(j)->alloc_rate()->average();
440 }
441 size_t chunk_size = 0;
442 if (alloc_rate > 0) {
443 LGRPSpace *ls = lgrp_spaces()->at(i);
444 chunk_size = (size_t)(ls->alloc_rate()->average() / alloc_rate * pages_available) * page_size();
445 }
446 chunk_size = MAX2(chunk_size, page_size());
447
448 if (limit > 0) {
449 limit = round_down(limit, page_size());
450 if (chunk_size > current_chunk_size(i)) {
451 size_t upper_bound = pages_available * page_size();
452 if (upper_bound > limit &&
453 current_chunk_size(i) < upper_bound - limit) {
454 // The resulting upper bound should not exceed the available
455 // amount of memory (pages_available * page_size()).
456 upper_bound = current_chunk_size(i) + limit;
457 }
458 chunk_size = MIN2(chunk_size, upper_bound);
459 } else {
460 size_t lower_bound = page_size();
461 if (current_chunk_size(i) > limit) { // lower_bound shouldn't underflow.
462 lower_bound = current_chunk_size(i) - limit;
463 }
464 chunk_size = MAX2(chunk_size, lower_bound);
465 }
466 }
467 assert(chunk_size <= pages_available * page_size(), "Chunk size out of range");
468 return chunk_size;
469 }
470
471
472 // Return the bottom_region and the top_region. Align them to page_size() boundary.
473 // |------------------new_region---------------------------------|
474 // |----bottom_region--|---intersection---|------top_region------|
475 void MutableNUMASpace::select_tails(MemRegion new_region, MemRegion intersection,
476 MemRegion* bottom_region, MemRegion *top_region) {
477 // Is there bottom?
478 if (new_region.start() < intersection.start()) { // Yes
479 // Try to coalesce small pages into a large one.
480 if (UseLargePages && page_size() >= alignment()) {
481 HeapWord* p = (HeapWord*)round_to((intptr_t) intersection.start(), alignment());
482 if (new_region.contains(p)
483 && pointer_delta(p, new_region.start(), sizeof(char)) >= alignment()) {
484 if (intersection.contains(p)) {
485 intersection = MemRegion(p, intersection.end());
486 } else {
487 intersection = MemRegion(p, p);
488 }
489 }
490 }
491 *bottom_region = MemRegion(new_region.start(), intersection.start());
492 } else {
493 *bottom_region = MemRegion();
494 }
495
496 // Is there top?
497 if (intersection.end() < new_region.end()) { // Yes
498 // Try to coalesce small pages into a large one.
499 if (UseLargePages && page_size() >= alignment()) {
500 HeapWord* p = (HeapWord*)round_down((intptr_t) intersection.end(), alignment());
501 if (new_region.contains(p)
502 && pointer_delta(new_region.end(), p, sizeof(char)) >= alignment()) {
503 if (intersection.contains(p)) {
504 intersection = MemRegion(intersection.start(), p);
505 } else {
506 intersection = MemRegion(p, p);
507 }
508 }
509 }
510 *top_region = MemRegion(intersection.end(), new_region.end());
511 } else {
512 *top_region = MemRegion();
513 }
514 }
515
516 // Try to merge the invalid region with the bottom or top region by decreasing
517 // the intersection area. Return the invalid_region aligned to the page_size()
518 // boundary if it's inside the intersection. Return non-empty invalid_region
519 // if it lies inside the intersection (also page-aligned).
520 // |------------------new_region---------------------------------|
521 // |----------------|-------invalid---|--------------------------|
522 // |----bottom_region--|---intersection---|------top_region------|
523 void MutableNUMASpace::merge_regions(MemRegion new_region, MemRegion* intersection,
524 MemRegion *invalid_region) {
525 if (intersection->start() >= invalid_region->start() && intersection->contains(invalid_region->end())) {
526 *intersection = MemRegion(invalid_region->end(), intersection->end());
527 *invalid_region = MemRegion();
528 } else
529 if (intersection->end() <= invalid_region->end() && intersection->contains(invalid_region->start())) {
530 *intersection = MemRegion(intersection->start(), invalid_region->start());
531 *invalid_region = MemRegion();
532 } else
533 if (intersection->equals(*invalid_region) || invalid_region->contains(*intersection)) {
534 *intersection = MemRegion(new_region.start(), new_region.start());
535 *invalid_region = MemRegion();
536 } else
537 if (intersection->contains(invalid_region)) {
538 // That's the only case we have to make an additional bias_region() call.
539 HeapWord* start = invalid_region->start();
540 HeapWord* end = invalid_region->end();
541 if (UseLargePages && page_size() >= alignment()) {
542 HeapWord *p = (HeapWord*)round_down((intptr_t) start, alignment());
543 if (new_region.contains(p)) {
544 start = p;
545 }
546 p = (HeapWord*)round_to((intptr_t) end, alignment());
547 if (new_region.contains(end)) {
548 end = p;
549 }
550 }
551 if (intersection->start() > start) {
552 *intersection = MemRegion(start, intersection->end());
553 }
554 if (intersection->end() < end) {
555 *intersection = MemRegion(intersection->start(), end);
556 }
557 *invalid_region = MemRegion(start, end);
558 }
559 }
560
561 void MutableNUMASpace::initialize(MemRegion mr,
562 bool clear_space,
563 bool mangle_space,
564 bool setup_pages) {
565 assert(clear_space, "Reallocation will destroy data!");
566 assert(lgrp_spaces()->length() > 0, "There should be at least one space");
567
568 MemRegion old_region = region(), new_region;
569 set_bottom(mr.start());
570 set_end(mr.end());
571 // Must always clear the space
572 clear(SpaceDecorator::DontMangle);
573
574 // Compute chunk sizes
575 size_t prev_page_size = page_size();
576 set_page_size(UseLargePages ? alignment() : os::vm_page_size());
577 HeapWord* rounded_bottom = (HeapWord*)round_to((intptr_t) bottom(), page_size());
578 HeapWord* rounded_end = (HeapWord*)round_down((intptr_t) end(), page_size());
579 size_t base_space_size_pages = pointer_delta(rounded_end, rounded_bottom, sizeof(char)) / page_size();
580
581 // Try small pages if the chunk size is too small
582 if (base_space_size_pages / lgrp_spaces()->length() == 0
583 && page_size() > (size_t)os::vm_page_size()) {
584 set_page_size(os::vm_page_size());
585 rounded_bottom = (HeapWord*)round_to((intptr_t) bottom(), page_size());
586 rounded_end = (HeapWord*)round_down((intptr_t) end(), page_size());
587 base_space_size_pages = pointer_delta(rounded_end, rounded_bottom, sizeof(char)) / page_size();
588 }
589 guarantee(base_space_size_pages / lgrp_spaces()->length() > 0, "Space too small");
590 set_base_space_size(base_space_size_pages);
591
592 // Handle space resize
593 MemRegion top_region, bottom_region;
594 if (!old_region.equals(region())) {
595 new_region = MemRegion(rounded_bottom, rounded_end);
596 MemRegion intersection = new_region.intersection(old_region);
597 if (intersection.start() == NULL ||
598 intersection.end() == NULL ||
599 prev_page_size > page_size()) { // If the page size got smaller we have to change
600 // the page size preference for the whole space.
601 intersection = MemRegion(new_region.start(), new_region.start());
602 }
603 select_tails(new_region, intersection, &bottom_region, &top_region);
604 bias_region(bottom_region, lgrp_spaces()->at(0)->lgrp_id());
605 bias_region(top_region, lgrp_spaces()->at(lgrp_spaces()->length() - 1)->lgrp_id());
606 }
607
608 // Check if the space layout has changed significantly?
609 // This happens when the space has been resized so that either head or tail
610 // chunk became less than a page.
611 bool layout_valid = UseAdaptiveNUMAChunkSizing &&
612 current_chunk_size(0) > page_size() &&
613 current_chunk_size(lgrp_spaces()->length() - 1) > page_size();
614
615
616 for (int i = 0; i < lgrp_spaces()->length(); i++) {
617 LGRPSpace *ls = lgrp_spaces()->at(i);
618 MutableSpace *s = ls->space();
619 old_region = s->region();
620
621 size_t chunk_byte_size = 0, old_chunk_byte_size = 0;
622 if (i < lgrp_spaces()->length() - 1) {
623 if (!UseAdaptiveNUMAChunkSizing ||
624 (UseAdaptiveNUMAChunkSizing && NUMAChunkResizeWeight == 0) ||
625 samples_count() < AdaptiveSizePolicyReadyThreshold) {
626 // No adaptation. Divide the space equally.
627 chunk_byte_size = default_chunk_size();
628 } else
629 if (!layout_valid || NUMASpaceResizeRate == 0) {
630 // Fast adaptation. If no space resize rate is set, resize
631 // the chunks instantly.
632 chunk_byte_size = adaptive_chunk_size(i, 0);
633 } else {
634 // Slow adaptation. Resize the chunks moving no more than
635 // NUMASpaceResizeRate bytes per collection.
636 size_t limit = NUMASpaceResizeRate /
637 (lgrp_spaces()->length() * (lgrp_spaces()->length() + 1) / 2);
638 chunk_byte_size = adaptive_chunk_size(i, MAX2(limit * (i + 1), page_size()));
639 }
640
641 assert(chunk_byte_size >= page_size(), "Chunk size too small");
642 assert(chunk_byte_size <= capacity_in_bytes(), "Sanity check");
643 }
644
645 if (i == 0) { // Bottom chunk
646 if (i != lgrp_spaces()->length() - 1) {
647 new_region = MemRegion(bottom(), rounded_bottom + (chunk_byte_size >> LogHeapWordSize));
648 } else {
649 new_region = MemRegion(bottom(), end());
650 }
651 } else
652 if (i < lgrp_spaces()->length() - 1) { // Middle chunks
653 MutableSpace *ps = lgrp_spaces()->at(i - 1)->space();
654 new_region = MemRegion(ps->end(),
655 ps->end() + (chunk_byte_size >> LogHeapWordSize));
656 } else { // Top chunk
657 MutableSpace *ps = lgrp_spaces()->at(i - 1)->space();
658 new_region = MemRegion(ps->end(), end());
659 }
660 guarantee(region().contains(new_region), "Region invariant");
661
662
663 // The general case:
664 // |---------------------|--invalid---|--------------------------|
665 // |------------------new_region---------------------------------|
666 // |----bottom_region--|---intersection---|------top_region------|
667 // |----old_region----|
668 // The intersection part has all pages in place we don't need to migrate them.
669 // Pages for the top and bottom part should be freed and then reallocated.
670
671 MemRegion intersection = old_region.intersection(new_region);
672
673 if (intersection.start() == NULL || intersection.end() == NULL) {
674 intersection = MemRegion(new_region.start(), new_region.start());
675 }
676
677 if (!os::numa_has_static_binding()) {
678 MemRegion invalid_region = ls->invalid_region().intersection(new_region);
679 // Invalid region is a range of memory that could've possibly
680 // been allocated on the other node. That's relevant only on Solaris where
681 // there is no static memory binding.
682 if (!invalid_region.is_empty()) {
683 merge_regions(new_region, &intersection, &invalid_region);
684 free_region(invalid_region);
685 ls->set_invalid_region(MemRegion());
686 }
687 }
688
689 select_tails(new_region, intersection, &bottom_region, &top_region);
690
691 if (!os::numa_has_static_binding()) {
692 // If that's a system with the first-touch policy then it's enough
693 // to free the pages.
694 free_region(bottom_region);
695 free_region(top_region);
696 } else {
697 // In a system with static binding we have to change the bias whenever
698 // we reshape the heap.
699 bias_region(bottom_region, ls->lgrp_id());
700 bias_region(top_region, ls->lgrp_id());
701 }
702
703 // Clear space (set top = bottom) but never mangle.
704 s->initialize(new_region, SpaceDecorator::Clear, SpaceDecorator::DontMangle, MutableSpace::DontSetupPages);
705
706 set_adaptation_cycles(samples_count());
707 }
708 }
709
710 // Set the top of the whole space.
711 // Mark the the holes in chunks below the top() as invalid.
712 void MutableNUMASpace::set_top(HeapWord* value) {
713 bool found_top = false;
714 for (int i = 0; i < lgrp_spaces()->length();) {
715 LGRPSpace *ls = lgrp_spaces()->at(i);
716 MutableSpace *s = ls->space();
717 HeapWord *top = MAX2((HeapWord*)round_down((intptr_t)s->top(), page_size()), s->bottom());
718
719 if (s->contains(value)) {
720 // Check if setting the chunk's top to a given value would create a hole less than
721 // a minimal object; assuming that's not the last chunk in which case we don't care.
722 if (i < lgrp_spaces()->length() - 1) {
723 size_t remainder = pointer_delta(s->end(), value);
724 const size_t min_fill_size = CollectedHeap::min_fill_size();
725 if (remainder < min_fill_size && remainder > 0) {
726 // Add a minimum size filler object; it will cross the chunk boundary.
727 CollectedHeap::fill_with_object(value, min_fill_size);
728 value += min_fill_size;
729 assert(!s->contains(value), "Should be in the next chunk");
730 // Restart the loop from the same chunk, since the value has moved
731 // to the next one.
732 continue;
733 }
734 }
735
736 if (!os::numa_has_static_binding() && top < value && top < s->end()) {
737 ls->add_invalid_region(MemRegion(top, value));
738 }
739 s->set_top(value);
740 found_top = true;
741 } else {
742 if (found_top) {
743 s->set_top(s->bottom());
744 } else {
745 if (!os::numa_has_static_binding() && top < s->end()) {
746 ls->add_invalid_region(MemRegion(top, s->end()));
747 }
748 s->set_top(s->end());
749 }
750 }
751 i++;
752 }
753 MutableSpace::set_top(value);
754 }
755
756 void MutableNUMASpace::clear(bool mangle_space) {
757 MutableSpace::set_top(bottom());
758 for (int i = 0; i < lgrp_spaces()->length(); i++) {
759 // Never mangle NUMA spaces because the mangling will
760 // bind the memory to a possibly unwanted lgroup.
761 lgrp_spaces()->at(i)->space()->clear(SpaceDecorator::DontMangle);
762 }
763 }
764
765 /*
766 Linux supports static memory binding, therefore the most part of the
767 logic dealing with the possible invalid page allocation is effectively
768 disabled. Besides there is no notion of the home node in Linux. A
769 thread is allowed to migrate freely. Although the scheduler is rather
770 reluctant to move threads between the nodes. We check for the current
771 node every allocation. And with a high probability a thread stays on
772 the same node for some time allowing local access to recently allocated
773 objects.
774 */
775
776 HeapWord* MutableNUMASpace::allocate(size_t size) {
777 Thread* thr = Thread::current();
778 int lgrp_id = thr->lgrp_id();
779 if (lgrp_id == -1 || !os::numa_has_group_homing()) {
780 lgrp_id = os::numa_get_group_id();
781 thr->set_lgrp_id(lgrp_id);
782 }
783
784 int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals);
785
786 // It is possible that a new CPU has been hotplugged and
787 // we haven't reshaped the space accordingly.
788 if (i == -1) {
789 i = os::random() % lgrp_spaces()->length();
790 }
791
792 LGRPSpace* ls = lgrp_spaces()->at(i);
793 MutableSpace *s = ls->space();
794 HeapWord *p = s->allocate(size);
795
796 if (p != NULL) {
797 size_t remainder = s->free_in_words();
798 if (remainder < CollectedHeap::min_fill_size() && remainder > 0) {
799 s->set_top(s->top() - size);
800 p = NULL;
801 }
802 }
803 if (p != NULL) {
804 if (top() < s->top()) { // Keep _top updated.
805 MutableSpace::set_top(s->top());
806 }
807 }
808 // Make the page allocation happen here if there is no static binding..
809 if (p != NULL && !os::numa_has_static_binding()) {
810 for (HeapWord *i = p; i < p + size; i += os::vm_page_size() >> LogHeapWordSize) {
811 *(int*)i = 0;
812 }
813 }
814 if (p == NULL) {
815 ls->set_allocation_failed();
816 }
817 return p;
818 }
819
820 // This version is lock-free.
821 HeapWord* MutableNUMASpace::cas_allocate(size_t size) {
822 Thread* thr = Thread::current();
823 int lgrp_id = thr->lgrp_id();
824 if (lgrp_id == -1 || !os::numa_has_group_homing()) {
825 lgrp_id = os::numa_get_group_id();
826 thr->set_lgrp_id(lgrp_id);
827 }
828
829 int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals);
830 // It is possible that a new CPU has been hotplugged and
831 // we haven't reshaped the space accordingly.
832 if (i == -1) {
833 i = os::random() % lgrp_spaces()->length();
834 }
835 LGRPSpace *ls = lgrp_spaces()->at(i);
836 MutableSpace *s = ls->space();
837 HeapWord *p = s->cas_allocate(size);
838 if (p != NULL) {
839 size_t remainder = pointer_delta(s->end(), p + size);
840 if (remainder < CollectedHeap::min_fill_size() && remainder > 0) {
841 if (s->cas_deallocate(p, size)) {
842 // We were the last to allocate and created a fragment less than
843 // a minimal object.
844 p = NULL;
845 } else {
846 guarantee(false, "Deallocation should always succeed");
847 }
848 }
849 }
850 if (p != NULL) {
851 HeapWord* cur_top, *cur_chunk_top = p + size;
852 while ((cur_top = top()) < cur_chunk_top) { // Keep _top updated.
853 if (Atomic::cmpxchg_ptr(cur_chunk_top, top_addr(), cur_top) == cur_top) {
854 break;
855 }
856 }
857 }
858
859 // Make the page allocation happen here if there is no static binding.
860 if (p != NULL && !os::numa_has_static_binding() ) {
861 for (HeapWord *i = p; i < p + size; i += os::vm_page_size() >> LogHeapWordSize) {
862 *(int*)i = 0;
863 }
864 }
865 if (p == NULL) {
866 ls->set_allocation_failed();
867 }
868 return p;
869 }
870
871 void MutableNUMASpace::print_short_on(outputStream* st) const {
872 MutableSpace::print_short_on(st);
873 st->print(" (");
874 for (int i = 0; i < lgrp_spaces()->length(); i++) {
875 st->print("lgrp %d: ", lgrp_spaces()->at(i)->lgrp_id());
876 lgrp_spaces()->at(i)->space()->print_short_on(st);
877 if (i < lgrp_spaces()->length() - 1) {
878 st->print(", ");
879 }
880 }
881 st->print(")");
882 }
883
884 void MutableNUMASpace::print_on(outputStream* st) const {
885 MutableSpace::print_on(st);
886 for (int i = 0; i < lgrp_spaces()->length(); i++) {
887 LGRPSpace *ls = lgrp_spaces()->at(i);
888 st->print(" lgrp %d", ls->lgrp_id());
889 ls->space()->print_on(st);
890 if (NUMAStats) {
891 for (int i = 0; i < lgrp_spaces()->length(); i++) {
892 lgrp_spaces()->at(i)->accumulate_statistics(page_size());
893 }
894 st->print(" local/remote/unbiased/uncommitted: " SIZE_FORMAT "K/"
895 SIZE_FORMAT "K/" SIZE_FORMAT "K/" SIZE_FORMAT
896 "K, large/small pages: " SIZE_FORMAT "/" SIZE_FORMAT "\n",
897 ls->space_stats()->_local_space / K,
898 ls->space_stats()->_remote_space / K,
899 ls->space_stats()->_unbiased_space / K,
900 ls->space_stats()->_uncommited_space / K,
901 ls->space_stats()->_large_pages,
902 ls->space_stats()->_small_pages);
903 }
904 }
905 }
906
907 void MutableNUMASpace::verify() {
908 // This can be called after setting an arbitrary value to the space's top,
909 // so an object can cross the chunk boundary. We ensure the parsability
910 // of the space and just walk the objects in linear fashion.
911 ensure_parsability();
912 MutableSpace::verify();
913 }
914
915 // Scan pages and gather stats about page placement and size.
916 void MutableNUMASpace::LGRPSpace::accumulate_statistics(size_t page_size) {
917 clear_space_stats();
918 char *start = (char*)round_to((intptr_t) space()->bottom(), page_size);
919 char* end = (char*)round_down((intptr_t) space()->end(), page_size);
920 if (start < end) {
921 for (char *p = start; p < end;) {
922 os::page_info info;
923 if (os::get_page_info(p, &info)) {
924 if (info.size > 0) {
925 if (info.size > (size_t)os::vm_page_size()) {
926 space_stats()->_large_pages++;
927 } else {
928 space_stats()->_small_pages++;
929 }
930 if (info.lgrp_id == lgrp_id()) {
931 space_stats()->_local_space += info.size;
932 } else {
933 space_stats()->_remote_space += info.size;
934 }
935 p += info.size;
936 } else {
937 p += os::vm_page_size();
938 space_stats()->_uncommited_space += os::vm_page_size();
939 }
940 } else {
941 return;
942 }
943 }
944 }
945 space_stats()->_unbiased_space = pointer_delta(start, space()->bottom(), sizeof(char)) +
946 pointer_delta(space()->end(), end, sizeof(char));
947
948 }
949
950 // Scan page_count pages and verify if they have the right size and right placement.
951 // If invalid pages are found they are freed in hope that subsequent reallocation
952 // will be more successful.
953 void MutableNUMASpace::LGRPSpace::scan_pages(size_t page_size, size_t page_count)
954 {
955 char* range_start = (char*)round_to((intptr_t) space()->bottom(), page_size);
956 char* range_end = (char*)round_down((intptr_t) space()->end(), page_size);
957
958 if (range_start > last_page_scanned() || last_page_scanned() >= range_end) {
959 set_last_page_scanned(range_start);
960 }
961
962 char *scan_start = last_page_scanned();
963 char* scan_end = MIN2(scan_start + page_size * page_count, range_end);
964
965 os::page_info page_expected, page_found;
966 page_expected.size = page_size;
967 page_expected.lgrp_id = lgrp_id();
968
969 char *s = scan_start;
970 while (s < scan_end) {
971 char *e = os::scan_pages(s, (char*)scan_end, &page_expected, &page_found);
972 if (e == NULL) {
973 break;
974 }
975 if (e != scan_end) {
976 assert(e < scan_end, err_msg("e: " PTR_FORMAT " scan_end: " PTR_FORMAT, e, scan_end));
977
978 if ((page_expected.size != page_size || page_expected.lgrp_id != lgrp_id())
979 && page_expected.size != 0) {
980 os::free_memory(s, pointer_delta(e, s, sizeof(char)), page_size);
981 }
982 page_expected = page_found;
983 }
984 s = e;
985 }
986
987 set_last_page_scanned(scan_end);
988 }