1
2 /*
3 * Copyright (c) 2006, 2010, Oracle and/or its affiliates. All rights reserved.
4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 *
6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "precompiled.hpp"
27 #include "gc_implementation/shared/mutableNUMASpace.hpp"
28 #include "gc_implementation/shared/spaceDecorator.hpp"
29 #include "memory/sharedHeap.hpp"
30 #include "oops/oop.inline.hpp"
31 #ifdef TARGET_OS_FAMILY_linux
32 # include "thread_linux.inline.hpp"
33 #endif
34 #ifdef TARGET_OS_FAMILY_solaris
35 # include "thread_solaris.inline.hpp"
36 #endif
37 #ifdef TARGET_OS_FAMILY_windows
38 # include "thread_windows.inline.hpp"
39 #endif
40 #ifdef TARGET_OS_FAMILY_bsd
41 # include "thread_bsd.inline.hpp"
42 #endif
43
44
45 MutableNUMASpace::MutableNUMASpace(size_t alignment) : MutableSpace(alignment) {
46 _lgrp_spaces = new (ResourceObj::C_HEAP) GrowableArray<LGRPSpace*>(0, true);
47 _page_size = os::vm_page_size();
48 _adaptation_cycles = 0;
49 _samples_count = 0;
50 update_layout(true);
51 }
52
53 MutableNUMASpace::~MutableNUMASpace() {
54 for (int i = 0; i < lgrp_spaces()->length(); i++) {
55 delete lgrp_spaces()->at(i);
56 }
57 delete lgrp_spaces();
58 }
59
60 #ifndef PRODUCT
61 void MutableNUMASpace::mangle_unused_area() {
62 // This method should do nothing.
63 // It can be called on a numa space during a full compaction.
64 }
65 void MutableNUMASpace::mangle_unused_area_complete() {
66 // This method should do nothing.
67 // It can be called on a numa space during a full compaction.
68 }
69 void MutableNUMASpace::mangle_region(MemRegion mr) {
70 // This method should do nothing because numa spaces are not mangled.
71 }
72 void MutableNUMASpace::set_top_for_allocations(HeapWord* v) {
73 assert(false, "Do not mangle MutableNUMASpace's");
74 }
75 void MutableNUMASpace::set_top_for_allocations() {
76 // This method should do nothing.
77 }
78 void MutableNUMASpace::check_mangled_unused_area(HeapWord* limit) {
79 // This method should do nothing.
80 }
81 void MutableNUMASpace::check_mangled_unused_area_complete() {
82 // This method should do nothing.
83 }
84 #endif // NOT_PRODUCT
85
86 // There may be unallocated holes in the middle chunks
87 // that should be filled with dead objects to ensure parseability.
88 void MutableNUMASpace::ensure_parsability() {
89 for (int i = 0; i < lgrp_spaces()->length(); i++) {
90 LGRPSpace *ls = lgrp_spaces()->at(i);
91 MutableSpace *s = ls->space();
92 if (s->top() < top()) { // For all spaces preceding the one containing top()
93 if (s->free_in_words() > 0) {
94 size_t area_touched_words = pointer_delta(s->end(), s->top());
95 CollectedHeap::fill_with_object(s->top(), area_touched_words);
96 #ifndef ASSERT
97 if (!ZapUnusedHeapArea) {
98 area_touched_words = MIN2((size_t)align_object_size(typeArrayOopDesc::header_size(T_INT)),
99 area_touched_words);
100 }
101 #endif
102 if (!os::numa_has_static_binding()) {
103 MemRegion invalid;
104 HeapWord *crossing_start = (HeapWord*)round_to((intptr_t)s->top(), os::vm_page_size());
105 HeapWord *crossing_end = (HeapWord*)round_to((intptr_t)(s->top() + area_touched_words),
106 os::vm_page_size());
107 if (crossing_start != crossing_end) {
108 // If object header crossed a small page boundary we mark the area
109 // as invalid rounding it to a page_size().
110 HeapWord *start = MAX2((HeapWord*)round_down((intptr_t)s->top(), page_size()), s->bottom());
111 HeapWord *end = MIN2((HeapWord*)round_to((intptr_t)(s->top() + area_touched_words), page_size()),
112 s->end());
113 invalid = MemRegion(start, end);
114 }
115
116 ls->add_invalid_region(invalid);
117 }
118 }
119 } else {
120 if (!os::numa_has_static_binding()) {
121 #ifdef ASSERT
122 MemRegion invalid(s->top(), s->end());
123 ls->add_invalid_region(invalid);
124 #else
125 if (ZapUnusedHeapArea) {
126 MemRegion invalid(s->top(), s->end());
127 ls->add_invalid_region(invalid);
128 } else {
129 return;
130 }
131 #endif
132 } else {
133 return;
134 }
135 }
136 }
137 }
138
139 size_t MutableNUMASpace::used_in_words() const {
140 size_t s = 0;
141 for (int i = 0; i < lgrp_spaces()->length(); i++) {
142 s += lgrp_spaces()->at(i)->space()->used_in_words();
143 }
144 return s;
145 }
146
147 size_t MutableNUMASpace::free_in_words() const {
148 size_t s = 0;
149 for (int i = 0; i < lgrp_spaces()->length(); i++) {
150 s += lgrp_spaces()->at(i)->space()->free_in_words();
151 }
152 return s;
153 }
154
155
156 size_t MutableNUMASpace::tlab_capacity(Thread *thr) const {
157 guarantee(thr != NULL, "No thread");
158 int lgrp_id = thr->lgrp_id();
159 if (lgrp_id == -1) {
160 // This case can occur after the topology of the system has
161 // changed. Thread can change their location, the new home
162 // group will be determined during the first allocation
163 // attempt. For now we can safely assume that all spaces
164 // have equal size because the whole space will be reinitialized.
165 if (lgrp_spaces()->length() > 0) {
166 return capacity_in_bytes() / lgrp_spaces()->length();
167 } else {
168 assert(false, "There should be at least one locality group");
169 return 0;
170 }
171 }
172 // That's the normal case, where we know the locality group of the thread.
173 int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals);
174 if (i == -1) {
175 return 0;
176 }
177 return lgrp_spaces()->at(i)->space()->capacity_in_bytes();
178 }
179
180 size_t MutableNUMASpace::unsafe_max_tlab_alloc(Thread *thr) const {
181 // Please see the comments for tlab_capacity().
182 guarantee(thr != NULL, "No thread");
183 int lgrp_id = thr->lgrp_id();
184 if (lgrp_id == -1) {
185 if (lgrp_spaces()->length() > 0) {
186 return free_in_bytes() / lgrp_spaces()->length();
187 } else {
188 assert(false, "There should be at least one locality group");
189 return 0;
190 }
191 }
192 int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals);
193 if (i == -1) {
194 return 0;
195 }
196 return lgrp_spaces()->at(i)->space()->free_in_bytes();
197 }
198
199
200 size_t MutableNUMASpace::capacity_in_words(Thread* thr) const {
201 guarantee(thr != NULL, "No thread");
202 int lgrp_id = thr->lgrp_id();
203 if (lgrp_id == -1) {
204 if (lgrp_spaces()->length() > 0) {
205 return capacity_in_words() / lgrp_spaces()->length();
206 } else {
207 assert(false, "There should be at least one locality group");
208 return 0;
209 }
210 }
211 int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals);
212 if (i == -1) {
213 return 0;
214 }
215 return lgrp_spaces()->at(i)->space()->capacity_in_words();
216 }
217
218 // Check if the NUMA topology has changed. Add and remove spaces if needed.
219 // The update can be forced by setting the force parameter equal to true.
220 bool MutableNUMASpace::update_layout(bool force) {
221 // Check if the topology had changed.
222 bool changed = os::numa_topology_changed();
223 if (force || changed) {
224 // Compute lgrp intersection. Add/remove spaces.
225 int lgrp_limit = (int)os::numa_get_groups_num();
226 int *lgrp_ids = NEW_C_HEAP_ARRAY(int, lgrp_limit);
227 int lgrp_num = (int)os::numa_get_leaf_groups(lgrp_ids, lgrp_limit);
228 assert(lgrp_num > 0, "There should be at least one locality group");
229 // Add new spaces for the new nodes
230 for (int i = 0; i < lgrp_num; i++) {
231 bool found = false;
232 for (int j = 0; j < lgrp_spaces()->length(); j++) {
233 if (lgrp_spaces()->at(j)->lgrp_id() == lgrp_ids[i]) {
234 found = true;
235 break;
236 }
237 }
238 if (!found) {
239 lgrp_spaces()->append(new LGRPSpace(lgrp_ids[i], alignment()));
240 }
241 }
242
243 // Remove spaces for the removed nodes.
244 for (int i = 0; i < lgrp_spaces()->length();) {
245 bool found = false;
246 for (int j = 0; j < lgrp_num; j++) {
247 if (lgrp_spaces()->at(i)->lgrp_id() == lgrp_ids[j]) {
248 found = true;
249 break;
250 }
251 }
252 if (!found) {
253 delete lgrp_spaces()->at(i);
254 lgrp_spaces()->remove_at(i);
255 } else {
256 i++;
257 }
258 }
259
260 FREE_C_HEAP_ARRAY(int, lgrp_ids);
261
262 if (changed) {
263 for (JavaThread *thread = Threads::first(); thread; thread = thread->next()) {
264 thread->set_lgrp_id(-1);
265 }
266 }
267 return true;
268 }
269 return false;
270 }
271
272 // Bias region towards the first-touching lgrp. Set the right page sizes.
273 void MutableNUMASpace::bias_region(MemRegion mr, int lgrp_id) {
274 HeapWord *start = (HeapWord*)round_to((intptr_t)mr.start(), page_size());
275 HeapWord *end = (HeapWord*)round_down((intptr_t)mr.end(), page_size());
276 if (end > start) {
277 MemRegion aligned_region(start, end);
278 assert((intptr_t)aligned_region.start() % page_size() == 0 &&
279 (intptr_t)aligned_region.byte_size() % page_size() == 0, "Bad alignment");
280 assert(region().contains(aligned_region), "Sanity");
281 // First we tell the OS which page size we want in the given range. The underlying
282 // large page can be broken down if we require small pages.
283 os::realign_memory((char*)aligned_region.start(), aligned_region.byte_size(), page_size());
284 // Then we uncommit the pages in the range.
285 os::free_memory((char*)aligned_region.start(), aligned_region.byte_size());
286 // And make them local/first-touch biased.
287 os::numa_make_local((char*)aligned_region.start(), aligned_region.byte_size(), lgrp_id);
288 }
289 }
290
291 // Free all pages in the region.
292 void MutableNUMASpace::free_region(MemRegion mr) {
293 HeapWord *start = (HeapWord*)round_to((intptr_t)mr.start(), page_size());
294 HeapWord *end = (HeapWord*)round_down((intptr_t)mr.end(), page_size());
295 if (end > start) {
296 MemRegion aligned_region(start, end);
297 assert((intptr_t)aligned_region.start() % page_size() == 0 &&
298 (intptr_t)aligned_region.byte_size() % page_size() == 0, "Bad alignment");
299 assert(region().contains(aligned_region), "Sanity");
300 os::free_memory((char*)aligned_region.start(), aligned_region.byte_size());
301 }
302 }
303
304 // Update space layout. Perform adaptation.
305 void MutableNUMASpace::update() {
306 if (update_layout(false)) {
307 // If the topology has changed, make all chunks zero-sized.
308 // And clear the alloc-rate statistics.
309 // In future we may want to handle this more gracefully in order
310 // to avoid the reallocation of the pages as much as possible.
311 for (int i = 0; i < lgrp_spaces()->length(); i++) {
312 LGRPSpace *ls = lgrp_spaces()->at(i);
313 MutableSpace *s = ls->space();
314 s->set_end(s->bottom());
315 s->set_top(s->bottom());
316 ls->clear_alloc_rate();
317 }
318 // A NUMA space is never mangled
319 initialize(region(),
320 SpaceDecorator::Clear,
321 SpaceDecorator::DontMangle);
322 } else {
323 bool should_initialize = false;
324 if (!os::numa_has_static_binding()) {
325 for (int i = 0; i < lgrp_spaces()->length(); i++) {
326 if (!lgrp_spaces()->at(i)->invalid_region().is_empty()) {
327 should_initialize = true;
328 break;
329 }
330 }
331 }
332
333 if (should_initialize ||
334 (UseAdaptiveNUMAChunkSizing && adaptation_cycles() < samples_count())) {
335 // A NUMA space is never mangled
336 initialize(region(),
337 SpaceDecorator::Clear,
338 SpaceDecorator::DontMangle);
339 }
340 }
341
342 if (NUMAStats) {
343 for (int i = 0; i < lgrp_spaces()->length(); i++) {
344 lgrp_spaces()->at(i)->accumulate_statistics(page_size());
345 }
346 }
347
348 scan_pages(NUMAPageScanRate);
349 }
350
351 // Scan pages. Free pages that have smaller size or wrong placement.
352 void MutableNUMASpace::scan_pages(size_t page_count)
353 {
354 size_t pages_per_chunk = page_count / lgrp_spaces()->length();
355 if (pages_per_chunk > 0) {
356 for (int i = 0; i < lgrp_spaces()->length(); i++) {
357 LGRPSpace *ls = lgrp_spaces()->at(i);
358 ls->scan_pages(page_size(), pages_per_chunk);
359 }
360 }
361 }
362
363 // Accumulate statistics about the allocation rate of each lgrp.
364 void MutableNUMASpace::accumulate_statistics() {
365 if (UseAdaptiveNUMAChunkSizing) {
366 for (int i = 0; i < lgrp_spaces()->length(); i++) {
367 lgrp_spaces()->at(i)->sample();
368 }
369 increment_samples_count();
370 }
371
372 if (NUMAStats) {
373 for (int i = 0; i < lgrp_spaces()->length(); i++) {
374 lgrp_spaces()->at(i)->accumulate_statistics(page_size());
375 }
376 }
377 }
378
379 // Get the current size of a chunk.
380 // This function computes the size of the chunk based on the
381 // difference between chunk ends. This allows it to work correctly in
382 // case the whole space is resized and during the process of adaptive
383 // chunk resizing.
384 size_t MutableNUMASpace::current_chunk_size(int i) {
385 HeapWord *cur_end, *prev_end;
386 if (i == 0) {
387 prev_end = bottom();
388 } else {
389 prev_end = lgrp_spaces()->at(i - 1)->space()->end();
390 }
391 if (i == lgrp_spaces()->length() - 1) {
392 cur_end = end();
393 } else {
394 cur_end = lgrp_spaces()->at(i)->space()->end();
395 }
396 if (cur_end > prev_end) {
397 return pointer_delta(cur_end, prev_end, sizeof(char));
398 }
399 return 0;
400 }
401
402 // Return the default chunk size by equally diving the space.
403 // page_size() aligned.
404 size_t MutableNUMASpace::default_chunk_size() {
405 return base_space_size() / lgrp_spaces()->length() * page_size();
406 }
407
408 // Produce a new chunk size. page_size() aligned.
409 // This function is expected to be called on sequence of i's from 0 to
410 // lgrp_spaces()->length().
411 size_t MutableNUMASpace::adaptive_chunk_size(int i, size_t limit) {
412 size_t pages_available = base_space_size();
413 for (int j = 0; j < i; j++) {
414 pages_available -= round_down(current_chunk_size(j), page_size()) / page_size();
415 }
416 pages_available -= lgrp_spaces()->length() - i - 1;
417 assert(pages_available > 0, "No pages left");
418 float alloc_rate = 0;
419 for (int j = i; j < lgrp_spaces()->length(); j++) {
420 alloc_rate += lgrp_spaces()->at(j)->alloc_rate()->average();
421 }
422 size_t chunk_size = 0;
423 if (alloc_rate > 0) {
424 LGRPSpace *ls = lgrp_spaces()->at(i);
425 chunk_size = (size_t)(ls->alloc_rate()->average() / alloc_rate * pages_available) * page_size();
426 }
427 chunk_size = MAX2(chunk_size, page_size());
428
429 if (limit > 0) {
430 limit = round_down(limit, page_size());
431 if (chunk_size > current_chunk_size(i)) {
432 size_t upper_bound = pages_available * page_size();
433 if (upper_bound > limit &&
434 current_chunk_size(i) < upper_bound - limit) {
435 // The resulting upper bound should not exceed the available
436 // amount of memory (pages_available * page_size()).
437 upper_bound = current_chunk_size(i) + limit;
438 }
439 chunk_size = MIN2(chunk_size, upper_bound);
440 } else {
441 size_t lower_bound = page_size();
442 if (current_chunk_size(i) > limit) { // lower_bound shouldn't underflow.
443 lower_bound = current_chunk_size(i) - limit;
444 }
445 chunk_size = MAX2(chunk_size, lower_bound);
446 }
447 }
448 assert(chunk_size <= pages_available * page_size(), "Chunk size out of range");
449 return chunk_size;
450 }
451
452
453 // Return the bottom_region and the top_region. Align them to page_size() boundary.
454 // |------------------new_region---------------------------------|
455 // |----bottom_region--|---intersection---|------top_region------|
456 void MutableNUMASpace::select_tails(MemRegion new_region, MemRegion intersection,
457 MemRegion* bottom_region, MemRegion *top_region) {
458 // Is there bottom?
459 if (new_region.start() < intersection.start()) { // Yes
460 // Try to coalesce small pages into a large one.
461 if (UseLargePages && page_size() >= alignment()) {
462 HeapWord* p = (HeapWord*)round_to((intptr_t) intersection.start(), alignment());
463 if (new_region.contains(p)
464 && pointer_delta(p, new_region.start(), sizeof(char)) >= alignment()) {
465 if (intersection.contains(p)) {
466 intersection = MemRegion(p, intersection.end());
467 } else {
468 intersection = MemRegion(p, p);
469 }
470 }
471 }
472 *bottom_region = MemRegion(new_region.start(), intersection.start());
473 } else {
474 *bottom_region = MemRegion();
475 }
476
477 // Is there top?
478 if (intersection.end() < new_region.end()) { // Yes
479 // Try to coalesce small pages into a large one.
480 if (UseLargePages && page_size() >= alignment()) {
481 HeapWord* p = (HeapWord*)round_down((intptr_t) intersection.end(), alignment());
482 if (new_region.contains(p)
483 && pointer_delta(new_region.end(), p, sizeof(char)) >= alignment()) {
484 if (intersection.contains(p)) {
485 intersection = MemRegion(intersection.start(), p);
486 } else {
487 intersection = MemRegion(p, p);
488 }
489 }
490 }
491 *top_region = MemRegion(intersection.end(), new_region.end());
492 } else {
493 *top_region = MemRegion();
494 }
495 }
496
497 // Try to merge the invalid region with the bottom or top region by decreasing
498 // the intersection area. Return the invalid_region aligned to the page_size()
499 // boundary if it's inside the intersection. Return non-empty invalid_region
500 // if it lies inside the intersection (also page-aligned).
501 // |------------------new_region---------------------------------|
502 // |----------------|-------invalid---|--------------------------|
503 // |----bottom_region--|---intersection---|------top_region------|
504 void MutableNUMASpace::merge_regions(MemRegion new_region, MemRegion* intersection,
505 MemRegion *invalid_region) {
506 if (intersection->start() >= invalid_region->start() && intersection->contains(invalid_region->end())) {
507 *intersection = MemRegion(invalid_region->end(), intersection->end());
508 *invalid_region = MemRegion();
509 } else
510 if (intersection->end() <= invalid_region->end() && intersection->contains(invalid_region->start())) {
511 *intersection = MemRegion(intersection->start(), invalid_region->start());
512 *invalid_region = MemRegion();
513 } else
514 if (intersection->equals(*invalid_region) || invalid_region->contains(*intersection)) {
515 *intersection = MemRegion(new_region.start(), new_region.start());
516 *invalid_region = MemRegion();
517 } else
518 if (intersection->contains(invalid_region)) {
519 // That's the only case we have to make an additional bias_region() call.
520 HeapWord* start = invalid_region->start();
521 HeapWord* end = invalid_region->end();
522 if (UseLargePages && page_size() >= alignment()) {
523 HeapWord *p = (HeapWord*)round_down((intptr_t) start, alignment());
524 if (new_region.contains(p)) {
525 start = p;
526 }
527 p = (HeapWord*)round_to((intptr_t) end, alignment());
528 if (new_region.contains(end)) {
529 end = p;
530 }
531 }
532 if (intersection->start() > start) {
533 *intersection = MemRegion(start, intersection->end());
534 }
535 if (intersection->end() < end) {
536 *intersection = MemRegion(intersection->start(), end);
537 }
538 *invalid_region = MemRegion(start, end);
539 }
540 }
541
542 void MutableNUMASpace::initialize(MemRegion mr,
543 bool clear_space,
544 bool mangle_space,
545 bool setup_pages) {
546 assert(clear_space, "Reallocation will destory data!");
547 assert(lgrp_spaces()->length() > 0, "There should be at least one space");
548
549 MemRegion old_region = region(), new_region;
550 set_bottom(mr.start());
551 set_end(mr.end());
552 // Must always clear the space
553 clear(SpaceDecorator::DontMangle);
554
555 // Compute chunk sizes
556 size_t prev_page_size = page_size();
557 set_page_size(UseLargePages ? alignment() : os::vm_page_size());
558 HeapWord* rounded_bottom = (HeapWord*)round_to((intptr_t) bottom(), page_size());
559 HeapWord* rounded_end = (HeapWord*)round_down((intptr_t) end(), page_size());
560 size_t base_space_size_pages = pointer_delta(rounded_end, rounded_bottom, sizeof(char)) / page_size();
561
562 // Try small pages if the chunk size is too small
563 if (base_space_size_pages / lgrp_spaces()->length() == 0
564 && page_size() > (size_t)os::vm_page_size()) {
565 set_page_size(os::vm_page_size());
566 rounded_bottom = (HeapWord*)round_to((intptr_t) bottom(), page_size());
567 rounded_end = (HeapWord*)round_down((intptr_t) end(), page_size());
568 base_space_size_pages = pointer_delta(rounded_end, rounded_bottom, sizeof(char)) / page_size();
569 }
570 guarantee(base_space_size_pages / lgrp_spaces()->length() > 0, "Space too small");
571 set_base_space_size(base_space_size_pages);
572
573 // Handle space resize
574 MemRegion top_region, bottom_region;
575 if (!old_region.equals(region())) {
576 new_region = MemRegion(rounded_bottom, rounded_end);
577 MemRegion intersection = new_region.intersection(old_region);
578 if (intersection.start() == NULL ||
579 intersection.end() == NULL ||
580 prev_page_size > page_size()) { // If the page size got smaller we have to change
581 // the page size preference for the whole space.
582 intersection = MemRegion(new_region.start(), new_region.start());
583 }
584 select_tails(new_region, intersection, &bottom_region, &top_region);
585 bias_region(bottom_region, lgrp_spaces()->at(0)->lgrp_id());
586 bias_region(top_region, lgrp_spaces()->at(lgrp_spaces()->length() - 1)->lgrp_id());
587 }
588
589 // Check if the space layout has changed significantly?
590 // This happens when the space has been resized so that either head or tail
591 // chunk became less than a page.
592 bool layout_valid = UseAdaptiveNUMAChunkSizing &&
593 current_chunk_size(0) > page_size() &&
594 current_chunk_size(lgrp_spaces()->length() - 1) > page_size();
595
596
597 for (int i = 0; i < lgrp_spaces()->length(); i++) {
598 LGRPSpace *ls = lgrp_spaces()->at(i);
599 MutableSpace *s = ls->space();
600 old_region = s->region();
601
602 size_t chunk_byte_size = 0, old_chunk_byte_size = 0;
603 if (i < lgrp_spaces()->length() - 1) {
604 if (!UseAdaptiveNUMAChunkSizing ||
605 (UseAdaptiveNUMAChunkSizing && NUMAChunkResizeWeight == 0) ||
606 samples_count() < AdaptiveSizePolicyReadyThreshold) {
607 // No adaptation. Divide the space equally.
608 chunk_byte_size = default_chunk_size();
609 } else
610 if (!layout_valid || NUMASpaceResizeRate == 0) {
611 // Fast adaptation. If no space resize rate is set, resize
612 // the chunks instantly.
613 chunk_byte_size = adaptive_chunk_size(i, 0);
614 } else {
615 // Slow adaptation. Resize the chunks moving no more than
616 // NUMASpaceResizeRate bytes per collection.
617 size_t limit = NUMASpaceResizeRate /
618 (lgrp_spaces()->length() * (lgrp_spaces()->length() + 1) / 2);
619 chunk_byte_size = adaptive_chunk_size(i, MAX2(limit * (i + 1), page_size()));
620 }
621
622 assert(chunk_byte_size >= page_size(), "Chunk size too small");
623 assert(chunk_byte_size <= capacity_in_bytes(), "Sanity check");
624 }
625
626 if (i == 0) { // Bottom chunk
627 if (i != lgrp_spaces()->length() - 1) {
628 new_region = MemRegion(bottom(), rounded_bottom + (chunk_byte_size >> LogHeapWordSize));
629 } else {
630 new_region = MemRegion(bottom(), end());
631 }
632 } else
633 if (i < lgrp_spaces()->length() - 1) { // Middle chunks
634 MutableSpace *ps = lgrp_spaces()->at(i - 1)->space();
635 new_region = MemRegion(ps->end(),
636 ps->end() + (chunk_byte_size >> LogHeapWordSize));
637 } else { // Top chunk
638 MutableSpace *ps = lgrp_spaces()->at(i - 1)->space();
639 new_region = MemRegion(ps->end(), end());
640 }
641 guarantee(region().contains(new_region), "Region invariant");
642
643
644 // The general case:
645 // |---------------------|--invalid---|--------------------------|
646 // |------------------new_region---------------------------------|
647 // |----bottom_region--|---intersection---|------top_region------|
648 // |----old_region----|
649 // The intersection part has all pages in place we don't need to migrate them.
650 // Pages for the top and bottom part should be freed and then reallocated.
651
652 MemRegion intersection = old_region.intersection(new_region);
653
654 if (intersection.start() == NULL || intersection.end() == NULL) {
655 intersection = MemRegion(new_region.start(), new_region.start());
656 }
657
658 if (!os::numa_has_static_binding()) {
659 MemRegion invalid_region = ls->invalid_region().intersection(new_region);
660 // Invalid region is a range of memory that could've possibly
661 // been allocated on the other node. That's relevant only on Solaris where
662 // there is no static memory binding.
663 if (!invalid_region.is_empty()) {
664 merge_regions(new_region, &intersection, &invalid_region);
665 free_region(invalid_region);
666 ls->set_invalid_region(MemRegion());
667 }
668 }
669
670 select_tails(new_region, intersection, &bottom_region, &top_region);
671
672 if (!os::numa_has_static_binding()) {
673 // If that's a system with the first-touch policy then it's enough
674 // to free the pages.
675 free_region(bottom_region);
676 free_region(top_region);
677 } else {
678 // In a system with static binding we have to change the bias whenever
679 // we reshape the heap.
680 bias_region(bottom_region, ls->lgrp_id());
681 bias_region(top_region, ls->lgrp_id());
682 }
683
684 // Clear space (set top = bottom) but never mangle.
685 s->initialize(new_region, SpaceDecorator::Clear, SpaceDecorator::DontMangle, MutableSpace::DontSetupPages);
686
687 set_adaptation_cycles(samples_count());
688 }
689 }
690
691 // Set the top of the whole space.
692 // Mark the the holes in chunks below the top() as invalid.
693 void MutableNUMASpace::set_top(HeapWord* value) {
694 bool found_top = false;
695 for (int i = 0; i < lgrp_spaces()->length();) {
696 LGRPSpace *ls = lgrp_spaces()->at(i);
697 MutableSpace *s = ls->space();
698 HeapWord *top = MAX2((HeapWord*)round_down((intptr_t)s->top(), page_size()), s->bottom());
699
700 if (s->contains(value)) {
701 // Check if setting the chunk's top to a given value would create a hole less than
702 // a minimal object; assuming that's not the last chunk in which case we don't care.
703 if (i < lgrp_spaces()->length() - 1) {
704 size_t remainder = pointer_delta(s->end(), value);
705 const size_t min_fill_size = CollectedHeap::min_fill_size();
706 if (remainder < min_fill_size && remainder > 0) {
707 // Add a minimum size filler object; it will cross the chunk boundary.
708 CollectedHeap::fill_with_object(value, min_fill_size);
709 value += min_fill_size;
710 assert(!s->contains(value), "Should be in the next chunk");
711 // Restart the loop from the same chunk, since the value has moved
712 // to the next one.
713 continue;
714 }
715 }
716
717 if (!os::numa_has_static_binding() && top < value && top < s->end()) {
718 ls->add_invalid_region(MemRegion(top, value));
719 }
720 s->set_top(value);
721 found_top = true;
722 } else {
723 if (found_top) {
724 s->set_top(s->bottom());
725 } else {
726 if (!os::numa_has_static_binding() && top < s->end()) {
727 ls->add_invalid_region(MemRegion(top, s->end()));
728 }
729 s->set_top(s->end());
730 }
731 }
732 i++;
733 }
734 MutableSpace::set_top(value);
735 }
736
737 void MutableNUMASpace::clear(bool mangle_space) {
738 MutableSpace::set_top(bottom());
739 for (int i = 0; i < lgrp_spaces()->length(); i++) {
740 // Never mangle NUMA spaces because the mangling will
741 // bind the memory to a possibly unwanted lgroup.
742 lgrp_spaces()->at(i)->space()->clear(SpaceDecorator::DontMangle);
743 }
744 }
745
746 /*
747 Linux supports static memory binding, therefore the most part of the
748 logic dealing with the possible invalid page allocation is effectively
749 disabled. Besides there is no notion of the home node in Linux. A
750 thread is allowed to migrate freely. Although the scheduler is rather
751 reluctant to move threads between the nodes. We check for the current
752 node every allocation. And with a high probability a thread stays on
753 the same node for some time allowing local access to recently allocated
754 objects.
755 */
756
757 HeapWord* MutableNUMASpace::allocate(size_t size) {
758 Thread* thr = Thread::current();
759 int lgrp_id = thr->lgrp_id();
760 if (lgrp_id == -1 || !os::numa_has_group_homing()) {
761 lgrp_id = os::numa_get_group_id();
762 thr->set_lgrp_id(lgrp_id);
763 }
764
765 int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals);
766
767 // It is possible that a new CPU has been hotplugged and
768 // we haven't reshaped the space accordingly.
769 if (i == -1) {
770 i = os::random() % lgrp_spaces()->length();
771 }
772
773 LGRPSpace* ls = lgrp_spaces()->at(i);
774 MutableSpace *s = ls->space();
775 HeapWord *p = s->allocate(size);
776
777 if (p != NULL) {
778 size_t remainder = s->free_in_words();
779 if (remainder < CollectedHeap::min_fill_size() && remainder > 0) {
780 s->set_top(s->top() - size);
781 p = NULL;
782 }
783 }
784 if (p != NULL) {
785 if (top() < s->top()) { // Keep _top updated.
786 MutableSpace::set_top(s->top());
787 }
788 }
789 // Make the page allocation happen here if there is no static binding..
790 if (p != NULL && !os::numa_has_static_binding()) {
791 for (HeapWord *i = p; i < p + size; i += os::vm_page_size() >> LogHeapWordSize) {
792 *(int*)i = 0;
793 }
794 }
795 if (p == NULL) {
796 ls->set_allocation_failed();
797 }
798 return p;
799 }
800
801 // This version is lock-free.
802 HeapWord* MutableNUMASpace::cas_allocate(size_t size) {
803 Thread* thr = Thread::current();
804 int lgrp_id = thr->lgrp_id();
805 if (lgrp_id == -1 || !os::numa_has_group_homing()) {
806 lgrp_id = os::numa_get_group_id();
807 thr->set_lgrp_id(lgrp_id);
808 }
809
810 int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals);
811 // It is possible that a new CPU has been hotplugged and
812 // we haven't reshaped the space accordingly.
813 if (i == -1) {
814 i = os::random() % lgrp_spaces()->length();
815 }
816 LGRPSpace *ls = lgrp_spaces()->at(i);
817 MutableSpace *s = ls->space();
818 HeapWord *p = s->cas_allocate(size);
819 if (p != NULL) {
820 size_t remainder = pointer_delta(s->end(), p + size);
821 if (remainder < CollectedHeap::min_fill_size() && remainder > 0) {
822 if (s->cas_deallocate(p, size)) {
823 // We were the last to allocate and created a fragment less than
824 // a minimal object.
825 p = NULL;
826 } else {
827 guarantee(false, "Deallocation should always succeed");
828 }
829 }
830 }
831 if (p != NULL) {
832 HeapWord* cur_top, *cur_chunk_top = p + size;
833 while ((cur_top = top()) < cur_chunk_top) { // Keep _top updated.
834 if (Atomic::cmpxchg_ptr(cur_chunk_top, top_addr(), cur_top) == cur_top) {
835 break;
836 }
837 }
838 }
839
840 // Make the page allocation happen here if there is no static binding.
841 if (p != NULL && !os::numa_has_static_binding() ) {
842 for (HeapWord *i = p; i < p + size; i += os::vm_page_size() >> LogHeapWordSize) {
843 *(int*)i = 0;
844 }
845 }
846 if (p == NULL) {
847 ls->set_allocation_failed();
848 }
849 return p;
850 }
851
852 void MutableNUMASpace::print_short_on(outputStream* st) const {
853 MutableSpace::print_short_on(st);
854 st->print(" (");
855 for (int i = 0; i < lgrp_spaces()->length(); i++) {
856 st->print("lgrp %d: ", lgrp_spaces()->at(i)->lgrp_id());
857 lgrp_spaces()->at(i)->space()->print_short_on(st);
858 if (i < lgrp_spaces()->length() - 1) {
859 st->print(", ");
860 }
861 }
862 st->print(")");
863 }
864
865 void MutableNUMASpace::print_on(outputStream* st) const {
866 MutableSpace::print_on(st);
867 for (int i = 0; i < lgrp_spaces()->length(); i++) {
868 LGRPSpace *ls = lgrp_spaces()->at(i);
869 st->print(" lgrp %d", ls->lgrp_id());
870 ls->space()->print_on(st);
871 if (NUMAStats) {
872 for (int i = 0; i < lgrp_spaces()->length(); i++) {
873 lgrp_spaces()->at(i)->accumulate_statistics(page_size());
874 }
875 st->print(" local/remote/unbiased/uncommitted: %dK/%dK/%dK/%dK, large/small pages: %d/%d\n",
876 ls->space_stats()->_local_space / K,
877 ls->space_stats()->_remote_space / K,
878 ls->space_stats()->_unbiased_space / K,
879 ls->space_stats()->_uncommited_space / K,
880 ls->space_stats()->_large_pages,
881 ls->space_stats()->_small_pages);
882 }
883 }
884 }
885
886 void MutableNUMASpace::verify(bool allow_dirty) {
887 // This can be called after setting an arbitary value to the space's top,
888 // so an object can cross the chunk boundary. We ensure the parsablity
889 // of the space and just walk the objects in linear fashion.
890 ensure_parsability();
891 MutableSpace::verify(allow_dirty);
892 }
893
894 // Scan pages and gather stats about page placement and size.
895 void MutableNUMASpace::LGRPSpace::accumulate_statistics(size_t page_size) {
896 clear_space_stats();
897 char *start = (char*)round_to((intptr_t) space()->bottom(), page_size);
898 char* end = (char*)round_down((intptr_t) space()->end(), page_size);
899 if (start < end) {
900 for (char *p = start; p < end;) {
901 os::page_info info;
902 if (os::get_page_info(p, &info)) {
903 if (info.size > 0) {
904 if (info.size > (size_t)os::vm_page_size()) {
905 space_stats()->_large_pages++;
906 } else {
907 space_stats()->_small_pages++;
908 }
909 if (info.lgrp_id == lgrp_id()) {
910 space_stats()->_local_space += info.size;
911 } else {
912 space_stats()->_remote_space += info.size;
913 }
914 p += info.size;
915 } else {
916 p += os::vm_page_size();
917 space_stats()->_uncommited_space += os::vm_page_size();
918 }
919 } else {
920 return;
921 }
922 }
923 }
924 space_stats()->_unbiased_space = pointer_delta(start, space()->bottom(), sizeof(char)) +
925 pointer_delta(space()->end(), end, sizeof(char));
926
927 }
928
929 // Scan page_count pages and verify if they have the right size and right placement.
930 // If invalid pages are found they are freed in hope that subsequent reallocation
931 // will be more successful.
932 void MutableNUMASpace::LGRPSpace::scan_pages(size_t page_size, size_t page_count)
933 {
934 char* range_start = (char*)round_to((intptr_t) space()->bottom(), page_size);
935 char* range_end = (char*)round_down((intptr_t) space()->end(), page_size);
936
937 if (range_start > last_page_scanned() || last_page_scanned() >= range_end) {
938 set_last_page_scanned(range_start);
939 }
940
941 char *scan_start = last_page_scanned();
942 char* scan_end = MIN2(scan_start + page_size * page_count, range_end);
943
944 os::page_info page_expected, page_found;
945 page_expected.size = page_size;
946 page_expected.lgrp_id = lgrp_id();
947
948 char *s = scan_start;
949 while (s < scan_end) {
950 char *e = os::scan_pages(s, (char*)scan_end, &page_expected, &page_found);
951 if (e == NULL) {
952 break;
953 }
954 if (e != scan_end) {
955 if ((page_expected.size != page_size || page_expected.lgrp_id != lgrp_id())
956 && page_expected.size != 0) {
957 os::free_memory(s, pointer_delta(e, s, sizeof(char)));
958 }
959 page_expected = page_found;
960 }
961 s = e;
962 }
963
964 set_last_page_scanned(scan_end);
965 }