1
2 /*
3 * Copyright (c) 2006, 2010, Oracle and/or its affiliates. All rights reserved.
4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 *
6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "precompiled.hpp"
27 #include "gc_implementation/shared/mutableNUMASpace.hpp"
28 #include "gc_implementation/shared/spaceDecorator.hpp"
29 #include "memory/sharedHeap.hpp"
30 #include "oops/oop.inline.hpp"
31 #ifdef TARGET_OS_FAMILY_linux
32 # include "thread_linux.inline.hpp"
33 #endif
34 #ifdef TARGET_OS_FAMILY_solaris
35 # include "thread_solaris.inline.hpp"
36 #endif
37 #ifdef TARGET_OS_FAMILY_windows
38 # include "thread_windows.inline.hpp"
39 #endif
40
41
42 MutableNUMASpace::MutableNUMASpace(size_t alignment) : MutableSpace(alignment) {
43 _lgrp_spaces = new (ResourceObj::C_HEAP) GrowableArray<LGRPSpace*>(0, true);
44 _page_size = os::vm_page_size();
45 _adaptation_cycles = 0;
46 _samples_count = 0;
47 update_layout(true);
48 }
49
50 MutableNUMASpace::~MutableNUMASpace() {
51 for (int i = 0; i < lgrp_spaces()->length(); i++) {
52 delete lgrp_spaces()->at(i);
53 }
54 delete lgrp_spaces();
55 }
56
57 #ifndef PRODUCT
58 void MutableNUMASpace::mangle_unused_area() {
59 // This method should do nothing.
60 // It can be called on a numa space during a full compaction.
61 }
62 void MutableNUMASpace::mangle_unused_area_complete() {
63 // This method should do nothing.
64 // It can be called on a numa space during a full compaction.
65 }
66 void MutableNUMASpace::mangle_region(MemRegion mr) {
67 // This method should do nothing because numa spaces are not mangled.
68 }
69 void MutableNUMASpace::set_top_for_allocations(HeapWord* v) {
70 assert(false, "Do not mangle MutableNUMASpace's");
71 }
72 void MutableNUMASpace::set_top_for_allocations() {
73 // This method should do nothing.
74 }
75 void MutableNUMASpace::check_mangled_unused_area(HeapWord* limit) {
76 // This method should do nothing.
77 }
78 void MutableNUMASpace::check_mangled_unused_area_complete() {
79 // This method should do nothing.
80 }
81 #endif // NOT_PRODUCT
82
83 // There may be unallocated holes in the middle chunks
84 // that should be filled with dead objects to ensure parseability.
85 void MutableNUMASpace::ensure_parsability() {
86 for (int i = 0; i < lgrp_spaces()->length(); i++) {
87 LGRPSpace *ls = lgrp_spaces()->at(i);
88 MutableSpace *s = ls->space();
89 if (s->top() < top()) { // For all spaces preceding the one containing top()
90 if (s->free_in_words() > 0) {
91 size_t area_touched_words = pointer_delta(s->end(), s->top());
92 CollectedHeap::fill_with_object(s->top(), area_touched_words);
93 #ifndef ASSERT
94 if (!ZapUnusedHeapArea) {
95 area_touched_words = MIN2((size_t)align_object_size(typeArrayOopDesc::header_size(T_INT)),
96 area_touched_words);
97 }
98 #endif
99 if (!os::numa_has_static_binding()) {
100 MemRegion invalid;
101 HeapWord *crossing_start = (HeapWord*)round_to((intptr_t)s->top(), os::vm_page_size());
102 HeapWord *crossing_end = (HeapWord*)round_to((intptr_t)(s->top() + area_touched_words),
103 os::vm_page_size());
104 if (crossing_start != crossing_end) {
105 // If object header crossed a small page boundary we mark the area
106 // as invalid rounding it to a page_size().
107 HeapWord *start = MAX2((HeapWord*)round_down((intptr_t)s->top(), page_size()), s->bottom());
108 HeapWord *end = MIN2((HeapWord*)round_to((intptr_t)(s->top() + area_touched_words), page_size()),
109 s->end());
110 invalid = MemRegion(start, end);
111 }
112
113 ls->add_invalid_region(invalid);
114 }
115 }
116 } else {
117 if (!os::numa_has_static_binding()) {
118 #ifdef ASSERT
119 MemRegion invalid(s->top(), s->end());
120 ls->add_invalid_region(invalid);
121 #else
122 if (ZapUnusedHeapArea) {
123 MemRegion invalid(s->top(), s->end());
124 ls->add_invalid_region(invalid);
125 } else {
126 return;
127 }
128 #endif
129 } else {
130 return;
131 }
132 }
133 }
134 }
135
136 size_t MutableNUMASpace::used_in_words() const {
137 size_t s = 0;
138 for (int i = 0; i < lgrp_spaces()->length(); i++) {
139 s += lgrp_spaces()->at(i)->space()->used_in_words();
140 }
141 return s;
142 }
143
144 size_t MutableNUMASpace::free_in_words() const {
145 size_t s = 0;
146 for (int i = 0; i < lgrp_spaces()->length(); i++) {
147 s += lgrp_spaces()->at(i)->space()->free_in_words();
148 }
149 return s;
150 }
151
152
153 size_t MutableNUMASpace::tlab_capacity(Thread *thr) const {
154 guarantee(thr != NULL, "No thread");
155 int lgrp_id = thr->lgrp_id();
156 if (lgrp_id == -1) {
157 // This case can occur after the topology of the system has
158 // changed. Thread can change their location, the new home
159 // group will be determined during the first allocation
160 // attempt. For now we can safely assume that all spaces
161 // have equal size because the whole space will be reinitialized.
162 if (lgrp_spaces()->length() > 0) {
163 return capacity_in_bytes() / lgrp_spaces()->length();
164 } else {
165 assert(false, "There should be at least one locality group");
166 return 0;
167 }
168 }
169 // That's the normal case, where we know the locality group of the thread.
170 int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals);
171 if (i == -1) {
172 return 0;
173 }
174 return lgrp_spaces()->at(i)->space()->capacity_in_bytes();
175 }
176
177 size_t MutableNUMASpace::unsafe_max_tlab_alloc(Thread *thr) const {
178 // Please see the comments for tlab_capacity().
179 guarantee(thr != NULL, "No thread");
180 int lgrp_id = thr->lgrp_id();
181 if (lgrp_id == -1) {
182 if (lgrp_spaces()->length() > 0) {
183 return free_in_bytes() / lgrp_spaces()->length();
184 } else {
185 assert(false, "There should be at least one locality group");
186 return 0;
187 }
188 }
189 int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals);
190 if (i == -1) {
191 return 0;
192 }
193 return lgrp_spaces()->at(i)->space()->free_in_bytes();
194 }
195
196
197 size_t MutableNUMASpace::capacity_in_words(Thread* thr) const {
198 guarantee(thr != NULL, "No thread");
199 int lgrp_id = thr->lgrp_id();
200 if (lgrp_id == -1) {
201 if (lgrp_spaces()->length() > 0) {
202 return capacity_in_words() / lgrp_spaces()->length();
203 } else {
204 assert(false, "There should be at least one locality group");
205 return 0;
206 }
207 }
208 int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals);
209 if (i == -1) {
210 return 0;
211 }
212 return lgrp_spaces()->at(i)->space()->capacity_in_words();
213 }
214
215 // Check if the NUMA topology has changed. Add and remove spaces if needed.
216 // The update can be forced by setting the force parameter equal to true.
217 bool MutableNUMASpace::update_layout(bool force) {
218 // Check if the topology had changed.
219 bool changed = os::numa_topology_changed();
220 if (force || changed) {
221 // Compute lgrp intersection. Add/remove spaces.
222 int lgrp_limit = (int)os::numa_get_groups_num();
223 int *lgrp_ids = NEW_C_HEAP_ARRAY(int, lgrp_limit);
224 int lgrp_num = (int)os::numa_get_leaf_groups(lgrp_ids, lgrp_limit);
225 assert(lgrp_num > 0, "There should be at least one locality group");
226 // Add new spaces for the new nodes
227 for (int i = 0; i < lgrp_num; i++) {
228 bool found = false;
229 for (int j = 0; j < lgrp_spaces()->length(); j++) {
230 if (lgrp_spaces()->at(j)->lgrp_id() == lgrp_ids[i]) {
231 found = true;
232 break;
233 }
234 }
235 if (!found) {
236 lgrp_spaces()->append(new LGRPSpace(lgrp_ids[i], alignment()));
237 }
238 }
239
240 // Remove spaces for the removed nodes.
241 for (int i = 0; i < lgrp_spaces()->length();) {
242 bool found = false;
243 for (int j = 0; j < lgrp_num; j++) {
244 if (lgrp_spaces()->at(i)->lgrp_id() == lgrp_ids[j]) {
245 found = true;
246 break;
247 }
248 }
249 if (!found) {
250 delete lgrp_spaces()->at(i);
251 lgrp_spaces()->remove_at(i);
252 } else {
253 i++;
254 }
255 }
256
257 FREE_C_HEAP_ARRAY(int, lgrp_ids);
258
259 if (changed) {
260 for (JavaThread *thread = Threads::first(); thread; thread = thread->next()) {
261 thread->set_lgrp_id(-1);
262 }
263 }
264 return true;
265 }
266 return false;
267 }
268
269 // Bias region towards the first-touching lgrp. Set the right page sizes.
270 void MutableNUMASpace::bias_region(MemRegion mr, int lgrp_id) {
271 HeapWord *start = (HeapWord*)round_to((intptr_t)mr.start(), page_size());
272 HeapWord *end = (HeapWord*)round_down((intptr_t)mr.end(), page_size());
273 if (end > start) {
274 MemRegion aligned_region(start, end);
275 assert((intptr_t)aligned_region.start() % page_size() == 0 &&
276 (intptr_t)aligned_region.byte_size() % page_size() == 0, "Bad alignment");
277 assert(region().contains(aligned_region), "Sanity");
278 // First we tell the OS which page size we want in the given range. The underlying
279 // large page can be broken down if we require small pages.
280 os::realign_memory((char*)aligned_region.start(), aligned_region.byte_size(), page_size());
281 // Then we uncommit the pages in the range.
282 os::free_memory((char*)aligned_region.start(), aligned_region.byte_size());
283 // And make them local/first-touch biased.
284 os::numa_make_local((char*)aligned_region.start(), aligned_region.byte_size(), lgrp_id);
285 }
286 }
287
288 // Free all pages in the region.
289 void MutableNUMASpace::free_region(MemRegion mr) {
290 HeapWord *start = (HeapWord*)round_to((intptr_t)mr.start(), page_size());
291 HeapWord *end = (HeapWord*)round_down((intptr_t)mr.end(), page_size());
292 if (end > start) {
293 MemRegion aligned_region(start, end);
294 assert((intptr_t)aligned_region.start() % page_size() == 0 &&
295 (intptr_t)aligned_region.byte_size() % page_size() == 0, "Bad alignment");
296 assert(region().contains(aligned_region), "Sanity");
297 os::free_memory((char*)aligned_region.start(), aligned_region.byte_size());
298 }
299 }
300
301 // Update space layout. Perform adaptation.
302 void MutableNUMASpace::update() {
303 if (update_layout(false)) {
304 // If the topology has changed, make all chunks zero-sized.
305 // And clear the alloc-rate statistics.
306 // In future we may want to handle this more gracefully in order
307 // to avoid the reallocation of the pages as much as possible.
308 for (int i = 0; i < lgrp_spaces()->length(); i++) {
309 LGRPSpace *ls = lgrp_spaces()->at(i);
310 MutableSpace *s = ls->space();
311 s->set_end(s->bottom());
312 s->set_top(s->bottom());
313 ls->clear_alloc_rate();
314 }
315 // A NUMA space is never mangled
316 initialize(region(),
317 SpaceDecorator::Clear,
318 SpaceDecorator::DontMangle);
319 } else {
320 bool should_initialize = false;
321 if (!os::numa_has_static_binding()) {
322 for (int i = 0; i < lgrp_spaces()->length(); i++) {
323 if (!lgrp_spaces()->at(i)->invalid_region().is_empty()) {
324 should_initialize = true;
325 break;
326 }
327 }
328 }
329
330 if (should_initialize ||
331 (UseAdaptiveNUMAChunkSizing && adaptation_cycles() < samples_count())) {
332 // A NUMA space is never mangled
333 initialize(region(),
334 SpaceDecorator::Clear,
335 SpaceDecorator::DontMangle);
336 }
337 }
338
339 if (NUMAStats) {
340 for (int i = 0; i < lgrp_spaces()->length(); i++) {
341 lgrp_spaces()->at(i)->accumulate_statistics(page_size());
342 }
343 }
344
345 scan_pages(NUMAPageScanRate);
346 }
347
348 // Scan pages. Free pages that have smaller size or wrong placement.
349 void MutableNUMASpace::scan_pages(size_t page_count)
350 {
351 size_t pages_per_chunk = page_count / lgrp_spaces()->length();
352 if (pages_per_chunk > 0) {
353 for (int i = 0; i < lgrp_spaces()->length(); i++) {
354 LGRPSpace *ls = lgrp_spaces()->at(i);
355 ls->scan_pages(page_size(), pages_per_chunk);
356 }
357 }
358 }
359
360 // Accumulate statistics about the allocation rate of each lgrp.
361 void MutableNUMASpace::accumulate_statistics() {
362 if (UseAdaptiveNUMAChunkSizing) {
363 for (int i = 0; i < lgrp_spaces()->length(); i++) {
364 lgrp_spaces()->at(i)->sample();
365 }
366 increment_samples_count();
367 }
368
369 if (NUMAStats) {
370 for (int i = 0; i < lgrp_spaces()->length(); i++) {
371 lgrp_spaces()->at(i)->accumulate_statistics(page_size());
372 }
373 }
374 }
375
376 // Get the current size of a chunk.
377 // This function computes the size of the chunk based on the
378 // difference between chunk ends. This allows it to work correctly in
379 // case the whole space is resized and during the process of adaptive
380 // chunk resizing.
381 size_t MutableNUMASpace::current_chunk_size(int i) {
382 HeapWord *cur_end, *prev_end;
383 if (i == 0) {
384 prev_end = bottom();
385 } else {
386 prev_end = lgrp_spaces()->at(i - 1)->space()->end();
387 }
388 if (i == lgrp_spaces()->length() - 1) {
389 cur_end = end();
390 } else {
391 cur_end = lgrp_spaces()->at(i)->space()->end();
392 }
393 if (cur_end > prev_end) {
394 return pointer_delta(cur_end, prev_end, sizeof(char));
395 }
396 return 0;
397 }
398
399 // Return the default chunk size by equally diving the space.
400 // page_size() aligned.
401 size_t MutableNUMASpace::default_chunk_size() {
402 return base_space_size() / lgrp_spaces()->length() * page_size();
403 }
404
405 // Produce a new chunk size. page_size() aligned.
406 // This function is expected to be called on sequence of i's from 0 to
407 // lgrp_spaces()->length().
408 size_t MutableNUMASpace::adaptive_chunk_size(int i, size_t limit) {
409 size_t pages_available = base_space_size();
410 for (int j = 0; j < i; j++) {
411 pages_available -= round_down(current_chunk_size(j), page_size()) / page_size();
412 }
413 pages_available -= lgrp_spaces()->length() - i - 1;
414 assert(pages_available > 0, "No pages left");
415 float alloc_rate = 0;
416 for (int j = i; j < lgrp_spaces()->length(); j++) {
417 alloc_rate += lgrp_spaces()->at(j)->alloc_rate()->average();
418 }
419 size_t chunk_size = 0;
420 if (alloc_rate > 0) {
421 LGRPSpace *ls = lgrp_spaces()->at(i);
422 chunk_size = (size_t)(ls->alloc_rate()->average() / alloc_rate * pages_available) * page_size();
423 }
424 chunk_size = MAX2(chunk_size, page_size());
425
426 if (limit > 0) {
427 limit = round_down(limit, page_size());
428 if (chunk_size > current_chunk_size(i)) {
429 size_t upper_bound = pages_available * page_size();
430 if (upper_bound > limit &&
431 current_chunk_size(i) < upper_bound - limit) {
432 // The resulting upper bound should not exceed the available
433 // amount of memory (pages_available * page_size()).
434 upper_bound = current_chunk_size(i) + limit;
435 }
436 chunk_size = MIN2(chunk_size, upper_bound);
437 } else {
438 size_t lower_bound = page_size();
439 if (current_chunk_size(i) > limit) { // lower_bound shouldn't underflow.
440 lower_bound = current_chunk_size(i) - limit;
441 }
442 chunk_size = MAX2(chunk_size, lower_bound);
443 }
444 }
445 assert(chunk_size <= pages_available * page_size(), "Chunk size out of range");
446 return chunk_size;
447 }
448
449
450 // Return the bottom_region and the top_region. Align them to page_size() boundary.
451 // |------------------new_region---------------------------------|
452 // |----bottom_region--|---intersection---|------top_region------|
453 void MutableNUMASpace::select_tails(MemRegion new_region, MemRegion intersection,
454 MemRegion* bottom_region, MemRegion *top_region) {
455 // Is there bottom?
456 if (new_region.start() < intersection.start()) { // Yes
457 // Try to coalesce small pages into a large one.
458 if (UseLargePages && page_size() >= alignment()) {
459 HeapWord* p = (HeapWord*)round_to((intptr_t) intersection.start(), alignment());
460 if (new_region.contains(p)
461 && pointer_delta(p, new_region.start(), sizeof(char)) >= alignment()) {
462 if (intersection.contains(p)) {
463 intersection = MemRegion(p, intersection.end());
464 } else {
465 intersection = MemRegion(p, p);
466 }
467 }
468 }
469 *bottom_region = MemRegion(new_region.start(), intersection.start());
470 } else {
471 *bottom_region = MemRegion();
472 }
473
474 // Is there top?
475 if (intersection.end() < new_region.end()) { // Yes
476 // Try to coalesce small pages into a large one.
477 if (UseLargePages && page_size() >= alignment()) {
478 HeapWord* p = (HeapWord*)round_down((intptr_t) intersection.end(), alignment());
479 if (new_region.contains(p)
480 && pointer_delta(new_region.end(), p, sizeof(char)) >= alignment()) {
481 if (intersection.contains(p)) {
482 intersection = MemRegion(intersection.start(), p);
483 } else {
484 intersection = MemRegion(p, p);
485 }
486 }
487 }
488 *top_region = MemRegion(intersection.end(), new_region.end());
489 } else {
490 *top_region = MemRegion();
491 }
492 }
493
494 // Try to merge the invalid region with the bottom or top region by decreasing
495 // the intersection area. Return the invalid_region aligned to the page_size()
496 // boundary if it's inside the intersection. Return non-empty invalid_region
497 // if it lies inside the intersection (also page-aligned).
498 // |------------------new_region---------------------------------|
499 // |----------------|-------invalid---|--------------------------|
500 // |----bottom_region--|---intersection---|------top_region------|
501 void MutableNUMASpace::merge_regions(MemRegion new_region, MemRegion* intersection,
502 MemRegion *invalid_region) {
503 if (intersection->start() >= invalid_region->start() && intersection->contains(invalid_region->end())) {
504 *intersection = MemRegion(invalid_region->end(), intersection->end());
505 *invalid_region = MemRegion();
506 } else
507 if (intersection->end() <= invalid_region->end() && intersection->contains(invalid_region->start())) {
508 *intersection = MemRegion(intersection->start(), invalid_region->start());
509 *invalid_region = MemRegion();
510 } else
511 if (intersection->equals(*invalid_region) || invalid_region->contains(*intersection)) {
512 *intersection = MemRegion(new_region.start(), new_region.start());
513 *invalid_region = MemRegion();
514 } else
515 if (intersection->contains(invalid_region)) {
516 // That's the only case we have to make an additional bias_region() call.
517 HeapWord* start = invalid_region->start();
518 HeapWord* end = invalid_region->end();
519 if (UseLargePages && page_size() >= alignment()) {
520 HeapWord *p = (HeapWord*)round_down((intptr_t) start, alignment());
521 if (new_region.contains(p)) {
522 start = p;
523 }
524 p = (HeapWord*)round_to((intptr_t) end, alignment());
525 if (new_region.contains(end)) {
526 end = p;
527 }
528 }
529 if (intersection->start() > start) {
530 *intersection = MemRegion(start, intersection->end());
531 }
532 if (intersection->end() < end) {
533 *intersection = MemRegion(intersection->start(), end);
534 }
535 *invalid_region = MemRegion(start, end);
536 }
537 }
538
539 void MutableNUMASpace::initialize(MemRegion mr,
540 bool clear_space,
541 bool mangle_space,
542 bool setup_pages) {
543 assert(clear_space, "Reallocation will destory data!");
544 assert(lgrp_spaces()->length() > 0, "There should be at least one space");
545
546 MemRegion old_region = region(), new_region;
547 set_bottom(mr.start());
548 set_end(mr.end());
549 // Must always clear the space
550 clear(SpaceDecorator::DontMangle);
551
552 // Compute chunk sizes
553 size_t prev_page_size = page_size();
554 set_page_size(UseLargePages ? alignment() : os::vm_page_size());
555 HeapWord* rounded_bottom = (HeapWord*)round_to((intptr_t) bottom(), page_size());
556 HeapWord* rounded_end = (HeapWord*)round_down((intptr_t) end(), page_size());
557 size_t base_space_size_pages = pointer_delta(rounded_end, rounded_bottom, sizeof(char)) / page_size();
558
559 // Try small pages if the chunk size is too small
560 if (base_space_size_pages / lgrp_spaces()->length() == 0
561 && page_size() > (size_t)os::vm_page_size()) {
562 set_page_size(os::vm_page_size());
563 rounded_bottom = (HeapWord*)round_to((intptr_t) bottom(), page_size());
564 rounded_end = (HeapWord*)round_down((intptr_t) end(), page_size());
565 base_space_size_pages = pointer_delta(rounded_end, rounded_bottom, sizeof(char)) / page_size();
566 }
567 guarantee(base_space_size_pages / lgrp_spaces()->length() > 0, "Space too small");
568 set_base_space_size(base_space_size_pages);
569
570 // Handle space resize
571 MemRegion top_region, bottom_region;
572 if (!old_region.equals(region())) {
573 new_region = MemRegion(rounded_bottom, rounded_end);
574 MemRegion intersection = new_region.intersection(old_region);
575 if (intersection.start() == NULL ||
576 intersection.end() == NULL ||
577 prev_page_size > page_size()) { // If the page size got smaller we have to change
578 // the page size preference for the whole space.
579 intersection = MemRegion(new_region.start(), new_region.start());
580 }
581 select_tails(new_region, intersection, &bottom_region, &top_region);
582 bias_region(bottom_region, lgrp_spaces()->at(0)->lgrp_id());
583 bias_region(top_region, lgrp_spaces()->at(lgrp_spaces()->length() - 1)->lgrp_id());
584 }
585
586 // Check if the space layout has changed significantly?
587 // This happens when the space has been resized so that either head or tail
588 // chunk became less than a page.
589 bool layout_valid = UseAdaptiveNUMAChunkSizing &&
590 current_chunk_size(0) > page_size() &&
591 current_chunk_size(lgrp_spaces()->length() - 1) > page_size();
592
593
594 for (int i = 0; i < lgrp_spaces()->length(); i++) {
595 LGRPSpace *ls = lgrp_spaces()->at(i);
596 MutableSpace *s = ls->space();
597 old_region = s->region();
598
599 size_t chunk_byte_size = 0, old_chunk_byte_size = 0;
600 if (i < lgrp_spaces()->length() - 1) {
601 if (!UseAdaptiveNUMAChunkSizing ||
602 (UseAdaptiveNUMAChunkSizing && NUMAChunkResizeWeight == 0) ||
603 samples_count() < AdaptiveSizePolicyReadyThreshold) {
604 // No adaptation. Divide the space equally.
605 chunk_byte_size = default_chunk_size();
606 } else
607 if (!layout_valid || NUMASpaceResizeRate == 0) {
608 // Fast adaptation. If no space resize rate is set, resize
609 // the chunks instantly.
610 chunk_byte_size = adaptive_chunk_size(i, 0);
611 } else {
612 // Slow adaptation. Resize the chunks moving no more than
613 // NUMASpaceResizeRate bytes per collection.
614 size_t limit = NUMASpaceResizeRate /
615 (lgrp_spaces()->length() * (lgrp_spaces()->length() + 1) / 2);
616 chunk_byte_size = adaptive_chunk_size(i, MAX2(limit * (i + 1), page_size()));
617 }
618
619 assert(chunk_byte_size >= page_size(), "Chunk size too small");
620 assert(chunk_byte_size <= capacity_in_bytes(), "Sanity check");
621 }
622
623 if (i == 0) { // Bottom chunk
624 if (i != lgrp_spaces()->length() - 1) {
625 new_region = MemRegion(bottom(), rounded_bottom + (chunk_byte_size >> LogHeapWordSize));
626 } else {
627 new_region = MemRegion(bottom(), end());
628 }
629 } else
630 if (i < lgrp_spaces()->length() - 1) { // Middle chunks
631 MutableSpace *ps = lgrp_spaces()->at(i - 1)->space();
632 new_region = MemRegion(ps->end(),
633 ps->end() + (chunk_byte_size >> LogHeapWordSize));
634 } else { // Top chunk
635 MutableSpace *ps = lgrp_spaces()->at(i - 1)->space();
636 new_region = MemRegion(ps->end(), end());
637 }
638 guarantee(region().contains(new_region), "Region invariant");
639
640
641 // The general case:
642 // |---------------------|--invalid---|--------------------------|
643 // |------------------new_region---------------------------------|
644 // |----bottom_region--|---intersection---|------top_region------|
645 // |----old_region----|
646 // The intersection part has all pages in place we don't need to migrate them.
647 // Pages for the top and bottom part should be freed and then reallocated.
648
649 MemRegion intersection = old_region.intersection(new_region);
650
651 if (intersection.start() == NULL || intersection.end() == NULL) {
652 intersection = MemRegion(new_region.start(), new_region.start());
653 }
654
655 if (!os::numa_has_static_binding()) {
656 MemRegion invalid_region = ls->invalid_region().intersection(new_region);
657 // Invalid region is a range of memory that could've possibly
658 // been allocated on the other node. That's relevant only on Solaris where
659 // there is no static memory binding.
660 if (!invalid_region.is_empty()) {
661 merge_regions(new_region, &intersection, &invalid_region);
662 free_region(invalid_region);
663 ls->set_invalid_region(MemRegion());
664 }
665 }
666
667 select_tails(new_region, intersection, &bottom_region, &top_region);
668
669 if (!os::numa_has_static_binding()) {
670 // If that's a system with the first-touch policy then it's enough
671 // to free the pages.
672 free_region(bottom_region);
673 free_region(top_region);
674 } else {
675 // In a system with static binding we have to change the bias whenever
676 // we reshape the heap.
677 bias_region(bottom_region, ls->lgrp_id());
678 bias_region(top_region, ls->lgrp_id());
679 }
680
681 // Clear space (set top = bottom) but never mangle.
682 s->initialize(new_region, SpaceDecorator::Clear, SpaceDecorator::DontMangle, MutableSpace::DontSetupPages);
683
684 set_adaptation_cycles(samples_count());
685 }
686 }
687
688 // Set the top of the whole space.
689 // Mark the the holes in chunks below the top() as invalid.
690 void MutableNUMASpace::set_top(HeapWord* value) {
691 bool found_top = false;
692 for (int i = 0; i < lgrp_spaces()->length();) {
693 LGRPSpace *ls = lgrp_spaces()->at(i);
694 MutableSpace *s = ls->space();
695 HeapWord *top = MAX2((HeapWord*)round_down((intptr_t)s->top(), page_size()), s->bottom());
696
697 if (s->contains(value)) {
698 // Check if setting the chunk's top to a given value would create a hole less than
699 // a minimal object; assuming that's not the last chunk in which case we don't care.
700 if (i < lgrp_spaces()->length() - 1) {
701 size_t remainder = pointer_delta(s->end(), value);
702 const size_t min_fill_size = CollectedHeap::min_fill_size();
703 if (remainder < min_fill_size && remainder > 0) {
704 // Add a minimum size filler object; it will cross the chunk boundary.
705 CollectedHeap::fill_with_object(value, min_fill_size);
706 value += min_fill_size;
707 assert(!s->contains(value), "Should be in the next chunk");
708 // Restart the loop from the same chunk, since the value has moved
709 // to the next one.
710 continue;
711 }
712 }
713
714 if (!os::numa_has_static_binding() && top < value && top < s->end()) {
715 ls->add_invalid_region(MemRegion(top, value));
716 }
717 s->set_top(value);
718 found_top = true;
719 } else {
720 if (found_top) {
721 s->set_top(s->bottom());
722 } else {
723 if (!os::numa_has_static_binding() && top < s->end()) {
724 ls->add_invalid_region(MemRegion(top, s->end()));
725 }
726 s->set_top(s->end());
727 }
728 }
729 i++;
730 }
731 MutableSpace::set_top(value);
732 }
733
734 void MutableNUMASpace::clear(bool mangle_space) {
735 MutableSpace::set_top(bottom());
736 for (int i = 0; i < lgrp_spaces()->length(); i++) {
737 // Never mangle NUMA spaces because the mangling will
738 // bind the memory to a possibly unwanted lgroup.
739 lgrp_spaces()->at(i)->space()->clear(SpaceDecorator::DontMangle);
740 }
741 }
742
743 /*
744 Linux supports static memory binding, therefore the most part of the
745 logic dealing with the possible invalid page allocation is effectively
746 disabled. Besides there is no notion of the home node in Linux. A
747 thread is allowed to migrate freely. Although the scheduler is rather
748 reluctant to move threads between the nodes. We check for the current
749 node every allocation. And with a high probability a thread stays on
750 the same node for some time allowing local access to recently allocated
751 objects.
752 */
753
754 HeapWord* MutableNUMASpace::allocate(size_t size) {
755 Thread* thr = Thread::current();
756 int lgrp_id = thr->lgrp_id();
757 if (lgrp_id == -1 || !os::numa_has_group_homing()) {
758 lgrp_id = os::numa_get_group_id();
759 thr->set_lgrp_id(lgrp_id);
760 }
761
762 int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals);
763
764 // It is possible that a new CPU has been hotplugged and
765 // we haven't reshaped the space accordingly.
766 if (i == -1) {
767 i = os::random() % lgrp_spaces()->length();
768 }
769
770 LGRPSpace* ls = lgrp_spaces()->at(i);
771 MutableSpace *s = ls->space();
772 HeapWord *p = s->allocate(size);
773
774 if (p != NULL) {
775 size_t remainder = s->free_in_words();
776 if (remainder < CollectedHeap::min_fill_size() && remainder > 0) {
777 s->set_top(s->top() - size);
778 p = NULL;
779 }
780 }
781 if (p != NULL) {
782 if (top() < s->top()) { // Keep _top updated.
783 MutableSpace::set_top(s->top());
784 }
785 }
786 // Make the page allocation happen here if there is no static binding..
787 if (p != NULL && !os::numa_has_static_binding()) {
788 for (HeapWord *i = p; i < p + size; i += os::vm_page_size() >> LogHeapWordSize) {
789 *(int*)i = 0;
790 }
791 }
792 if (p == NULL) {
793 ls->set_allocation_failed();
794 }
795 return p;
796 }
797
798 // This version is lock-free.
799 HeapWord* MutableNUMASpace::cas_allocate(size_t size) {
800 Thread* thr = Thread::current();
801 int lgrp_id = thr->lgrp_id();
802 if (lgrp_id == -1 || !os::numa_has_group_homing()) {
803 lgrp_id = os::numa_get_group_id();
804 thr->set_lgrp_id(lgrp_id);
805 }
806
807 int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals);
808 // It is possible that a new CPU has been hotplugged and
809 // we haven't reshaped the space accordingly.
810 if (i == -1) {
811 i = os::random() % lgrp_spaces()->length();
812 }
813 LGRPSpace *ls = lgrp_spaces()->at(i);
814 MutableSpace *s = ls->space();
815 HeapWord *p = s->cas_allocate(size);
816 if (p != NULL) {
817 size_t remainder = pointer_delta(s->end(), p + size);
818 if (remainder < CollectedHeap::min_fill_size() && remainder > 0) {
819 if (s->cas_deallocate(p, size)) {
820 // We were the last to allocate and created a fragment less than
821 // a minimal object.
822 p = NULL;
823 } else {
824 guarantee(false, "Deallocation should always succeed");
825 }
826 }
827 }
828 if (p != NULL) {
829 HeapWord* cur_top, *cur_chunk_top = p + size;
830 while ((cur_top = top()) < cur_chunk_top) { // Keep _top updated.
831 if (Atomic::cmpxchg_ptr(cur_chunk_top, top_addr(), cur_top) == cur_top) {
832 break;
833 }
834 }
835 }
836
837 // Make the page allocation happen here if there is no static binding.
838 if (p != NULL && !os::numa_has_static_binding() ) {
839 for (HeapWord *i = p; i < p + size; i += os::vm_page_size() >> LogHeapWordSize) {
840 *(int*)i = 0;
841 }
842 }
843 if (p == NULL) {
844 ls->set_allocation_failed();
845 }
846 return p;
847 }
848
849 void MutableNUMASpace::print_short_on(outputStream* st) const {
850 MutableSpace::print_short_on(st);
851 st->print(" (");
852 for (int i = 0; i < lgrp_spaces()->length(); i++) {
853 st->print("lgrp %d: ", lgrp_spaces()->at(i)->lgrp_id());
854 lgrp_spaces()->at(i)->space()->print_short_on(st);
855 if (i < lgrp_spaces()->length() - 1) {
856 st->print(", ");
857 }
858 }
859 st->print(")");
860 }
861
862 void MutableNUMASpace::print_on(outputStream* st) const {
863 MutableSpace::print_on(st);
864 for (int i = 0; i < lgrp_spaces()->length(); i++) {
865 LGRPSpace *ls = lgrp_spaces()->at(i);
866 st->print(" lgrp %d", ls->lgrp_id());
867 ls->space()->print_on(st);
868 if (NUMAStats) {
869 for (int i = 0; i < lgrp_spaces()->length(); i++) {
870 lgrp_spaces()->at(i)->accumulate_statistics(page_size());
871 }
872 st->print(" local/remote/unbiased/uncommitted: %dK/%dK/%dK/%dK, large/small pages: %d/%d\n",
873 ls->space_stats()->_local_space / K,
874 ls->space_stats()->_remote_space / K,
875 ls->space_stats()->_unbiased_space / K,
876 ls->space_stats()->_uncommited_space / K,
877 ls->space_stats()->_large_pages,
878 ls->space_stats()->_small_pages);
879 }
880 }
881 }
882
883 void MutableNUMASpace::verify(bool allow_dirty) {
884 // This can be called after setting an arbitary value to the space's top,
885 // so an object can cross the chunk boundary. We ensure the parsablity
886 // of the space and just walk the objects in linear fashion.
887 ensure_parsability();
888 MutableSpace::verify(allow_dirty);
889 }
890
891 // Scan pages and gather stats about page placement and size.
892 void MutableNUMASpace::LGRPSpace::accumulate_statistics(size_t page_size) {
893 clear_space_stats();
894 char *start = (char*)round_to((intptr_t) space()->bottom(), page_size);
895 char* end = (char*)round_down((intptr_t) space()->end(), page_size);
896 if (start < end) {
897 for (char *p = start; p < end;) {
898 os::page_info info;
899 if (os::get_page_info(p, &info)) {
900 if (info.size > 0) {
901 if (info.size > (size_t)os::vm_page_size()) {
902 space_stats()->_large_pages++;
903 } else {
904 space_stats()->_small_pages++;
905 }
906 if (info.lgrp_id == lgrp_id()) {
907 space_stats()->_local_space += info.size;
908 } else {
909 space_stats()->_remote_space += info.size;
910 }
911 p += info.size;
912 } else {
913 p += os::vm_page_size();
914 space_stats()->_uncommited_space += os::vm_page_size();
915 }
916 } else {
917 return;
918 }
919 }
920 }
921 space_stats()->_unbiased_space = pointer_delta(start, space()->bottom(), sizeof(char)) +
922 pointer_delta(space()->end(), end, sizeof(char));
923
924 }
925
926 // Scan page_count pages and verify if they have the right size and right placement.
927 // If invalid pages are found they are freed in hope that subsequent reallocation
928 // will be more successful.
929 void MutableNUMASpace::LGRPSpace::scan_pages(size_t page_size, size_t page_count)
930 {
931 char* range_start = (char*)round_to((intptr_t) space()->bottom(), page_size);
932 char* range_end = (char*)round_down((intptr_t) space()->end(), page_size);
933
934 if (range_start > last_page_scanned() || last_page_scanned() >= range_end) {
935 set_last_page_scanned(range_start);
936 }
937
938 char *scan_start = last_page_scanned();
939 char* scan_end = MIN2(scan_start + page_size * page_count, range_end);
940
941 os::page_info page_expected, page_found;
942 page_expected.size = page_size;
943 page_expected.lgrp_id = lgrp_id();
944
945 char *s = scan_start;
946 while (s < scan_end) {
947 char *e = os::scan_pages(s, (char*)scan_end, &page_expected, &page_found);
948 if (e == NULL) {
949 break;
950 }
951 if (e != scan_end) {
952 if ((page_expected.size != page_size || page_expected.lgrp_id != lgrp_id())
953 && page_expected.size != 0) {
954 os::free_memory(s, pointer_delta(e, s, sizeof(char)));
955 }
956 page_expected = page_found;
957 }
958 s = e;
959 }
960
961 set_last_page_scanned(scan_end);
962 }