1 /* 2 * Copyright (c) 2006, 2016, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 * 23 */ 24 25 #include "precompiled.hpp" 26 #include "gc/parallel/mutableNUMASpace.hpp" 27 #include "gc/shared/collectedHeap.hpp" 28 #include "gc/shared/spaceDecorator.hpp" 29 #include "oops/oop.inline.hpp" 30 #include "runtime/atomic.hpp" 31 #include "runtime/thread.inline.hpp" 32 #include "utilities/align.hpp" 33 34 MutableNUMASpace::MutableNUMASpace(size_t alignment) : MutableSpace(alignment), _must_use_large_pages(false) { 35 _lgrp_spaces = new (ResourceObj::C_HEAP, mtGC) GrowableArray<LGRPSpace*>(0, true); 36 _page_size = os::vm_page_size(); 37 _adaptation_cycles = 0; 38 _samples_count = 0; 39 40 #ifdef LINUX 41 // Changing the page size can lead to freeing of memory. When using large pages 42 // and the memory has been both reserved and committed, Linux does not support 43 // freeing parts of it. 44 if (UseLargePages && !os::can_commit_large_page_memory()) { 45 _must_use_large_pages = true; 46 } 47 #endif // LINUX 48 49 update_layout(true); 50 } 51 52 MutableNUMASpace::~MutableNUMASpace() { 53 for (int i = 0; i < lgrp_spaces()->length(); i++) { 54 delete lgrp_spaces()->at(i); 55 } 56 delete lgrp_spaces(); 57 } 58 59 #ifndef PRODUCT 60 void MutableNUMASpace::mangle_unused_area() { 61 // This method should do nothing. 62 // It can be called on a numa space during a full compaction. 63 } 64 void MutableNUMASpace::mangle_unused_area_complete() { 65 // This method should do nothing. 66 // It can be called on a numa space during a full compaction. 67 } 68 void MutableNUMASpace::mangle_region(MemRegion mr) { 69 // This method should do nothing because numa spaces are not mangled. 70 } 71 void MutableNUMASpace::set_top_for_allocations(HeapWord* v) { 72 assert(false, "Do not mangle MutableNUMASpace's"); 73 } 74 void MutableNUMASpace::set_top_for_allocations() { 75 // This method should do nothing. 76 } 77 void MutableNUMASpace::check_mangled_unused_area(HeapWord* limit) { 78 // This method should do nothing. 79 } 80 void MutableNUMASpace::check_mangled_unused_area_complete() { 81 // This method should do nothing. 82 } 83 #endif // NOT_PRODUCT 84 85 // There may be unallocated holes in the middle chunks 86 // that should be filled with dead objects to ensure parsability. 87 void MutableNUMASpace::ensure_parsability() { 88 for (int i = 0; i < lgrp_spaces()->length(); i++) { 89 LGRPSpace *ls = lgrp_spaces()->at(i); 90 MutableSpace *s = ls->space(); 91 if (s->top() < top()) { // For all spaces preceding the one containing top() 92 if (s->free_in_words() > 0) { 93 intptr_t cur_top = (intptr_t)s->top(); 94 size_t words_left_to_fill = pointer_delta(s->end(), s->top());; 95 while (words_left_to_fill > 0) { 96 size_t words_to_fill = MIN2(words_left_to_fill, CollectedHeap::filler_array_max_size()); 97 assert(words_to_fill >= CollectedHeap::min_fill_size(), 98 "Remaining size (" SIZE_FORMAT ") is too small to fill (based on " SIZE_FORMAT " and " SIZE_FORMAT ")", 99 words_to_fill, words_left_to_fill, CollectedHeap::filler_array_max_size()); 100 CollectedHeap::fill_with_object((HeapWord*)cur_top, words_to_fill); 101 if (!os::numa_has_static_binding()) { 102 size_t touched_words = words_to_fill; 103 #ifndef ASSERT 104 if (!ZapUnusedHeapArea) { 105 touched_words = MIN2((size_t)align_object_size(typeArrayOopDesc::header_size(T_INT)), 106 touched_words); 107 } 108 #endif 109 MemRegion invalid; 110 HeapWord *crossing_start = align_up((HeapWord*)cur_top, os::vm_page_size()); 111 HeapWord *crossing_end = align_down((HeapWord*)(cur_top + touched_words), os::vm_page_size()); 112 if (crossing_start != crossing_end) { 113 // If object header crossed a small page boundary we mark the area 114 // as invalid rounding it to a page_size(). 115 HeapWord *start = MAX2(align_down((HeapWord*)cur_top, page_size()), s->bottom()); 116 HeapWord *end = MIN2(align_up((HeapWord*)(cur_top + touched_words), page_size()), s->end()); 117 invalid = MemRegion(start, end); 118 } 119 120 ls->add_invalid_region(invalid); 121 } 122 cur_top = cur_top + (words_to_fill * HeapWordSize); 123 words_left_to_fill -= words_to_fill; 124 } 125 } 126 } else { 127 if (!os::numa_has_static_binding()) { 128 #ifdef ASSERT 129 MemRegion invalid(s->top(), s->end()); 130 ls->add_invalid_region(invalid); 131 #else 132 if (ZapUnusedHeapArea) { 133 MemRegion invalid(s->top(), s->end()); 134 ls->add_invalid_region(invalid); 135 } else { 136 return; 137 } 138 #endif 139 } else { 140 return; 141 } 142 } 143 } 144 } 145 146 size_t MutableNUMASpace::used_in_words() const { 147 size_t s = 0; 148 for (int i = 0; i < lgrp_spaces()->length(); i++) { 149 s += lgrp_spaces()->at(i)->space()->used_in_words(); 150 } 151 return s; 152 } 153 154 size_t MutableNUMASpace::free_in_words() const { 155 size_t s = 0; 156 for (int i = 0; i < lgrp_spaces()->length(); i++) { 157 s += lgrp_spaces()->at(i)->space()->free_in_words(); 158 } 159 return s; 160 } 161 162 163 size_t MutableNUMASpace::tlab_capacity(Thread *thr) const { 164 guarantee(thr != NULL, "No thread"); 165 int lgrp_id = thr->lgrp_id(); 166 if (lgrp_id == -1) { 167 // This case can occur after the topology of the system has 168 // changed. Thread can change their location, the new home 169 // group will be determined during the first allocation 170 // attempt. For now we can safely assume that all spaces 171 // have equal size because the whole space will be reinitialized. 172 if (lgrp_spaces()->length() > 0) { 173 return capacity_in_bytes() / lgrp_spaces()->length(); 174 } else { 175 assert(false, "There should be at least one locality group"); 176 return 0; 177 } 178 } 179 // That's the normal case, where we know the locality group of the thread. 180 int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals); 181 if (i == -1) { 182 return 0; 183 } 184 return lgrp_spaces()->at(i)->space()->capacity_in_bytes(); 185 } 186 187 size_t MutableNUMASpace::tlab_used(Thread *thr) const { 188 // Please see the comments for tlab_capacity(). 189 guarantee(thr != NULL, "No thread"); 190 int lgrp_id = thr->lgrp_id(); 191 if (lgrp_id == -1) { 192 if (lgrp_spaces()->length() > 0) { 193 return (used_in_bytes()) / lgrp_spaces()->length(); 194 } else { 195 assert(false, "There should be at least one locality group"); 196 return 0; 197 } 198 } 199 int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals); 200 if (i == -1) { 201 return 0; 202 } 203 return lgrp_spaces()->at(i)->space()->used_in_bytes(); 204 } 205 206 207 size_t MutableNUMASpace::unsafe_max_tlab_alloc(Thread *thr) const { 208 // Please see the comments for tlab_capacity(). 209 guarantee(thr != NULL, "No thread"); 210 int lgrp_id = thr->lgrp_id(); 211 if (lgrp_id == -1) { 212 if (lgrp_spaces()->length() > 0) { 213 return free_in_bytes() / lgrp_spaces()->length(); 214 } else { 215 assert(false, "There should be at least one locality group"); 216 return 0; 217 } 218 } 219 int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals); 220 if (i == -1) { 221 return 0; 222 } 223 return lgrp_spaces()->at(i)->space()->free_in_bytes(); 224 } 225 226 227 size_t MutableNUMASpace::capacity_in_words(Thread* thr) const { 228 guarantee(thr != NULL, "No thread"); 229 int lgrp_id = thr->lgrp_id(); 230 if (lgrp_id == -1) { 231 if (lgrp_spaces()->length() > 0) { 232 return capacity_in_words() / lgrp_spaces()->length(); 233 } else { 234 assert(false, "There should be at least one locality group"); 235 return 0; 236 } 237 } 238 int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals); 239 if (i == -1) { 240 return 0; 241 } 242 return lgrp_spaces()->at(i)->space()->capacity_in_words(); 243 } 244 245 // Check if the NUMA topology has changed. Add and remove spaces if needed. 246 // The update can be forced by setting the force parameter equal to true. 247 bool MutableNUMASpace::update_layout(bool force) { 248 // Check if the topology had changed. 249 bool changed = os::numa_topology_changed(); 250 if (force || changed) { 251 // Compute lgrp intersection. Add/remove spaces. 252 int lgrp_limit = (int)os::numa_get_groups_num(); 253 int *lgrp_ids = NEW_C_HEAP_ARRAY(int, lgrp_limit, mtGC); 254 int lgrp_num = (int)os::numa_get_leaf_groups(lgrp_ids, lgrp_limit); 255 assert(lgrp_num > 0, "There should be at least one locality group"); 256 // Add new spaces for the new nodes 257 for (int i = 0; i < lgrp_num; i++) { 258 bool found = false; 259 for (int j = 0; j < lgrp_spaces()->length(); j++) { 260 if (lgrp_spaces()->at(j)->lgrp_id() == lgrp_ids[i]) { 261 found = true; 262 break; 263 } 264 } 265 if (!found) { 266 lgrp_spaces()->append(new LGRPSpace(lgrp_ids[i], alignment())); 267 } 268 } 269 270 // Remove spaces for the removed nodes. 271 for (int i = 0; i < lgrp_spaces()->length();) { 272 bool found = false; 273 for (int j = 0; j < lgrp_num; j++) { 274 if (lgrp_spaces()->at(i)->lgrp_id() == lgrp_ids[j]) { 275 found = true; 276 break; 277 } 278 } 279 if (!found) { 280 delete lgrp_spaces()->at(i); 281 lgrp_spaces()->remove_at(i); 282 } else { 283 i++; 284 } 285 } 286 287 FREE_C_HEAP_ARRAY(int, lgrp_ids); 288 289 if (changed) { 290 for (JavaThread *thread = Threads::first(); thread; thread = thread->next()) { 291 thread->set_lgrp_id(-1); 292 } 293 } 294 return true; 295 } 296 return false; 297 } 298 299 // Bias region towards the first-touching lgrp. Set the right page sizes. 300 void MutableNUMASpace::bias_region(MemRegion mr, int lgrp_id) { 301 HeapWord *start = align_up(mr.start(), page_size()); 302 HeapWord *end = align_down(mr.end(), page_size()); 303 if (end > start) { 304 MemRegion aligned_region(start, end); 305 assert((intptr_t)aligned_region.start() % page_size() == 0 && 306 (intptr_t)aligned_region.byte_size() % page_size() == 0, "Bad alignment"); 307 assert(region().contains(aligned_region), "Sanity"); 308 // First we tell the OS which page size we want in the given range. The underlying 309 // large page can be broken down if we require small pages. 310 os::realign_memory((char*)aligned_region.start(), aligned_region.byte_size(), page_size()); 311 // Then we uncommit the pages in the range. 312 os::free_memory((char*)aligned_region.start(), aligned_region.byte_size(), page_size()); 313 // And make them local/first-touch biased. 314 os::numa_make_local((char*)aligned_region.start(), aligned_region.byte_size(), lgrp_id); 315 } 316 } 317 318 // Free all pages in the region. 319 void MutableNUMASpace::free_region(MemRegion mr) { 320 HeapWord *start = align_up(mr.start(), page_size()); 321 HeapWord *end = align_down(mr.end(), page_size()); 322 if (end > start) { 323 MemRegion aligned_region(start, end); 324 assert((intptr_t)aligned_region.start() % page_size() == 0 && 325 (intptr_t)aligned_region.byte_size() % page_size() == 0, "Bad alignment"); 326 assert(region().contains(aligned_region), "Sanity"); 327 os::free_memory((char*)aligned_region.start(), aligned_region.byte_size(), page_size()); 328 } 329 } 330 331 // Update space layout. Perform adaptation. 332 void MutableNUMASpace::update() { 333 if (update_layout(false)) { 334 // If the topology has changed, make all chunks zero-sized. 335 // And clear the alloc-rate statistics. 336 // In future we may want to handle this more gracefully in order 337 // to avoid the reallocation of the pages as much as possible. 338 for (int i = 0; i < lgrp_spaces()->length(); i++) { 339 LGRPSpace *ls = lgrp_spaces()->at(i); 340 MutableSpace *s = ls->space(); 341 s->set_end(s->bottom()); 342 s->set_top(s->bottom()); 343 ls->clear_alloc_rate(); 344 } 345 // A NUMA space is never mangled 346 initialize(region(), 347 SpaceDecorator::Clear, 348 SpaceDecorator::DontMangle); 349 } else { 350 bool should_initialize = false; 351 if (!os::numa_has_static_binding()) { 352 for (int i = 0; i < lgrp_spaces()->length(); i++) { 353 if (!lgrp_spaces()->at(i)->invalid_region().is_empty()) { 354 should_initialize = true; 355 break; 356 } 357 } 358 } 359 360 if (should_initialize || 361 (UseAdaptiveNUMAChunkSizing && adaptation_cycles() < samples_count())) { 362 // A NUMA space is never mangled 363 initialize(region(), 364 SpaceDecorator::Clear, 365 SpaceDecorator::DontMangle); 366 } 367 } 368 369 if (NUMAStats) { 370 for (int i = 0; i < lgrp_spaces()->length(); i++) { 371 lgrp_spaces()->at(i)->accumulate_statistics(page_size()); 372 } 373 } 374 375 scan_pages(NUMAPageScanRate); 376 } 377 378 // Scan pages. Free pages that have smaller size or wrong placement. 379 void MutableNUMASpace::scan_pages(size_t page_count) 380 { 381 size_t pages_per_chunk = page_count / lgrp_spaces()->length(); 382 if (pages_per_chunk > 0) { 383 for (int i = 0; i < lgrp_spaces()->length(); i++) { 384 LGRPSpace *ls = lgrp_spaces()->at(i); 385 ls->scan_pages(page_size(), pages_per_chunk); 386 } 387 } 388 } 389 390 // Accumulate statistics about the allocation rate of each lgrp. 391 void MutableNUMASpace::accumulate_statistics() { 392 if (UseAdaptiveNUMAChunkSizing) { 393 for (int i = 0; i < lgrp_spaces()->length(); i++) { 394 lgrp_spaces()->at(i)->sample(); 395 } 396 increment_samples_count(); 397 } 398 399 if (NUMAStats) { 400 for (int i = 0; i < lgrp_spaces()->length(); i++) { 401 lgrp_spaces()->at(i)->accumulate_statistics(page_size()); 402 } 403 } 404 } 405 406 // Get the current size of a chunk. 407 // This function computes the size of the chunk based on the 408 // difference between chunk ends. This allows it to work correctly in 409 // case the whole space is resized and during the process of adaptive 410 // chunk resizing. 411 size_t MutableNUMASpace::current_chunk_size(int i) { 412 HeapWord *cur_end, *prev_end; 413 if (i == 0) { 414 prev_end = bottom(); 415 } else { 416 prev_end = lgrp_spaces()->at(i - 1)->space()->end(); 417 } 418 if (i == lgrp_spaces()->length() - 1) { 419 cur_end = end(); 420 } else { 421 cur_end = lgrp_spaces()->at(i)->space()->end(); 422 } 423 if (cur_end > prev_end) { 424 return pointer_delta(cur_end, prev_end, sizeof(char)); 425 } 426 return 0; 427 } 428 429 // Return the default chunk size by equally diving the space. 430 // page_size() aligned. 431 size_t MutableNUMASpace::default_chunk_size() { 432 return base_space_size() / lgrp_spaces()->length() * page_size(); 433 } 434 435 // Produce a new chunk size. page_size() aligned. 436 // This function is expected to be called on sequence of i's from 0 to 437 // lgrp_spaces()->length(). 438 size_t MutableNUMASpace::adaptive_chunk_size(int i, size_t limit) { 439 size_t pages_available = base_space_size(); 440 for (int j = 0; j < i; j++) { 441 pages_available -= align_down(current_chunk_size(j), page_size()) / page_size(); 442 } 443 pages_available -= lgrp_spaces()->length() - i - 1; 444 assert(pages_available > 0, "No pages left"); 445 float alloc_rate = 0; 446 for (int j = i; j < lgrp_spaces()->length(); j++) { 447 alloc_rate += lgrp_spaces()->at(j)->alloc_rate()->average(); 448 } 449 size_t chunk_size = 0; 450 if (alloc_rate > 0) { 451 LGRPSpace *ls = lgrp_spaces()->at(i); 452 chunk_size = (size_t)(ls->alloc_rate()->average() / alloc_rate * pages_available) * page_size(); 453 } 454 chunk_size = MAX2(chunk_size, page_size()); 455 456 if (limit > 0) { 457 limit = align_down(limit, page_size()); 458 if (chunk_size > current_chunk_size(i)) { 459 size_t upper_bound = pages_available * page_size(); 460 if (upper_bound > limit && 461 current_chunk_size(i) < upper_bound - limit) { 462 // The resulting upper bound should not exceed the available 463 // amount of memory (pages_available * page_size()). 464 upper_bound = current_chunk_size(i) + limit; 465 } 466 chunk_size = MIN2(chunk_size, upper_bound); 467 } else { 468 size_t lower_bound = page_size(); 469 if (current_chunk_size(i) > limit) { // lower_bound shouldn't underflow. 470 lower_bound = current_chunk_size(i) - limit; 471 } 472 chunk_size = MAX2(chunk_size, lower_bound); 473 } 474 } 475 assert(chunk_size <= pages_available * page_size(), "Chunk size out of range"); 476 return chunk_size; 477 } 478 479 480 // Return the bottom_region and the top_region. Align them to page_size() boundary. 481 // |------------------new_region---------------------------------| 482 // |----bottom_region--|---intersection---|------top_region------| 483 void MutableNUMASpace::select_tails(MemRegion new_region, MemRegion intersection, 484 MemRegion* bottom_region, MemRegion *top_region) { 485 // Is there bottom? 486 if (new_region.start() < intersection.start()) { // Yes 487 // Try to coalesce small pages into a large one. 488 if (UseLargePages && page_size() >= alignment()) { 489 HeapWord* p = align_up(intersection.start(), alignment()); 490 if (new_region.contains(p) 491 && pointer_delta(p, new_region.start(), sizeof(char)) >= alignment()) { 492 if (intersection.contains(p)) { 493 intersection = MemRegion(p, intersection.end()); 494 } else { 495 intersection = MemRegion(p, p); 496 } 497 } 498 } 499 *bottom_region = MemRegion(new_region.start(), intersection.start()); 500 } else { 501 *bottom_region = MemRegion(); 502 } 503 504 // Is there top? 505 if (intersection.end() < new_region.end()) { // Yes 506 // Try to coalesce small pages into a large one. 507 if (UseLargePages && page_size() >= alignment()) { 508 HeapWord* p = align_down(intersection.end(), alignment()); 509 if (new_region.contains(p) 510 && pointer_delta(new_region.end(), p, sizeof(char)) >= alignment()) { 511 if (intersection.contains(p)) { 512 intersection = MemRegion(intersection.start(), p); 513 } else { 514 intersection = MemRegion(p, p); 515 } 516 } 517 } 518 *top_region = MemRegion(intersection.end(), new_region.end()); 519 } else { 520 *top_region = MemRegion(); 521 } 522 } 523 524 // Try to merge the invalid region with the bottom or top region by decreasing 525 // the intersection area. Return the invalid_region aligned to the page_size() 526 // boundary if it's inside the intersection. Return non-empty invalid_region 527 // if it lies inside the intersection (also page-aligned). 528 // |------------------new_region---------------------------------| 529 // |----------------|-------invalid---|--------------------------| 530 // |----bottom_region--|---intersection---|------top_region------| 531 void MutableNUMASpace::merge_regions(MemRegion new_region, MemRegion* intersection, 532 MemRegion *invalid_region) { 533 if (intersection->start() >= invalid_region->start() && intersection->contains(invalid_region->end())) { 534 *intersection = MemRegion(invalid_region->end(), intersection->end()); 535 *invalid_region = MemRegion(); 536 } else 537 if (intersection->end() <= invalid_region->end() && intersection->contains(invalid_region->start())) { 538 *intersection = MemRegion(intersection->start(), invalid_region->start()); 539 *invalid_region = MemRegion(); 540 } else 541 if (intersection->equals(*invalid_region) || invalid_region->contains(*intersection)) { 542 *intersection = MemRegion(new_region.start(), new_region.start()); 543 *invalid_region = MemRegion(); 544 } else 545 if (intersection->contains(invalid_region)) { 546 // That's the only case we have to make an additional bias_region() call. 547 HeapWord* start = invalid_region->start(); 548 HeapWord* end = invalid_region->end(); 549 if (UseLargePages && page_size() >= alignment()) { 550 HeapWord *p = align_down(start, alignment()); 551 if (new_region.contains(p)) { 552 start = p; 553 } 554 p = align_up(end, alignment()); 555 if (new_region.contains(end)) { 556 end = p; 557 } 558 } 559 if (intersection->start() > start) { 560 *intersection = MemRegion(start, intersection->end()); 561 } 562 if (intersection->end() < end) { 563 *intersection = MemRegion(intersection->start(), end); 564 } 565 *invalid_region = MemRegion(start, end); 566 } 567 } 568 569 void MutableNUMASpace::initialize(MemRegion mr, 570 bool clear_space, 571 bool mangle_space, 572 bool setup_pages) { 573 assert(clear_space, "Reallocation will destroy data!"); 574 assert(lgrp_spaces()->length() > 0, "There should be at least one space"); 575 576 MemRegion old_region = region(), new_region; 577 set_bottom(mr.start()); 578 set_end(mr.end()); 579 // Must always clear the space 580 clear(SpaceDecorator::DontMangle); 581 582 // Compute chunk sizes 583 size_t prev_page_size = page_size(); 584 set_page_size(UseLargePages ? alignment() : os::vm_page_size()); 585 HeapWord* rounded_bottom = align_up(bottom(), page_size()); 586 HeapWord* rounded_end = align_down(end(), page_size()); 587 size_t base_space_size_pages = pointer_delta(rounded_end, rounded_bottom, sizeof(char)) / page_size(); 588 589 // Try small pages if the chunk size is too small 590 if (base_space_size_pages / lgrp_spaces()->length() == 0 591 && page_size() > (size_t)os::vm_page_size()) { 592 // Changing the page size below can lead to freeing of memory. So we fail initialization. 593 if (_must_use_large_pages) { 594 vm_exit_during_initialization("Failed initializing NUMA with large pages. Too small heap size"); 595 } 596 set_page_size(os::vm_page_size()); 597 rounded_bottom = align_up(bottom(), page_size()); 598 rounded_end = align_down(end(), page_size()); 599 base_space_size_pages = pointer_delta(rounded_end, rounded_bottom, sizeof(char)) / page_size(); 600 } 601 guarantee(base_space_size_pages / lgrp_spaces()->length() > 0, "Space too small"); 602 set_base_space_size(base_space_size_pages); 603 604 // Handle space resize 605 MemRegion top_region, bottom_region; 606 if (!old_region.equals(region())) { 607 new_region = MemRegion(rounded_bottom, rounded_end); 608 MemRegion intersection = new_region.intersection(old_region); 609 if (intersection.start() == NULL || 610 intersection.end() == NULL || 611 prev_page_size > page_size()) { // If the page size got smaller we have to change 612 // the page size preference for the whole space. 613 intersection = MemRegion(new_region.start(), new_region.start()); 614 } 615 select_tails(new_region, intersection, &bottom_region, &top_region); 616 bias_region(bottom_region, lgrp_spaces()->at(0)->lgrp_id()); 617 bias_region(top_region, lgrp_spaces()->at(lgrp_spaces()->length() - 1)->lgrp_id()); 618 } 619 620 // Check if the space layout has changed significantly? 621 // This happens when the space has been resized so that either head or tail 622 // chunk became less than a page. 623 bool layout_valid = UseAdaptiveNUMAChunkSizing && 624 current_chunk_size(0) > page_size() && 625 current_chunk_size(lgrp_spaces()->length() - 1) > page_size(); 626 627 628 for (int i = 0; i < lgrp_spaces()->length(); i++) { 629 LGRPSpace *ls = lgrp_spaces()->at(i); 630 MutableSpace *s = ls->space(); 631 old_region = s->region(); 632 633 size_t chunk_byte_size = 0, old_chunk_byte_size = 0; 634 if (i < lgrp_spaces()->length() - 1) { 635 if (!UseAdaptiveNUMAChunkSizing || 636 (UseAdaptiveNUMAChunkSizing && NUMAChunkResizeWeight == 0) || 637 samples_count() < AdaptiveSizePolicyReadyThreshold) { 638 // No adaptation. Divide the space equally. 639 chunk_byte_size = default_chunk_size(); 640 } else 641 if (!layout_valid || NUMASpaceResizeRate == 0) { 642 // Fast adaptation. If no space resize rate is set, resize 643 // the chunks instantly. 644 chunk_byte_size = adaptive_chunk_size(i, 0); 645 } else { 646 // Slow adaptation. Resize the chunks moving no more than 647 // NUMASpaceResizeRate bytes per collection. 648 size_t limit = NUMASpaceResizeRate / 649 (lgrp_spaces()->length() * (lgrp_spaces()->length() + 1) / 2); 650 chunk_byte_size = adaptive_chunk_size(i, MAX2(limit * (i + 1), page_size())); 651 } 652 653 assert(chunk_byte_size >= page_size(), "Chunk size too small"); 654 assert(chunk_byte_size <= capacity_in_bytes(), "Sanity check"); 655 } 656 657 if (i == 0) { // Bottom chunk 658 if (i != lgrp_spaces()->length() - 1) { 659 new_region = MemRegion(bottom(), rounded_bottom + (chunk_byte_size >> LogHeapWordSize)); 660 } else { 661 new_region = MemRegion(bottom(), end()); 662 } 663 } else 664 if (i < lgrp_spaces()->length() - 1) { // Middle chunks 665 MutableSpace *ps = lgrp_spaces()->at(i - 1)->space(); 666 new_region = MemRegion(ps->end(), 667 ps->end() + (chunk_byte_size >> LogHeapWordSize)); 668 } else { // Top chunk 669 MutableSpace *ps = lgrp_spaces()->at(i - 1)->space(); 670 new_region = MemRegion(ps->end(), end()); 671 } 672 guarantee(region().contains(new_region), "Region invariant"); 673 674 675 // The general case: 676 // |---------------------|--invalid---|--------------------------| 677 // |------------------new_region---------------------------------| 678 // |----bottom_region--|---intersection---|------top_region------| 679 // |----old_region----| 680 // The intersection part has all pages in place we don't need to migrate them. 681 // Pages for the top and bottom part should be freed and then reallocated. 682 683 MemRegion intersection = old_region.intersection(new_region); 684 685 if (intersection.start() == NULL || intersection.end() == NULL) { 686 intersection = MemRegion(new_region.start(), new_region.start()); 687 } 688 689 if (!os::numa_has_static_binding()) { 690 MemRegion invalid_region = ls->invalid_region().intersection(new_region); 691 // Invalid region is a range of memory that could've possibly 692 // been allocated on the other node. That's relevant only on Solaris where 693 // there is no static memory binding. 694 if (!invalid_region.is_empty()) { 695 merge_regions(new_region, &intersection, &invalid_region); 696 free_region(invalid_region); 697 ls->set_invalid_region(MemRegion()); 698 } 699 } 700 701 select_tails(new_region, intersection, &bottom_region, &top_region); 702 703 if (!os::numa_has_static_binding()) { 704 // If that's a system with the first-touch policy then it's enough 705 // to free the pages. 706 free_region(bottom_region); 707 free_region(top_region); 708 } else { 709 // In a system with static binding we have to change the bias whenever 710 // we reshape the heap. 711 bias_region(bottom_region, ls->lgrp_id()); 712 bias_region(top_region, ls->lgrp_id()); 713 } 714 715 // Clear space (set top = bottom) but never mangle. 716 s->initialize(new_region, SpaceDecorator::Clear, SpaceDecorator::DontMangle, MutableSpace::DontSetupPages); 717 718 set_adaptation_cycles(samples_count()); 719 } 720 } 721 722 // Set the top of the whole space. 723 // Mark the the holes in chunks below the top() as invalid. 724 void MutableNUMASpace::set_top(HeapWord* value) { 725 bool found_top = false; 726 for (int i = 0; i < lgrp_spaces()->length();) { 727 LGRPSpace *ls = lgrp_spaces()->at(i); 728 MutableSpace *s = ls->space(); 729 HeapWord *top = MAX2(align_down(s->top(), page_size()), s->bottom()); 730 731 if (s->contains(value)) { 732 // Check if setting the chunk's top to a given value would create a hole less than 733 // a minimal object; assuming that's not the last chunk in which case we don't care. 734 if (i < lgrp_spaces()->length() - 1) { 735 size_t remainder = pointer_delta(s->end(), value); 736 const size_t min_fill_size = CollectedHeap::min_fill_size(); 737 if (remainder < min_fill_size && remainder > 0) { 738 // Add a minimum size filler object; it will cross the chunk boundary. 739 CollectedHeap::fill_with_object(value, min_fill_size); 740 value += min_fill_size; 741 assert(!s->contains(value), "Should be in the next chunk"); 742 // Restart the loop from the same chunk, since the value has moved 743 // to the next one. 744 continue; 745 } 746 } 747 748 if (!os::numa_has_static_binding() && top < value && top < s->end()) { 749 ls->add_invalid_region(MemRegion(top, value)); 750 } 751 s->set_top(value); 752 found_top = true; 753 } else { 754 if (found_top) { 755 s->set_top(s->bottom()); 756 } else { 757 if (!os::numa_has_static_binding() && top < s->end()) { 758 ls->add_invalid_region(MemRegion(top, s->end())); 759 } 760 s->set_top(s->end()); 761 } 762 } 763 i++; 764 } 765 MutableSpace::set_top(value); 766 } 767 768 void MutableNUMASpace::clear(bool mangle_space) { 769 MutableSpace::set_top(bottom()); 770 for (int i = 0; i < lgrp_spaces()->length(); i++) { 771 // Never mangle NUMA spaces because the mangling will 772 // bind the memory to a possibly unwanted lgroup. 773 lgrp_spaces()->at(i)->space()->clear(SpaceDecorator::DontMangle); 774 } 775 } 776 777 /* 778 Linux supports static memory binding, therefore the most part of the 779 logic dealing with the possible invalid page allocation is effectively 780 disabled. Besides there is no notion of the home node in Linux. A 781 thread is allowed to migrate freely. Although the scheduler is rather 782 reluctant to move threads between the nodes. We check for the current 783 node every allocation. And with a high probability a thread stays on 784 the same node for some time allowing local access to recently allocated 785 objects. 786 */ 787 788 HeapWord* MutableNUMASpace::allocate(size_t size) { 789 Thread* thr = Thread::current(); 790 int lgrp_id = thr->lgrp_id(); 791 if (lgrp_id == -1 || !os::numa_has_group_homing()) { 792 lgrp_id = os::numa_get_group_id(); 793 thr->set_lgrp_id(lgrp_id); 794 } 795 796 int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals); 797 798 // It is possible that a new CPU has been hotplugged and 799 // we haven't reshaped the space accordingly. 800 if (i == -1) { 801 i = os::random() % lgrp_spaces()->length(); 802 } 803 804 LGRPSpace* ls = lgrp_spaces()->at(i); 805 MutableSpace *s = ls->space(); 806 HeapWord *p = s->allocate(size); 807 808 if (p != NULL) { 809 size_t remainder = s->free_in_words(); 810 if (remainder < CollectedHeap::min_fill_size() && remainder > 0) { 811 s->set_top(s->top() - size); 812 p = NULL; 813 } 814 } 815 if (p != NULL) { 816 if (top() < s->top()) { // Keep _top updated. 817 MutableSpace::set_top(s->top()); 818 } 819 } 820 // Make the page allocation happen here if there is no static binding.. 821 if (p != NULL && !os::numa_has_static_binding()) { 822 for (HeapWord *i = p; i < p + size; i += os::vm_page_size() >> LogHeapWordSize) { 823 *(int*)i = 0; 824 } 825 } 826 if (p == NULL) { 827 ls->set_allocation_failed(); 828 } 829 return p; 830 } 831 832 // This version is lock-free. 833 HeapWord* MutableNUMASpace::cas_allocate(size_t size) { 834 Thread* thr = Thread::current(); 835 int lgrp_id = thr->lgrp_id(); 836 if (lgrp_id == -1 || !os::numa_has_group_homing()) { 837 lgrp_id = os::numa_get_group_id(); 838 thr->set_lgrp_id(lgrp_id); 839 } 840 841 int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals); 842 // It is possible that a new CPU has been hotplugged and 843 // we haven't reshaped the space accordingly. 844 if (i == -1) { 845 i = os::random() % lgrp_spaces()->length(); 846 } 847 LGRPSpace *ls = lgrp_spaces()->at(i); 848 MutableSpace *s = ls->space(); 849 HeapWord *p = s->cas_allocate(size); 850 if (p != NULL) { 851 size_t remainder = pointer_delta(s->end(), p + size); 852 if (remainder < CollectedHeap::min_fill_size() && remainder > 0) { 853 if (s->cas_deallocate(p, size)) { 854 // We were the last to allocate and created a fragment less than 855 // a minimal object. 856 p = NULL; 857 } else { 858 guarantee(false, "Deallocation should always succeed"); 859 } 860 } 861 } 862 if (p != NULL) { 863 HeapWord* cur_top, *cur_chunk_top = p + size; 864 while ((cur_top = top()) < cur_chunk_top) { // Keep _top updated. 865 if (Atomic::cmpxchg_ptr(cur_chunk_top, top_addr(), cur_top) == cur_top) { 866 break; 867 } 868 } 869 } 870 871 // Make the page allocation happen here if there is no static binding. 872 if (p != NULL && !os::numa_has_static_binding() ) { 873 for (HeapWord *i = p; i < p + size; i += os::vm_page_size() >> LogHeapWordSize) { 874 *(int*)i = 0; 875 } 876 } 877 if (p == NULL) { 878 ls->set_allocation_failed(); 879 } 880 return p; 881 } 882 883 void MutableNUMASpace::print_short_on(outputStream* st) const { 884 MutableSpace::print_short_on(st); 885 st->print(" ("); 886 for (int i = 0; i < lgrp_spaces()->length(); i++) { 887 st->print("lgrp %d: ", lgrp_spaces()->at(i)->lgrp_id()); 888 lgrp_spaces()->at(i)->space()->print_short_on(st); 889 if (i < lgrp_spaces()->length() - 1) { 890 st->print(", "); 891 } 892 } 893 st->print(")"); 894 } 895 896 void MutableNUMASpace::print_on(outputStream* st) const { 897 MutableSpace::print_on(st); 898 for (int i = 0; i < lgrp_spaces()->length(); i++) { 899 LGRPSpace *ls = lgrp_spaces()->at(i); 900 st->print(" lgrp %d", ls->lgrp_id()); 901 ls->space()->print_on(st); 902 if (NUMAStats) { 903 for (int i = 0; i < lgrp_spaces()->length(); i++) { 904 lgrp_spaces()->at(i)->accumulate_statistics(page_size()); 905 } 906 st->print(" local/remote/unbiased/uncommitted: " SIZE_FORMAT "K/" 907 SIZE_FORMAT "K/" SIZE_FORMAT "K/" SIZE_FORMAT 908 "K, large/small pages: " SIZE_FORMAT "/" SIZE_FORMAT "\n", 909 ls->space_stats()->_local_space / K, 910 ls->space_stats()->_remote_space / K, 911 ls->space_stats()->_unbiased_space / K, 912 ls->space_stats()->_uncommited_space / K, 913 ls->space_stats()->_large_pages, 914 ls->space_stats()->_small_pages); 915 } 916 } 917 } 918 919 void MutableNUMASpace::verify() { 920 // This can be called after setting an arbitrary value to the space's top, 921 // so an object can cross the chunk boundary. We ensure the parsability 922 // of the space and just walk the objects in linear fashion. 923 ensure_parsability(); 924 MutableSpace::verify(); 925 } 926 927 // Scan pages and gather stats about page placement and size. 928 void MutableNUMASpace::LGRPSpace::accumulate_statistics(size_t page_size) { 929 clear_space_stats(); 930 char *start = (char*)align_up(space()->bottom(), page_size); 931 char* end = (char*)align_down(space()->end(), page_size); 932 if (start < end) { 933 for (char *p = start; p < end;) { 934 os::page_info info; 935 if (os::get_page_info(p, &info)) { 936 if (info.size > 0) { 937 if (info.size > (size_t)os::vm_page_size()) { 938 space_stats()->_large_pages++; 939 } else { 940 space_stats()->_small_pages++; 941 } 942 if (info.lgrp_id == lgrp_id()) { 943 space_stats()->_local_space += info.size; 944 } else { 945 space_stats()->_remote_space += info.size; 946 } 947 p += info.size; 948 } else { 949 p += os::vm_page_size(); 950 space_stats()->_uncommited_space += os::vm_page_size(); 951 } 952 } else { 953 return; 954 } 955 } 956 } 957 space_stats()->_unbiased_space = pointer_delta(start, space()->bottom(), sizeof(char)) + 958 pointer_delta(space()->end(), end, sizeof(char)); 959 960 } 961 962 // Scan page_count pages and verify if they have the right size and right placement. 963 // If invalid pages are found they are freed in hope that subsequent reallocation 964 // will be more successful. 965 void MutableNUMASpace::LGRPSpace::scan_pages(size_t page_size, size_t page_count) 966 { 967 char* range_start = (char*)align_up(space()->bottom(), page_size); 968 char* range_end = (char*)align_down(space()->end(), page_size); 969 970 if (range_start > last_page_scanned() || last_page_scanned() >= range_end) { 971 set_last_page_scanned(range_start); 972 } 973 974 char *scan_start = last_page_scanned(); 975 char* scan_end = MIN2(scan_start + page_size * page_count, range_end); 976 977 os::page_info page_expected, page_found; 978 page_expected.size = page_size; 979 page_expected.lgrp_id = lgrp_id(); 980 981 char *s = scan_start; 982 while (s < scan_end) { 983 char *e = os::scan_pages(s, (char*)scan_end, &page_expected, &page_found); 984 if (e == NULL) { 985 break; 986 } 987 if (e != scan_end) { 988 assert(e < scan_end, "e: " PTR_FORMAT " scan_end: " PTR_FORMAT, p2i(e), p2i(scan_end)); 989 990 if ((page_expected.size != page_size || page_expected.lgrp_id != lgrp_id()) 991 && page_expected.size != 0) { 992 os::free_memory(s, pointer_delta(e, s, sizeof(char)), page_size); 993 } 994 page_expected = page_found; 995 } 996 s = e; 997 } 998 999 set_last_page_scanned(scan_end); 1000 }