1 2 /* 3 * Copyright (c) 2006, 2010, Oracle and/or its affiliates. All rights reserved. 4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 5 * 6 * This code is free software; you can redistribute it and/or modify it 7 * under the terms of the GNU General Public License version 2 only, as 8 * published by the Free Software Foundation. 9 * 10 * This code is distributed in the hope that it will be useful, but WITHOUT 11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 13 * version 2 for more details (a copy is included in the LICENSE file that 14 * accompanied this code). 15 * 16 * You should have received a copy of the GNU General Public License version 17 * 2 along with this work; if not, write to the Free Software Foundation, 18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 19 * 20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 21 * or visit www.oracle.com if you need additional information or have any 22 * questions. 23 * 24 */ 25 26 #include "precompiled.hpp" 27 #include "gc_implementation/shared/mutableNUMASpace.hpp" 28 #include "gc_implementation/shared/spaceDecorator.hpp" 29 #include "memory/sharedHeap.hpp" 30 #include "oops/oop.inline.hpp" 31 #ifdef TARGET_OS_FAMILY_linux 32 # include "thread_linux.inline.hpp" 33 #endif 34 #ifdef TARGET_OS_FAMILY_solaris 35 # include "thread_solaris.inline.hpp" 36 #endif 37 #ifdef TARGET_OS_FAMILY_windows 38 # include "thread_windows.inline.hpp" 39 #endif 40 41 42 MutableNUMASpace::MutableNUMASpace(size_t alignment) : MutableSpace(alignment) { 43 _lgrp_spaces = new (ResourceObj::C_HEAP) GrowableArray<LGRPSpace*>(0, true); 44 _page_size = os::vm_page_size(); 45 _adaptation_cycles = 0; 46 _samples_count = 0; 47 update_layout(true); 48 } 49 50 MutableNUMASpace::~MutableNUMASpace() { 51 for (int i = 0; i < lgrp_spaces()->length(); i++) { 52 delete lgrp_spaces()->at(i); 53 } 54 delete lgrp_spaces(); 55 } 56 57 #ifndef PRODUCT 58 void MutableNUMASpace::mangle_unused_area() { 59 // This method should do nothing. 60 // It can be called on a numa space during a full compaction. 61 } 62 void MutableNUMASpace::mangle_unused_area_complete() { 63 // This method should do nothing. 64 // It can be called on a numa space during a full compaction. 65 } 66 void MutableNUMASpace::mangle_region(MemRegion mr) { 67 // This method should do nothing because numa spaces are not mangled. 68 } 69 void MutableNUMASpace::set_top_for_allocations(HeapWord* v) { 70 assert(false, "Do not mangle MutableNUMASpace's"); 71 } 72 void MutableNUMASpace::set_top_for_allocations() { 73 // This method should do nothing. 74 } 75 void MutableNUMASpace::check_mangled_unused_area(HeapWord* limit) { 76 // This method should do nothing. 77 } 78 void MutableNUMASpace::check_mangled_unused_area_complete() { 79 // This method should do nothing. 80 } 81 #endif // NOT_PRODUCT 82 83 // There may be unallocated holes in the middle chunks 84 // that should be filled with dead objects to ensure parseability. 85 void MutableNUMASpace::ensure_parsability() { 86 for (int i = 0; i < lgrp_spaces()->length(); i++) { 87 LGRPSpace *ls = lgrp_spaces()->at(i); 88 MutableSpace *s = ls->space(); 89 if (s->top() < top()) { // For all spaces preceding the one containing top() 90 if (s->free_in_words() > 0) { 91 size_t area_touched_words = pointer_delta(s->end(), s->top()); 92 CollectedHeap::fill_with_object(s->top(), area_touched_words); 93 #ifndef ASSERT 94 if (!ZapUnusedHeapArea) { 95 area_touched_words = MIN2((size_t)align_object_size(typeArrayOopDesc::header_size(T_INT)), 96 area_touched_words); 97 } 98 #endif 99 if (!os::numa_has_static_binding()) { 100 MemRegion invalid; 101 HeapWord *crossing_start = (HeapWord*)round_to((intptr_t)s->top(), os::vm_page_size()); 102 HeapWord *crossing_end = (HeapWord*)round_to((intptr_t)(s->top() + area_touched_words), 103 os::vm_page_size()); 104 if (crossing_start != crossing_end) { 105 // If object header crossed a small page boundary we mark the area 106 // as invalid rounding it to a page_size(). 107 HeapWord *start = MAX2((HeapWord*)round_down((intptr_t)s->top(), page_size()), s->bottom()); 108 HeapWord *end = MIN2((HeapWord*)round_to((intptr_t)(s->top() + area_touched_words), page_size()), 109 s->end()); 110 invalid = MemRegion(start, end); 111 } 112 113 ls->add_invalid_region(invalid); 114 } 115 } 116 } else { 117 if (!os::numa_has_static_binding()) { 118 #ifdef ASSERT 119 MemRegion invalid(s->top(), s->end()); 120 ls->add_invalid_region(invalid); 121 #else 122 if (ZapUnusedHeapArea) { 123 MemRegion invalid(s->top(), s->end()); 124 ls->add_invalid_region(invalid); 125 } else { 126 return; 127 } 128 #endif 129 } else { 130 return; 131 } 132 } 133 } 134 } 135 136 size_t MutableNUMASpace::used_in_words() const { 137 size_t s = 0; 138 for (int i = 0; i < lgrp_spaces()->length(); i++) { 139 s += lgrp_spaces()->at(i)->space()->used_in_words(); 140 } 141 return s; 142 } 143 144 size_t MutableNUMASpace::free_in_words() const { 145 size_t s = 0; 146 for (int i = 0; i < lgrp_spaces()->length(); i++) { 147 s += lgrp_spaces()->at(i)->space()->free_in_words(); 148 } 149 return s; 150 } 151 152 153 size_t MutableNUMASpace::tlab_capacity(Thread *thr) const { 154 guarantee(thr != NULL, "No thread"); 155 int lgrp_id = thr->lgrp_id(); 156 if (lgrp_id == -1) { 157 // This case can occur after the topology of the system has 158 // changed. Thread can change their location, the new home 159 // group will be determined during the first allocation 160 // attempt. For now we can safely assume that all spaces 161 // have equal size because the whole space will be reinitialized. 162 if (lgrp_spaces()->length() > 0) { 163 return capacity_in_bytes() / lgrp_spaces()->length(); 164 } else { 165 assert(false, "There should be at least one locality group"); 166 return 0; 167 } 168 } 169 // That's the normal case, where we know the locality group of the thread. 170 int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals); 171 if (i == -1) { 172 return 0; 173 } 174 return lgrp_spaces()->at(i)->space()->capacity_in_bytes(); 175 } 176 177 size_t MutableNUMASpace::unsafe_max_tlab_alloc(Thread *thr) const { 178 // Please see the comments for tlab_capacity(). 179 guarantee(thr != NULL, "No thread"); 180 int lgrp_id = thr->lgrp_id(); 181 if (lgrp_id == -1) { 182 if (lgrp_spaces()->length() > 0) { 183 return free_in_bytes() / lgrp_spaces()->length(); 184 } else { 185 assert(false, "There should be at least one locality group"); 186 return 0; 187 } 188 } 189 int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals); 190 if (i == -1) { 191 return 0; 192 } 193 return lgrp_spaces()->at(i)->space()->free_in_bytes(); 194 } 195 196 197 size_t MutableNUMASpace::capacity_in_words(Thread* thr) const { 198 guarantee(thr != NULL, "No thread"); 199 int lgrp_id = thr->lgrp_id(); 200 if (lgrp_id == -1) { 201 if (lgrp_spaces()->length() > 0) { 202 return capacity_in_words() / lgrp_spaces()->length(); 203 } else { 204 assert(false, "There should be at least one locality group"); 205 return 0; 206 } 207 } 208 int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals); 209 if (i == -1) { 210 return 0; 211 } 212 return lgrp_spaces()->at(i)->space()->capacity_in_words(); 213 } 214 215 // Check if the NUMA topology has changed. Add and remove spaces if needed. 216 // The update can be forced by setting the force parameter equal to true. 217 bool MutableNUMASpace::update_layout(bool force) { 218 // Check if the topology had changed. 219 bool changed = os::numa_topology_changed(); 220 if (force || changed) { 221 // Compute lgrp intersection. Add/remove spaces. 222 int lgrp_limit = (int)os::numa_get_groups_num(); 223 int *lgrp_ids = NEW_C_HEAP_ARRAY(int, lgrp_limit); 224 int lgrp_num = (int)os::numa_get_leaf_groups(lgrp_ids, lgrp_limit); 225 assert(lgrp_num > 0, "There should be at least one locality group"); 226 // Add new spaces for the new nodes 227 for (int i = 0; i < lgrp_num; i++) { 228 bool found = false; 229 for (int j = 0; j < lgrp_spaces()->length(); j++) { 230 if (lgrp_spaces()->at(j)->lgrp_id() == lgrp_ids[i]) { 231 found = true; 232 break; 233 } 234 } 235 if (!found) { 236 lgrp_spaces()->append(new LGRPSpace(lgrp_ids[i], alignment())); 237 } 238 } 239 240 // Remove spaces for the removed nodes. 241 for (int i = 0; i < lgrp_spaces()->length();) { 242 bool found = false; 243 for (int j = 0; j < lgrp_num; j++) { 244 if (lgrp_spaces()->at(i)->lgrp_id() == lgrp_ids[j]) { 245 found = true; 246 break; 247 } 248 } 249 if (!found) { 250 delete lgrp_spaces()->at(i); 251 lgrp_spaces()->remove_at(i); 252 } else { 253 i++; 254 } 255 } 256 257 FREE_C_HEAP_ARRAY(int, lgrp_ids); 258 259 if (changed) { 260 for (JavaThread *thread = Threads::first(); thread; thread = thread->next()) { 261 thread->set_lgrp_id(-1); 262 } 263 } 264 return true; 265 } 266 return false; 267 } 268 269 // Bias region towards the first-touching lgrp. Set the right page sizes. 270 void MutableNUMASpace::bias_region(MemRegion mr, int lgrp_id) { 271 HeapWord *start = (HeapWord*)round_to((intptr_t)mr.start(), page_size()); 272 HeapWord *end = (HeapWord*)round_down((intptr_t)mr.end(), page_size()); 273 if (end > start) { 274 MemRegion aligned_region(start, end); 275 assert((intptr_t)aligned_region.start() % page_size() == 0 && 276 (intptr_t)aligned_region.byte_size() % page_size() == 0, "Bad alignment"); 277 assert(region().contains(aligned_region), "Sanity"); 278 // First we tell the OS which page size we want in the given range. The underlying 279 // large page can be broken down if we require small pages. 280 os::realign_memory((char*)aligned_region.start(), aligned_region.byte_size(), page_size()); 281 // Then we uncommit the pages in the range. 282 os::free_memory((char*)aligned_region.start(), aligned_region.byte_size()); 283 // And make them local/first-touch biased. 284 os::numa_make_local((char*)aligned_region.start(), aligned_region.byte_size(), lgrp_id); 285 } 286 } 287 288 // Free all pages in the region. 289 void MutableNUMASpace::free_region(MemRegion mr) { 290 HeapWord *start = (HeapWord*)round_to((intptr_t)mr.start(), page_size()); 291 HeapWord *end = (HeapWord*)round_down((intptr_t)mr.end(), page_size()); 292 if (end > start) { 293 MemRegion aligned_region(start, end); 294 assert((intptr_t)aligned_region.start() % page_size() == 0 && 295 (intptr_t)aligned_region.byte_size() % page_size() == 0, "Bad alignment"); 296 assert(region().contains(aligned_region), "Sanity"); 297 os::free_memory((char*)aligned_region.start(), aligned_region.byte_size()); 298 } 299 } 300 301 // Update space layout. Perform adaptation. 302 void MutableNUMASpace::update() { 303 if (update_layout(false)) { 304 // If the topology has changed, make all chunks zero-sized. 305 // And clear the alloc-rate statistics. 306 // In future we may want to handle this more gracefully in order 307 // to avoid the reallocation of the pages as much as possible. 308 for (int i = 0; i < lgrp_spaces()->length(); i++) { 309 LGRPSpace *ls = lgrp_spaces()->at(i); 310 MutableSpace *s = ls->space(); 311 s->set_end(s->bottom()); 312 s->set_top(s->bottom()); 313 ls->clear_alloc_rate(); 314 } 315 // A NUMA space is never mangled 316 initialize(region(), 317 SpaceDecorator::Clear, 318 SpaceDecorator::DontMangle); 319 } else { 320 bool should_initialize = false; 321 if (!os::numa_has_static_binding()) { 322 for (int i = 0; i < lgrp_spaces()->length(); i++) { 323 if (!lgrp_spaces()->at(i)->invalid_region().is_empty()) { 324 should_initialize = true; 325 break; 326 } 327 } 328 } 329 330 if (should_initialize || 331 (UseAdaptiveNUMAChunkSizing && adaptation_cycles() < samples_count())) { 332 // A NUMA space is never mangled 333 initialize(region(), 334 SpaceDecorator::Clear, 335 SpaceDecorator::DontMangle); 336 } 337 } 338 339 if (NUMAStats) { 340 for (int i = 0; i < lgrp_spaces()->length(); i++) { 341 lgrp_spaces()->at(i)->accumulate_statistics(page_size()); 342 } 343 } 344 345 scan_pages(NUMAPageScanRate); 346 } 347 348 // Scan pages. Free pages that have smaller size or wrong placement. 349 void MutableNUMASpace::scan_pages(size_t page_count) 350 { 351 size_t pages_per_chunk = page_count / lgrp_spaces()->length(); 352 if (pages_per_chunk > 0) { 353 for (int i = 0; i < lgrp_spaces()->length(); i++) { 354 LGRPSpace *ls = lgrp_spaces()->at(i); 355 ls->scan_pages(page_size(), pages_per_chunk); 356 } 357 } 358 } 359 360 // Accumulate statistics about the allocation rate of each lgrp. 361 void MutableNUMASpace::accumulate_statistics() { 362 if (UseAdaptiveNUMAChunkSizing) { 363 for (int i = 0; i < lgrp_spaces()->length(); i++) { 364 lgrp_spaces()->at(i)->sample(); 365 } 366 increment_samples_count(); 367 } 368 369 if (NUMAStats) { 370 for (int i = 0; i < lgrp_spaces()->length(); i++) { 371 lgrp_spaces()->at(i)->accumulate_statistics(page_size()); 372 } 373 } 374 } 375 376 // Get the current size of a chunk. 377 // This function computes the size of the chunk based on the 378 // difference between chunk ends. This allows it to work correctly in 379 // case the whole space is resized and during the process of adaptive 380 // chunk resizing. 381 size_t MutableNUMASpace::current_chunk_size(int i) { 382 HeapWord *cur_end, *prev_end; 383 if (i == 0) { 384 prev_end = bottom(); 385 } else { 386 prev_end = lgrp_spaces()->at(i - 1)->space()->end(); 387 } 388 if (i == lgrp_spaces()->length() - 1) { 389 cur_end = end(); 390 } else { 391 cur_end = lgrp_spaces()->at(i)->space()->end(); 392 } 393 if (cur_end > prev_end) { 394 return pointer_delta(cur_end, prev_end, sizeof(char)); 395 } 396 return 0; 397 } 398 399 // Return the default chunk size by equally diving the space. 400 // page_size() aligned. 401 size_t MutableNUMASpace::default_chunk_size() { 402 return base_space_size() / lgrp_spaces()->length() * page_size(); 403 } 404 405 // Produce a new chunk size. page_size() aligned. 406 // This function is expected to be called on sequence of i's from 0 to 407 // lgrp_spaces()->length(). 408 size_t MutableNUMASpace::adaptive_chunk_size(int i, size_t limit) { 409 size_t pages_available = base_space_size(); 410 for (int j = 0; j < i; j++) { 411 pages_available -= round_down(current_chunk_size(j), page_size()) / page_size(); 412 } 413 pages_available -= lgrp_spaces()->length() - i - 1; 414 assert(pages_available > 0, "No pages left"); 415 float alloc_rate = 0; 416 for (int j = i; j < lgrp_spaces()->length(); j++) { 417 alloc_rate += lgrp_spaces()->at(j)->alloc_rate()->average(); 418 } 419 size_t chunk_size = 0; 420 if (alloc_rate > 0) { 421 LGRPSpace *ls = lgrp_spaces()->at(i); 422 chunk_size = (size_t)(ls->alloc_rate()->average() / alloc_rate * pages_available) * page_size(); 423 } 424 chunk_size = MAX2(chunk_size, page_size()); 425 426 if (limit > 0) { 427 limit = round_down(limit, page_size()); 428 if (chunk_size > current_chunk_size(i)) { 429 size_t upper_bound = pages_available * page_size(); 430 if (upper_bound > limit && 431 current_chunk_size(i) < upper_bound - limit) { 432 // The resulting upper bound should not exceed the available 433 // amount of memory (pages_available * page_size()). 434 upper_bound = current_chunk_size(i) + limit; 435 } 436 chunk_size = MIN2(chunk_size, upper_bound); 437 } else { 438 size_t lower_bound = page_size(); 439 if (current_chunk_size(i) > limit) { // lower_bound shouldn't underflow. 440 lower_bound = current_chunk_size(i) - limit; 441 } 442 chunk_size = MAX2(chunk_size, lower_bound); 443 } 444 } 445 assert(chunk_size <= pages_available * page_size(), "Chunk size out of range"); 446 return chunk_size; 447 } 448 449 450 // Return the bottom_region and the top_region. Align them to page_size() boundary. 451 // |------------------new_region---------------------------------| 452 // |----bottom_region--|---intersection---|------top_region------| 453 void MutableNUMASpace::select_tails(MemRegion new_region, MemRegion intersection, 454 MemRegion* bottom_region, MemRegion *top_region) { 455 // Is there bottom? 456 if (new_region.start() < intersection.start()) { // Yes 457 // Try to coalesce small pages into a large one. 458 if (UseLargePages && page_size() >= alignment()) { 459 HeapWord* p = (HeapWord*)round_to((intptr_t) intersection.start(), alignment()); 460 if (new_region.contains(p) 461 && pointer_delta(p, new_region.start(), sizeof(char)) >= alignment()) { 462 if (intersection.contains(p)) { 463 intersection = MemRegion(p, intersection.end()); 464 } else { 465 intersection = MemRegion(p, p); 466 } 467 } 468 } 469 *bottom_region = MemRegion(new_region.start(), intersection.start()); 470 } else { 471 *bottom_region = MemRegion(); 472 } 473 474 // Is there top? 475 if (intersection.end() < new_region.end()) { // Yes 476 // Try to coalesce small pages into a large one. 477 if (UseLargePages && page_size() >= alignment()) { 478 HeapWord* p = (HeapWord*)round_down((intptr_t) intersection.end(), alignment()); 479 if (new_region.contains(p) 480 && pointer_delta(new_region.end(), p, sizeof(char)) >= alignment()) { 481 if (intersection.contains(p)) { 482 intersection = MemRegion(intersection.start(), p); 483 } else { 484 intersection = MemRegion(p, p); 485 } 486 } 487 } 488 *top_region = MemRegion(intersection.end(), new_region.end()); 489 } else { 490 *top_region = MemRegion(); 491 } 492 } 493 494 // Try to merge the invalid region with the bottom or top region by decreasing 495 // the intersection area. Return the invalid_region aligned to the page_size() 496 // boundary if it's inside the intersection. Return non-empty invalid_region 497 // if it lies inside the intersection (also page-aligned). 498 // |------------------new_region---------------------------------| 499 // |----------------|-------invalid---|--------------------------| 500 // |----bottom_region--|---intersection---|------top_region------| 501 void MutableNUMASpace::merge_regions(MemRegion new_region, MemRegion* intersection, 502 MemRegion *invalid_region) { 503 if (intersection->start() >= invalid_region->start() && intersection->contains(invalid_region->end())) { 504 *intersection = MemRegion(invalid_region->end(), intersection->end()); 505 *invalid_region = MemRegion(); 506 } else 507 if (intersection->end() <= invalid_region->end() && intersection->contains(invalid_region->start())) { 508 *intersection = MemRegion(intersection->start(), invalid_region->start()); 509 *invalid_region = MemRegion(); 510 } else 511 if (intersection->equals(*invalid_region) || invalid_region->contains(*intersection)) { 512 *intersection = MemRegion(new_region.start(), new_region.start()); 513 *invalid_region = MemRegion(); 514 } else 515 if (intersection->contains(invalid_region)) { 516 // That's the only case we have to make an additional bias_region() call. 517 HeapWord* start = invalid_region->start(); 518 HeapWord* end = invalid_region->end(); 519 if (UseLargePages && page_size() >= alignment()) { 520 HeapWord *p = (HeapWord*)round_down((intptr_t) start, alignment()); 521 if (new_region.contains(p)) { 522 start = p; 523 } 524 p = (HeapWord*)round_to((intptr_t) end, alignment()); 525 if (new_region.contains(end)) { 526 end = p; 527 } 528 } 529 if (intersection->start() > start) { 530 *intersection = MemRegion(start, intersection->end()); 531 } 532 if (intersection->end() < end) { 533 *intersection = MemRegion(intersection->start(), end); 534 } 535 *invalid_region = MemRegion(start, end); 536 } 537 } 538 539 void MutableNUMASpace::initialize(MemRegion mr, 540 bool clear_space, 541 bool mangle_space, 542 bool setup_pages) { 543 assert(clear_space, "Reallocation will destory data!"); 544 assert(lgrp_spaces()->length() > 0, "There should be at least one space"); 545 546 MemRegion old_region = region(), new_region; 547 set_bottom(mr.start()); 548 set_end(mr.end()); 549 // Must always clear the space 550 clear(SpaceDecorator::DontMangle); 551 552 // Compute chunk sizes 553 size_t prev_page_size = page_size(); 554 set_page_size(UseLargePages ? alignment() : os::vm_page_size()); 555 HeapWord* rounded_bottom = (HeapWord*)round_to((intptr_t) bottom(), page_size()); 556 HeapWord* rounded_end = (HeapWord*)round_down((intptr_t) end(), page_size()); 557 size_t base_space_size_pages = pointer_delta(rounded_end, rounded_bottom, sizeof(char)) / page_size(); 558 559 // Try small pages if the chunk size is too small 560 if (base_space_size_pages / lgrp_spaces()->length() == 0 561 && page_size() > (size_t)os::vm_page_size()) { 562 set_page_size(os::vm_page_size()); 563 rounded_bottom = (HeapWord*)round_to((intptr_t) bottom(), page_size()); 564 rounded_end = (HeapWord*)round_down((intptr_t) end(), page_size()); 565 base_space_size_pages = pointer_delta(rounded_end, rounded_bottom, sizeof(char)) / page_size(); 566 } 567 guarantee(base_space_size_pages / lgrp_spaces()->length() > 0, "Space too small"); 568 set_base_space_size(base_space_size_pages); 569 570 // Handle space resize 571 MemRegion top_region, bottom_region; 572 if (!old_region.equals(region())) { 573 new_region = MemRegion(rounded_bottom, rounded_end); 574 MemRegion intersection = new_region.intersection(old_region); 575 if (intersection.start() == NULL || 576 intersection.end() == NULL || 577 prev_page_size > page_size()) { // If the page size got smaller we have to change 578 // the page size preference for the whole space. 579 intersection = MemRegion(new_region.start(), new_region.start()); 580 } 581 select_tails(new_region, intersection, &bottom_region, &top_region); 582 bias_region(bottom_region, lgrp_spaces()->at(0)->lgrp_id()); 583 bias_region(top_region, lgrp_spaces()->at(lgrp_spaces()->length() - 1)->lgrp_id()); 584 } 585 586 // Check if the space layout has changed significantly? 587 // This happens when the space has been resized so that either head or tail 588 // chunk became less than a page. 589 bool layout_valid = UseAdaptiveNUMAChunkSizing && 590 current_chunk_size(0) > page_size() && 591 current_chunk_size(lgrp_spaces()->length() - 1) > page_size(); 592 593 594 for (int i = 0; i < lgrp_spaces()->length(); i++) { 595 LGRPSpace *ls = lgrp_spaces()->at(i); 596 MutableSpace *s = ls->space(); 597 old_region = s->region(); 598 599 size_t chunk_byte_size = 0, old_chunk_byte_size = 0; 600 if (i < lgrp_spaces()->length() - 1) { 601 if (!UseAdaptiveNUMAChunkSizing || 602 (UseAdaptiveNUMAChunkSizing && NUMAChunkResizeWeight == 0) || 603 samples_count() < AdaptiveSizePolicyReadyThreshold) { 604 // No adaptation. Divide the space equally. 605 chunk_byte_size = default_chunk_size(); 606 } else 607 if (!layout_valid || NUMASpaceResizeRate == 0) { 608 // Fast adaptation. If no space resize rate is set, resize 609 // the chunks instantly. 610 chunk_byte_size = adaptive_chunk_size(i, 0); 611 } else { 612 // Slow adaptation. Resize the chunks moving no more than 613 // NUMASpaceResizeRate bytes per collection. 614 size_t limit = NUMASpaceResizeRate / 615 (lgrp_spaces()->length() * (lgrp_spaces()->length() + 1) / 2); 616 chunk_byte_size = adaptive_chunk_size(i, MAX2(limit * (i + 1), page_size())); 617 } 618 619 assert(chunk_byte_size >= page_size(), "Chunk size too small"); 620 assert(chunk_byte_size <= capacity_in_bytes(), "Sanity check"); 621 } 622 623 if (i == 0) { // Bottom chunk 624 if (i != lgrp_spaces()->length() - 1) { 625 new_region = MemRegion(bottom(), rounded_bottom + (chunk_byte_size >> LogHeapWordSize)); 626 } else { 627 new_region = MemRegion(bottom(), end()); 628 } 629 } else 630 if (i < lgrp_spaces()->length() - 1) { // Middle chunks 631 MutableSpace *ps = lgrp_spaces()->at(i - 1)->space(); 632 new_region = MemRegion(ps->end(), 633 ps->end() + (chunk_byte_size >> LogHeapWordSize)); 634 } else { // Top chunk 635 MutableSpace *ps = lgrp_spaces()->at(i - 1)->space(); 636 new_region = MemRegion(ps->end(), end()); 637 } 638 guarantee(region().contains(new_region), "Region invariant"); 639 640 641 // The general case: 642 // |---------------------|--invalid---|--------------------------| 643 // |------------------new_region---------------------------------| 644 // |----bottom_region--|---intersection---|------top_region------| 645 // |----old_region----| 646 // The intersection part has all pages in place we don't need to migrate them. 647 // Pages for the top and bottom part should be freed and then reallocated. 648 649 MemRegion intersection = old_region.intersection(new_region); 650 651 if (intersection.start() == NULL || intersection.end() == NULL) { 652 intersection = MemRegion(new_region.start(), new_region.start()); 653 } 654 655 if (!os::numa_has_static_binding()) { 656 MemRegion invalid_region = ls->invalid_region().intersection(new_region); 657 // Invalid region is a range of memory that could've possibly 658 // been allocated on the other node. That's relevant only on Solaris where 659 // there is no static memory binding. 660 if (!invalid_region.is_empty()) { 661 merge_regions(new_region, &intersection, &invalid_region); 662 free_region(invalid_region); 663 ls->set_invalid_region(MemRegion()); 664 } 665 } 666 667 select_tails(new_region, intersection, &bottom_region, &top_region); 668 669 if (!os::numa_has_static_binding()) { 670 // If that's a system with the first-touch policy then it's enough 671 // to free the pages. 672 free_region(bottom_region); 673 free_region(top_region); 674 } else { 675 // In a system with static binding we have to change the bias whenever 676 // we reshape the heap. 677 bias_region(bottom_region, ls->lgrp_id()); 678 bias_region(top_region, ls->lgrp_id()); 679 } 680 681 // Clear space (set top = bottom) but never mangle. 682 s->initialize(new_region, SpaceDecorator::Clear, SpaceDecorator::DontMangle, MutableSpace::DontSetupPages); 683 684 set_adaptation_cycles(samples_count()); 685 } 686 } 687 688 // Set the top of the whole space. 689 // Mark the the holes in chunks below the top() as invalid. 690 void MutableNUMASpace::set_top(HeapWord* value) { 691 bool found_top = false; 692 for (int i = 0; i < lgrp_spaces()->length();) { 693 LGRPSpace *ls = lgrp_spaces()->at(i); 694 MutableSpace *s = ls->space(); 695 HeapWord *top = MAX2((HeapWord*)round_down((intptr_t)s->top(), page_size()), s->bottom()); 696 697 if (s->contains(value)) { 698 // Check if setting the chunk's top to a given value would create a hole less than 699 // a minimal object; assuming that's not the last chunk in which case we don't care. 700 if (i < lgrp_spaces()->length() - 1) { 701 size_t remainder = pointer_delta(s->end(), value); 702 const size_t min_fill_size = CollectedHeap::min_fill_size(); 703 if (remainder < min_fill_size && remainder > 0) { 704 // Add a minimum size filler object; it will cross the chunk boundary. 705 CollectedHeap::fill_with_object(value, min_fill_size); 706 value += min_fill_size; 707 assert(!s->contains(value), "Should be in the next chunk"); 708 // Restart the loop from the same chunk, since the value has moved 709 // to the next one. 710 continue; 711 } 712 } 713 714 if (!os::numa_has_static_binding() && top < value && top < s->end()) { 715 ls->add_invalid_region(MemRegion(top, value)); 716 } 717 s->set_top(value); 718 found_top = true; 719 } else { 720 if (found_top) { 721 s->set_top(s->bottom()); 722 } else { 723 if (!os::numa_has_static_binding() && top < s->end()) { 724 ls->add_invalid_region(MemRegion(top, s->end())); 725 } 726 s->set_top(s->end()); 727 } 728 } 729 i++; 730 } 731 MutableSpace::set_top(value); 732 } 733 734 void MutableNUMASpace::clear(bool mangle_space) { 735 MutableSpace::set_top(bottom()); 736 for (int i = 0; i < lgrp_spaces()->length(); i++) { 737 // Never mangle NUMA spaces because the mangling will 738 // bind the memory to a possibly unwanted lgroup. 739 lgrp_spaces()->at(i)->space()->clear(SpaceDecorator::DontMangle); 740 } 741 } 742 743 /* 744 Linux supports static memory binding, therefore the most part of the 745 logic dealing with the possible invalid page allocation is effectively 746 disabled. Besides there is no notion of the home node in Linux. A 747 thread is allowed to migrate freely. Although the scheduler is rather 748 reluctant to move threads between the nodes. We check for the current 749 node every allocation. And with a high probability a thread stays on 750 the same node for some time allowing local access to recently allocated 751 objects. 752 */ 753 754 HeapWord* MutableNUMASpace::allocate(size_t size) { 755 Thread* thr = Thread::current(); 756 int lgrp_id = thr->lgrp_id(); 757 if (lgrp_id == -1 || !os::numa_has_group_homing()) { 758 lgrp_id = os::numa_get_group_id(); 759 thr->set_lgrp_id(lgrp_id); 760 } 761 762 int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals); 763 764 // It is possible that a new CPU has been hotplugged and 765 // we haven't reshaped the space accordingly. 766 if (i == -1) { 767 i = os::random() % lgrp_spaces()->length(); 768 } 769 770 LGRPSpace* ls = lgrp_spaces()->at(i); 771 MutableSpace *s = ls->space(); 772 HeapWord *p = s->allocate(size); 773 774 if (p != NULL) { 775 size_t remainder = s->free_in_words(); 776 if (remainder < CollectedHeap::min_fill_size() && remainder > 0) { 777 s->set_top(s->top() - size); 778 p = NULL; 779 } 780 } 781 if (p != NULL) { 782 if (top() < s->top()) { // Keep _top updated. 783 MutableSpace::set_top(s->top()); 784 } 785 } 786 // Make the page allocation happen here if there is no static binding.. 787 if (p != NULL && !os::numa_has_static_binding()) { 788 for (HeapWord *i = p; i < p + size; i += os::vm_page_size() >> LogHeapWordSize) { 789 *(int*)i = 0; 790 } 791 } 792 if (p == NULL) { 793 ls->set_allocation_failed(); 794 } 795 return p; 796 } 797 798 // This version is lock-free. 799 HeapWord* MutableNUMASpace::cas_allocate(size_t size) { 800 Thread* thr = Thread::current(); 801 int lgrp_id = thr->lgrp_id(); 802 if (lgrp_id == -1 || !os::numa_has_group_homing()) { 803 lgrp_id = os::numa_get_group_id(); 804 thr->set_lgrp_id(lgrp_id); 805 } 806 807 int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals); 808 // It is possible that a new CPU has been hotplugged and 809 // we haven't reshaped the space accordingly. 810 if (i == -1) { 811 i = os::random() % lgrp_spaces()->length(); 812 } 813 LGRPSpace *ls = lgrp_spaces()->at(i); 814 MutableSpace *s = ls->space(); 815 HeapWord *p = s->cas_allocate(size); 816 if (p != NULL) { 817 size_t remainder = pointer_delta(s->end(), p + size); 818 if (remainder < CollectedHeap::min_fill_size() && remainder > 0) { 819 if (s->cas_deallocate(p, size)) { 820 // We were the last to allocate and created a fragment less than 821 // a minimal object. 822 p = NULL; 823 } else { 824 guarantee(false, "Deallocation should always succeed"); 825 } 826 } 827 } 828 if (p != NULL) { 829 HeapWord* cur_top, *cur_chunk_top = p + size; 830 while ((cur_top = top()) < cur_chunk_top) { // Keep _top updated. 831 if (Atomic::cmpxchg_ptr(cur_chunk_top, top_addr(), cur_top) == cur_top) { 832 break; 833 } 834 } 835 } 836 837 // Make the page allocation happen here if there is no static binding. 838 if (p != NULL && !os::numa_has_static_binding() ) { 839 for (HeapWord *i = p; i < p + size; i += os::vm_page_size() >> LogHeapWordSize) { 840 *(int*)i = 0; 841 } 842 } 843 if (p == NULL) { 844 ls->set_allocation_failed(); 845 } 846 return p; 847 } 848 849 void MutableNUMASpace::print_short_on(outputStream* st) const { 850 MutableSpace::print_short_on(st); 851 st->print(" ("); 852 for (int i = 0; i < lgrp_spaces()->length(); i++) { 853 st->print("lgrp %d: ", lgrp_spaces()->at(i)->lgrp_id()); 854 lgrp_spaces()->at(i)->space()->print_short_on(st); 855 if (i < lgrp_spaces()->length() - 1) { 856 st->print(", "); 857 } 858 } 859 st->print(")"); 860 } 861 862 void MutableNUMASpace::print_on(outputStream* st) const { 863 MutableSpace::print_on(st); 864 for (int i = 0; i < lgrp_spaces()->length(); i++) { 865 LGRPSpace *ls = lgrp_spaces()->at(i); 866 st->print(" lgrp %d", ls->lgrp_id()); 867 ls->space()->print_on(st); 868 if (NUMAStats) { 869 for (int i = 0; i < lgrp_spaces()->length(); i++) { 870 lgrp_spaces()->at(i)->accumulate_statistics(page_size()); 871 } 872 st->print(" local/remote/unbiased/uncommitted: %dK/%dK/%dK/%dK, large/small pages: %d/%d\n", 873 ls->space_stats()->_local_space / K, 874 ls->space_stats()->_remote_space / K, 875 ls->space_stats()->_unbiased_space / K, 876 ls->space_stats()->_uncommited_space / K, 877 ls->space_stats()->_large_pages, 878 ls->space_stats()->_small_pages); 879 } 880 } 881 } 882 883 void MutableNUMASpace::verify(bool allow_dirty) { 884 // This can be called after setting an arbitary value to the space's top, 885 // so an object can cross the chunk boundary. We ensure the parsablity 886 // of the space and just walk the objects in linear fashion. 887 ensure_parsability(); 888 MutableSpace::verify(allow_dirty); 889 } 890 891 // Scan pages and gather stats about page placement and size. 892 void MutableNUMASpace::LGRPSpace::accumulate_statistics(size_t page_size) { 893 clear_space_stats(); 894 char *start = (char*)round_to((intptr_t) space()->bottom(), page_size); 895 char* end = (char*)round_down((intptr_t) space()->end(), page_size); 896 if (start < end) { 897 for (char *p = start; p < end;) { 898 os::page_info info; 899 if (os::get_page_info(p, &info)) { 900 if (info.size > 0) { 901 if (info.size > (size_t)os::vm_page_size()) { 902 space_stats()->_large_pages++; 903 } else { 904 space_stats()->_small_pages++; 905 } 906 if (info.lgrp_id == lgrp_id()) { 907 space_stats()->_local_space += info.size; 908 } else { 909 space_stats()->_remote_space += info.size; 910 } 911 p += info.size; 912 } else { 913 p += os::vm_page_size(); 914 space_stats()->_uncommited_space += os::vm_page_size(); 915 } 916 } else { 917 return; 918 } 919 } 920 } 921 space_stats()->_unbiased_space = pointer_delta(start, space()->bottom(), sizeof(char)) + 922 pointer_delta(space()->end(), end, sizeof(char)); 923 924 } 925 926 // Scan page_count pages and verify if they have the right size and right placement. 927 // If invalid pages are found they are freed in hope that subsequent reallocation 928 // will be more successful. 929 void MutableNUMASpace::LGRPSpace::scan_pages(size_t page_size, size_t page_count) 930 { 931 char* range_start = (char*)round_to((intptr_t) space()->bottom(), page_size); 932 char* range_end = (char*)round_down((intptr_t) space()->end(), page_size); 933 934 if (range_start > last_page_scanned() || last_page_scanned() >= range_end) { 935 set_last_page_scanned(range_start); 936 } 937 938 char *scan_start = last_page_scanned(); 939 char* scan_end = MIN2(scan_start + page_size * page_count, range_end); 940 941 os::page_info page_expected, page_found; 942 page_expected.size = page_size; 943 page_expected.lgrp_id = lgrp_id(); 944 945 char *s = scan_start; 946 while (s < scan_end) { 947 char *e = os::scan_pages(s, (char*)scan_end, &page_expected, &page_found); 948 if (e == NULL) { 949 break; 950 } 951 if (e != scan_end) { 952 if ((page_expected.size != page_size || page_expected.lgrp_id != lgrp_id()) 953 && page_expected.size != 0) { 954 os::free_memory(s, pointer_delta(e, s, sizeof(char))); 955 } 956 page_expected = page_found; 957 } 958 s = e; 959 } 960 961 set_last_page_scanned(scan_end); 962 }