1 
   2 /*
   3  * Copyright (c) 2006, 2015, Oracle and/or its affiliates. All rights reserved.
   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This code is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU General Public License version 2 only, as
   8  * published by the Free Software Foundation.
   9  *
  10  * This code is distributed in the hope that it will be useful, but WITHOUT
  11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  13  * version 2 for more details (a copy is included in the LICENSE file that
  14  * accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License version
  17  * 2 along with this work; if not, write to the Free Software Foundation,
  18  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  19  *
  20  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  21  * or visit www.oracle.com if you need additional information or have any
  22  * questions.
  23  *
  24  */
  25 
  26 #include "precompiled.hpp"
  27 #include "gc/parallel/mutableNUMASpace.hpp"
  28 #include "gc/shared/collectedHeap.hpp"
  29 #include "gc/shared/spaceDecorator.hpp"
  30 #include "oops/oop.inline.hpp"
  31 #include "runtime/atomic.inline.hpp"
  32 #include "runtime/thread.inline.hpp"
  33 
  34 MutableNUMASpace::MutableNUMASpace(size_t alignment) : MutableSpace(alignment) {
  35   _lgrp_spaces = new (ResourceObj::C_HEAP, mtGC) GrowableArray<LGRPSpace*>(0, true);
  36   _page_size = os::vm_page_size();
  37   _adaptation_cycles = 0;
  38   _samples_count = 0;
  39   update_layout(true);
  40 }
  41 
  42 MutableNUMASpace::~MutableNUMASpace() {
  43   for (int i = 0; i < lgrp_spaces()->length(); i++) {
  44     delete lgrp_spaces()->at(i);
  45   }
  46   delete lgrp_spaces();
  47 }
  48 
  49 #ifndef PRODUCT
  50 void MutableNUMASpace::mangle_unused_area() {
  51   // This method should do nothing.
  52   // It can be called on a numa space during a full compaction.
  53 }
  54 void MutableNUMASpace::mangle_unused_area_complete() {
  55   // This method should do nothing.
  56   // It can be called on a numa space during a full compaction.
  57 }
  58 void MutableNUMASpace::mangle_region(MemRegion mr) {
  59   // This method should do nothing because numa spaces are not mangled.
  60 }
  61 void MutableNUMASpace::set_top_for_allocations(HeapWord* v) {
  62   assert(false, "Do not mangle MutableNUMASpace's");
  63 }
  64 void MutableNUMASpace::set_top_for_allocations() {
  65   // This method should do nothing.
  66 }
  67 void MutableNUMASpace::check_mangled_unused_area(HeapWord* limit) {
  68   // This method should do nothing.
  69 }
  70 void MutableNUMASpace::check_mangled_unused_area_complete() {
  71   // This method should do nothing.
  72 }
  73 #endif  // NOT_PRODUCT
  74 
  75 // There may be unallocated holes in the middle chunks
  76 // that should be filled with dead objects to ensure parsability.
  77 void MutableNUMASpace::ensure_parsability() {
  78   for (int i = 0; i < lgrp_spaces()->length(); i++) {
  79     LGRPSpace *ls = lgrp_spaces()->at(i);
  80     MutableSpace *s = ls->space();
  81     if (s->top() < top()) { // For all spaces preceding the one containing top()
  82       if (s->free_in_words() > 0) {
  83         intptr_t cur_top = (intptr_t)s->top();
  84         size_t words_left_to_fill = pointer_delta(s->end(), s->top());;
  85         while (words_left_to_fill > 0) {
  86           size_t words_to_fill = MIN2(words_left_to_fill, CollectedHeap::filler_array_max_size());
  87           assert(words_to_fill >= CollectedHeap::min_fill_size(),
  88                  "Remaining size (" SIZE_FORMAT ") is too small to fill (based on " SIZE_FORMAT " and " SIZE_FORMAT ")",
  89                  words_to_fill, words_left_to_fill, CollectedHeap::filler_array_max_size());
  90           CollectedHeap::fill_with_object((HeapWord*)cur_top, words_to_fill);
  91           if (!os::numa_has_static_binding()) {
  92             size_t touched_words = words_to_fill;
  93 #ifndef ASSERT
  94             if (!ZapUnusedHeapArea) {
  95               touched_words = MIN2((size_t)align_object_size(typeArrayOopDesc::header_size(T_INT)),
  96                 touched_words);
  97             }
  98 #endif
  99             MemRegion invalid;
 100             HeapWord *crossing_start = (HeapWord*)round_to(cur_top, os::vm_page_size());
 101             HeapWord *crossing_end = (HeapWord*)round_to(cur_top + touched_words, os::vm_page_size());
 102             if (crossing_start != crossing_end) {
 103               // If object header crossed a small page boundary we mark the area
 104               // as invalid rounding it to a page_size().
 105               HeapWord *start = MAX2((HeapWord*)round_down(cur_top, page_size()), s->bottom());
 106               HeapWord *end = MIN2((HeapWord*)round_to(cur_top + touched_words, page_size()), s->end());
 107               invalid = MemRegion(start, end);
 108             }
 109 
 110             ls->add_invalid_region(invalid);
 111           }
 112           cur_top = cur_top + (words_to_fill * HeapWordSize);
 113           words_left_to_fill -= words_to_fill;
 114         }
 115       }
 116     } else {
 117       if (!os::numa_has_static_binding()) {
 118 #ifdef ASSERT
 119         MemRegion invalid(s->top(), s->end());
 120         ls->add_invalid_region(invalid);
 121 #else
 122         if (ZapUnusedHeapArea) {
 123           MemRegion invalid(s->top(), s->end());
 124           ls->add_invalid_region(invalid);
 125         } else {
 126           return;
 127         }
 128 #endif
 129       } else {
 130           return;
 131       }
 132     }
 133   }
 134 }
 135 
 136 size_t MutableNUMASpace::used_in_words() const {
 137   size_t s = 0;
 138   for (int i = 0; i < lgrp_spaces()->length(); i++) {
 139     s += lgrp_spaces()->at(i)->space()->used_in_words();
 140   }
 141   return s;
 142 }
 143 
 144 size_t MutableNUMASpace::free_in_words() const {
 145   size_t s = 0;
 146   for (int i = 0; i < lgrp_spaces()->length(); i++) {
 147     s += lgrp_spaces()->at(i)->space()->free_in_words();
 148   }
 149   return s;
 150 }
 151 
 152 
 153 size_t MutableNUMASpace::tlab_capacity(Thread *thr) const {
 154   guarantee(thr != NULL, "No thread");
 155   int lgrp_id = thr->lgrp_id();
 156   if (lgrp_id == -1) {
 157     // This case can occur after the topology of the system has
 158     // changed. Thread can change their location, the new home
 159     // group will be determined during the first allocation
 160     // attempt. For now we can safely assume that all spaces
 161     // have equal size because the whole space will be reinitialized.
 162     if (lgrp_spaces()->length() > 0) {
 163       return capacity_in_bytes() / lgrp_spaces()->length();
 164     } else {
 165       assert(false, "There should be at least one locality group");
 166       return 0;
 167     }
 168   }
 169   // That's the normal case, where we know the locality group of the thread.
 170   int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals);
 171   if (i == -1) {
 172     return 0;
 173   }
 174   return lgrp_spaces()->at(i)->space()->capacity_in_bytes();
 175 }
 176 
 177 size_t MutableNUMASpace::tlab_used(Thread *thr) const {
 178   // Please see the comments for tlab_capacity().
 179   guarantee(thr != NULL, "No thread");
 180   int lgrp_id = thr->lgrp_id();
 181   if (lgrp_id == -1) {
 182     if (lgrp_spaces()->length() > 0) {
 183       return (used_in_bytes()) / lgrp_spaces()->length();
 184     } else {
 185       assert(false, "There should be at least one locality group");
 186       return 0;
 187     }
 188   }
 189   int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals);
 190   if (i == -1) {
 191     return 0;
 192   }
 193   return lgrp_spaces()->at(i)->space()->used_in_bytes();
 194 }
 195 
 196 
 197 size_t MutableNUMASpace::unsafe_max_tlab_alloc(Thread *thr) const {
 198   // Please see the comments for tlab_capacity().
 199   guarantee(thr != NULL, "No thread");
 200   int lgrp_id = thr->lgrp_id();
 201   if (lgrp_id == -1) {
 202     if (lgrp_spaces()->length() > 0) {
 203       return free_in_bytes() / lgrp_spaces()->length();
 204     } else {
 205       assert(false, "There should be at least one locality group");
 206       return 0;
 207     }
 208   }
 209   int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals);
 210   if (i == -1) {
 211     return 0;
 212   }
 213   return lgrp_spaces()->at(i)->space()->free_in_bytes();
 214 }
 215 
 216 
 217 size_t MutableNUMASpace::capacity_in_words(Thread* thr) const {
 218   guarantee(thr != NULL, "No thread");
 219   int lgrp_id = thr->lgrp_id();
 220   if (lgrp_id == -1) {
 221     if (lgrp_spaces()->length() > 0) {
 222       return capacity_in_words() / lgrp_spaces()->length();
 223     } else {
 224       assert(false, "There should be at least one locality group");
 225       return 0;
 226     }
 227   }
 228   int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals);
 229   if (i == -1) {
 230     return 0;
 231   }
 232   return lgrp_spaces()->at(i)->space()->capacity_in_words();
 233 }
 234 
 235 // Check if the NUMA topology has changed. Add and remove spaces if needed.
 236 // The update can be forced by setting the force parameter equal to true.
 237 bool MutableNUMASpace::update_layout(bool force) {
 238   // Check if the topology had changed.
 239   bool changed = os::numa_topology_changed();
 240   if (force || changed) {
 241     // Compute lgrp intersection. Add/remove spaces.
 242     int lgrp_limit = (int)os::numa_get_groups_num();
 243     int *lgrp_ids = NEW_C_HEAP_ARRAY(int, lgrp_limit, mtGC);
 244     int lgrp_num = (int)os::numa_get_leaf_groups(lgrp_ids, lgrp_limit);
 245     assert(lgrp_num > 0, "There should be at least one locality group");
 246     // Add new spaces for the new nodes
 247     for (int i = 0; i < lgrp_num; i++) {
 248       bool found = false;
 249       for (int j = 0; j < lgrp_spaces()->length(); j++) {
 250         if (lgrp_spaces()->at(j)->lgrp_id() == lgrp_ids[i]) {
 251           found = true;
 252           break;
 253         }
 254       }
 255       if (!found) {
 256         lgrp_spaces()->append(new LGRPSpace(lgrp_ids[i], alignment()));
 257       }
 258     }
 259 
 260     // Remove spaces for the removed nodes.
 261     for (int i = 0; i < lgrp_spaces()->length();) {
 262       bool found = false;
 263       for (int j = 0; j < lgrp_num; j++) {
 264         if (lgrp_spaces()->at(i)->lgrp_id() == lgrp_ids[j]) {
 265           found = true;
 266           break;
 267         }
 268       }
 269       if (!found) {
 270         delete lgrp_spaces()->at(i);
 271         lgrp_spaces()->remove_at(i);
 272       } else {
 273         i++;
 274       }
 275     }
 276 
 277     FREE_C_HEAP_ARRAY(int, lgrp_ids);
 278 
 279     if (changed) {
 280       for (JavaThread *thread = Threads::first(); thread; thread = thread->next()) {
 281         thread->set_lgrp_id(-1);
 282       }
 283     }
 284     return true;
 285   }
 286   return false;
 287 }
 288 
 289 // Bias region towards the first-touching lgrp. Set the right page sizes.
 290 void MutableNUMASpace::bias_region(MemRegion mr, int lgrp_id) {
 291   HeapWord *start = (HeapWord*)round_to((intptr_t)mr.start(), page_size());
 292   HeapWord *end = (HeapWord*)round_down((intptr_t)mr.end(), page_size());
 293   if (end > start) {
 294     MemRegion aligned_region(start, end);
 295     assert((intptr_t)aligned_region.start()     % page_size() == 0 &&
 296            (intptr_t)aligned_region.byte_size() % page_size() == 0, "Bad alignment");
 297     assert(region().contains(aligned_region), "Sanity");
 298     // First we tell the OS which page size we want in the given range. The underlying
 299     // large page can be broken down if we require small pages.
 300     os::realign_memory((char*)aligned_region.start(), aligned_region.byte_size(), page_size());
 301     // Then we uncommit the pages in the range.
 302     os::free_memory((char*)aligned_region.start(), aligned_region.byte_size(), page_size());
 303     // And make them local/first-touch biased.
 304     os::numa_make_local((char*)aligned_region.start(), aligned_region.byte_size(), lgrp_id);
 305   }
 306 }
 307 
 308 // Free all pages in the region.
 309 void MutableNUMASpace::free_region(MemRegion mr) {
 310   HeapWord *start = (HeapWord*)round_to((intptr_t)mr.start(), page_size());
 311   HeapWord *end = (HeapWord*)round_down((intptr_t)mr.end(), page_size());
 312   if (end > start) {
 313     MemRegion aligned_region(start, end);
 314     assert((intptr_t)aligned_region.start()     % page_size() == 0 &&
 315            (intptr_t)aligned_region.byte_size() % page_size() == 0, "Bad alignment");
 316     assert(region().contains(aligned_region), "Sanity");
 317     os::free_memory((char*)aligned_region.start(), aligned_region.byte_size(), page_size());
 318   }
 319 }
 320 
 321 // Update space layout. Perform adaptation.
 322 void MutableNUMASpace::update() {
 323   if (update_layout(false)) {
 324     // If the topology has changed, make all chunks zero-sized.
 325     // And clear the alloc-rate statistics.
 326     // In future we may want to handle this more gracefully in order
 327     // to avoid the reallocation of the pages as much as possible.
 328     for (int i = 0; i < lgrp_spaces()->length(); i++) {
 329       LGRPSpace *ls = lgrp_spaces()->at(i);
 330       MutableSpace *s = ls->space();
 331       s->set_end(s->bottom());
 332       s->set_top(s->bottom());
 333       ls->clear_alloc_rate();
 334     }
 335     // A NUMA space is never mangled
 336     initialize(region(),
 337                SpaceDecorator::Clear,
 338                SpaceDecorator::DontMangle);
 339   } else {
 340     bool should_initialize = false;
 341     if (!os::numa_has_static_binding()) {
 342       for (int i = 0; i < lgrp_spaces()->length(); i++) {
 343         if (!lgrp_spaces()->at(i)->invalid_region().is_empty()) {
 344           should_initialize = true;
 345           break;
 346         }
 347       }
 348     }
 349 
 350     if (should_initialize ||
 351         (UseAdaptiveNUMAChunkSizing && adaptation_cycles() < samples_count())) {
 352       // A NUMA space is never mangled
 353       initialize(region(),
 354                  SpaceDecorator::Clear,
 355                  SpaceDecorator::DontMangle);
 356     }
 357   }
 358 
 359   if (NUMAStats) {
 360     for (int i = 0; i < lgrp_spaces()->length(); i++) {
 361       lgrp_spaces()->at(i)->accumulate_statistics(page_size());
 362     }
 363   }
 364 
 365   scan_pages(NUMAPageScanRate);
 366 }
 367 
 368 // Scan pages. Free pages that have smaller size or wrong placement.
 369 void MutableNUMASpace::scan_pages(size_t page_count)
 370 {
 371   size_t pages_per_chunk = page_count / lgrp_spaces()->length();
 372   if (pages_per_chunk > 0) {
 373     for (int i = 0; i < lgrp_spaces()->length(); i++) {
 374       LGRPSpace *ls = lgrp_spaces()->at(i);
 375       ls->scan_pages(page_size(), pages_per_chunk);
 376     }
 377   }
 378 }
 379 
 380 // Accumulate statistics about the allocation rate of each lgrp.
 381 void MutableNUMASpace::accumulate_statistics() {
 382   if (UseAdaptiveNUMAChunkSizing) {
 383     for (int i = 0; i < lgrp_spaces()->length(); i++) {
 384       lgrp_spaces()->at(i)->sample();
 385     }
 386     increment_samples_count();
 387   }
 388 
 389   if (NUMAStats) {
 390     for (int i = 0; i < lgrp_spaces()->length(); i++) {
 391       lgrp_spaces()->at(i)->accumulate_statistics(page_size());
 392     }
 393   }
 394 }
 395 
 396 // Get the current size of a chunk.
 397 // This function computes the size of the chunk based on the
 398 // difference between chunk ends. This allows it to work correctly in
 399 // case the whole space is resized and during the process of adaptive
 400 // chunk resizing.
 401 size_t MutableNUMASpace::current_chunk_size(int i) {
 402   HeapWord *cur_end, *prev_end;
 403   if (i == 0) {
 404     prev_end = bottom();
 405   } else {
 406     prev_end = lgrp_spaces()->at(i - 1)->space()->end();
 407   }
 408   if (i == lgrp_spaces()->length() - 1) {
 409     cur_end = end();
 410   } else {
 411     cur_end = lgrp_spaces()->at(i)->space()->end();
 412   }
 413   if (cur_end > prev_end) {
 414     return pointer_delta(cur_end, prev_end, sizeof(char));
 415   }
 416   return 0;
 417 }
 418 
 419 // Return the default chunk size by equally diving the space.
 420 // page_size() aligned.
 421 size_t MutableNUMASpace::default_chunk_size() {
 422   return base_space_size() / lgrp_spaces()->length() * page_size();
 423 }
 424 
 425 // Produce a new chunk size. page_size() aligned.
 426 // This function is expected to be called on sequence of i's from 0 to
 427 // lgrp_spaces()->length().
 428 size_t MutableNUMASpace::adaptive_chunk_size(int i, size_t limit) {
 429   size_t pages_available = base_space_size();
 430   for (int j = 0; j < i; j++) {
 431     pages_available -= round_down(current_chunk_size(j), page_size()) / page_size();
 432   }
 433   pages_available -= lgrp_spaces()->length() - i - 1;
 434   assert(pages_available > 0, "No pages left");
 435   float alloc_rate = 0;
 436   for (int j = i; j < lgrp_spaces()->length(); j++) {
 437     alloc_rate += lgrp_spaces()->at(j)->alloc_rate()->average();
 438   }
 439   size_t chunk_size = 0;
 440   if (alloc_rate > 0) {
 441     LGRPSpace *ls = lgrp_spaces()->at(i);
 442     chunk_size = (size_t)(ls->alloc_rate()->average() / alloc_rate * pages_available) * page_size();
 443   }
 444   chunk_size = MAX2(chunk_size, page_size());
 445 
 446   if (limit > 0) {
 447     limit = round_down(limit, page_size());
 448     if (chunk_size > current_chunk_size(i)) {
 449       size_t upper_bound = pages_available * page_size();
 450       if (upper_bound > limit &&
 451           current_chunk_size(i) < upper_bound - limit) {
 452         // The resulting upper bound should not exceed the available
 453         // amount of memory (pages_available * page_size()).
 454         upper_bound = current_chunk_size(i) + limit;
 455       }
 456       chunk_size = MIN2(chunk_size, upper_bound);
 457     } else {
 458       size_t lower_bound = page_size();
 459       if (current_chunk_size(i) > limit) { // lower_bound shouldn't underflow.
 460         lower_bound = current_chunk_size(i) - limit;
 461       }
 462       chunk_size = MAX2(chunk_size, lower_bound);
 463     }
 464   }
 465   assert(chunk_size <= pages_available * page_size(), "Chunk size out of range");
 466   return chunk_size;
 467 }
 468 
 469 
 470 // Return the bottom_region and the top_region. Align them to page_size() boundary.
 471 // |------------------new_region---------------------------------|
 472 // |----bottom_region--|---intersection---|------top_region------|
 473 void MutableNUMASpace::select_tails(MemRegion new_region, MemRegion intersection,
 474                                     MemRegion* bottom_region, MemRegion *top_region) {
 475   // Is there bottom?
 476   if (new_region.start() < intersection.start()) { // Yes
 477     // Try to coalesce small pages into a large one.
 478     if (UseLargePages && page_size() >= alignment()) {
 479       HeapWord* p = (HeapWord*)round_to((intptr_t) intersection.start(), alignment());
 480       if (new_region.contains(p)
 481           && pointer_delta(p, new_region.start(), sizeof(char)) >= alignment()) {
 482         if (intersection.contains(p)) {
 483           intersection = MemRegion(p, intersection.end());
 484         } else {
 485           intersection = MemRegion(p, p);
 486         }
 487       }
 488     }
 489     *bottom_region = MemRegion(new_region.start(), intersection.start());
 490   } else {
 491     *bottom_region = MemRegion();
 492   }
 493 
 494   // Is there top?
 495   if (intersection.end() < new_region.end()) { // Yes
 496     // Try to coalesce small pages into a large one.
 497     if (UseLargePages && page_size() >= alignment()) {
 498       HeapWord* p = (HeapWord*)round_down((intptr_t) intersection.end(), alignment());
 499       if (new_region.contains(p)
 500           && pointer_delta(new_region.end(), p, sizeof(char)) >= alignment()) {
 501         if (intersection.contains(p)) {
 502           intersection = MemRegion(intersection.start(), p);
 503         } else {
 504           intersection = MemRegion(p, p);
 505         }
 506       }
 507     }
 508     *top_region = MemRegion(intersection.end(), new_region.end());
 509   } else {
 510     *top_region = MemRegion();
 511   }
 512 }
 513 
 514 // Try to merge the invalid region with the bottom or top region by decreasing
 515 // the intersection area. Return the invalid_region aligned to the page_size()
 516 // boundary if it's inside the intersection. Return non-empty invalid_region
 517 // if it lies inside the intersection (also page-aligned).
 518 // |------------------new_region---------------------------------|
 519 // |----------------|-------invalid---|--------------------------|
 520 // |----bottom_region--|---intersection---|------top_region------|
 521 void MutableNUMASpace::merge_regions(MemRegion new_region, MemRegion* intersection,
 522                                      MemRegion *invalid_region) {
 523   if (intersection->start() >= invalid_region->start() && intersection->contains(invalid_region->end())) {
 524     *intersection = MemRegion(invalid_region->end(), intersection->end());
 525     *invalid_region = MemRegion();
 526   } else
 527     if (intersection->end() <= invalid_region->end() && intersection->contains(invalid_region->start())) {
 528       *intersection = MemRegion(intersection->start(), invalid_region->start());
 529       *invalid_region = MemRegion();
 530     } else
 531       if (intersection->equals(*invalid_region) || invalid_region->contains(*intersection)) {
 532         *intersection = MemRegion(new_region.start(), new_region.start());
 533         *invalid_region = MemRegion();
 534       } else
 535         if (intersection->contains(invalid_region)) {
 536             // That's the only case we have to make an additional bias_region() call.
 537             HeapWord* start = invalid_region->start();
 538             HeapWord* end = invalid_region->end();
 539             if (UseLargePages && page_size() >= alignment()) {
 540               HeapWord *p = (HeapWord*)round_down((intptr_t) start, alignment());
 541               if (new_region.contains(p)) {
 542                 start = p;
 543               }
 544               p = (HeapWord*)round_to((intptr_t) end, alignment());
 545               if (new_region.contains(end)) {
 546                 end = p;
 547               }
 548             }
 549             if (intersection->start() > start) {
 550               *intersection = MemRegion(start, intersection->end());
 551             }
 552             if (intersection->end() < end) {
 553               *intersection = MemRegion(intersection->start(), end);
 554             }
 555             *invalid_region = MemRegion(start, end);
 556         }
 557 }
 558 
 559 void MutableNUMASpace::initialize(MemRegion mr,
 560                                   bool clear_space,
 561                                   bool mangle_space,
 562                                   bool setup_pages) {
 563   assert(clear_space, "Reallocation will destroy data!");
 564   assert(lgrp_spaces()->length() > 0, "There should be at least one space");
 565 
 566   MemRegion old_region = region(), new_region;
 567   set_bottom(mr.start());
 568   set_end(mr.end());
 569   // Must always clear the space
 570   clear(SpaceDecorator::DontMangle);
 571 
 572   // Compute chunk sizes
 573   size_t prev_page_size = page_size();
 574   set_page_size(UseLargePages ? alignment() : os::vm_page_size());
 575   HeapWord* rounded_bottom = (HeapWord*)round_to((intptr_t) bottom(), page_size());
 576   HeapWord* rounded_end = (HeapWord*)round_down((intptr_t) end(), page_size());
 577   size_t base_space_size_pages = pointer_delta(rounded_end, rounded_bottom, sizeof(char)) / page_size();
 578 
 579   // Try small pages if the chunk size is too small
 580   if (base_space_size_pages / lgrp_spaces()->length() == 0
 581       && page_size() > (size_t)os::vm_page_size()) {
 582     set_page_size(os::vm_page_size());
 583     rounded_bottom = (HeapWord*)round_to((intptr_t) bottom(), page_size());
 584     rounded_end = (HeapWord*)round_down((intptr_t) end(), page_size());
 585     base_space_size_pages = pointer_delta(rounded_end, rounded_bottom, sizeof(char)) / page_size();
 586   }
 587   guarantee(base_space_size_pages / lgrp_spaces()->length() > 0, "Space too small");
 588   set_base_space_size(base_space_size_pages);
 589 
 590   // Handle space resize
 591   MemRegion top_region, bottom_region;
 592   if (!old_region.equals(region())) {
 593     new_region = MemRegion(rounded_bottom, rounded_end);
 594     MemRegion intersection = new_region.intersection(old_region);
 595     if (intersection.start() == NULL ||
 596         intersection.end() == NULL   ||
 597         prev_page_size > page_size()) { // If the page size got smaller we have to change
 598                                         // the page size preference for the whole space.
 599       intersection = MemRegion(new_region.start(), new_region.start());
 600     }
 601     select_tails(new_region, intersection, &bottom_region, &top_region);
 602     bias_region(bottom_region, lgrp_spaces()->at(0)->lgrp_id());
 603     bias_region(top_region, lgrp_spaces()->at(lgrp_spaces()->length() - 1)->lgrp_id());
 604   }
 605 
 606   // Check if the space layout has changed significantly?
 607   // This happens when the space has been resized so that either head or tail
 608   // chunk became less than a page.
 609   bool layout_valid = UseAdaptiveNUMAChunkSizing          &&
 610                       current_chunk_size(0) > page_size() &&
 611                       current_chunk_size(lgrp_spaces()->length() - 1) > page_size();
 612 
 613 
 614   for (int i = 0; i < lgrp_spaces()->length(); i++) {
 615     LGRPSpace *ls = lgrp_spaces()->at(i);
 616     MutableSpace *s = ls->space();
 617     old_region = s->region();
 618 
 619     size_t chunk_byte_size = 0, old_chunk_byte_size = 0;
 620     if (i < lgrp_spaces()->length() - 1) {
 621       if (!UseAdaptiveNUMAChunkSizing                                ||
 622           (UseAdaptiveNUMAChunkSizing && NUMAChunkResizeWeight == 0) ||
 623            samples_count() < AdaptiveSizePolicyReadyThreshold) {
 624         // No adaptation. Divide the space equally.
 625         chunk_byte_size = default_chunk_size();
 626       } else
 627         if (!layout_valid || NUMASpaceResizeRate == 0) {
 628           // Fast adaptation. If no space resize rate is set, resize
 629           // the chunks instantly.
 630           chunk_byte_size = adaptive_chunk_size(i, 0);
 631         } else {
 632           // Slow adaptation. Resize the chunks moving no more than
 633           // NUMASpaceResizeRate bytes per collection.
 634           size_t limit = NUMASpaceResizeRate /
 635                          (lgrp_spaces()->length() * (lgrp_spaces()->length() + 1) / 2);
 636           chunk_byte_size = adaptive_chunk_size(i, MAX2(limit * (i + 1), page_size()));
 637         }
 638 
 639       assert(chunk_byte_size >= page_size(), "Chunk size too small");
 640       assert(chunk_byte_size <= capacity_in_bytes(), "Sanity check");
 641     }
 642 
 643     if (i == 0) { // Bottom chunk
 644       if (i != lgrp_spaces()->length() - 1) {
 645         new_region = MemRegion(bottom(), rounded_bottom + (chunk_byte_size >> LogHeapWordSize));
 646       } else {
 647         new_region = MemRegion(bottom(), end());
 648       }
 649     } else
 650       if (i < lgrp_spaces()->length() - 1) { // Middle chunks
 651         MutableSpace *ps = lgrp_spaces()->at(i - 1)->space();
 652         new_region = MemRegion(ps->end(),
 653                                ps->end() + (chunk_byte_size >> LogHeapWordSize));
 654       } else { // Top chunk
 655         MutableSpace *ps = lgrp_spaces()->at(i - 1)->space();
 656         new_region = MemRegion(ps->end(), end());
 657       }
 658     guarantee(region().contains(new_region), "Region invariant");
 659 
 660 
 661     // The general case:
 662     // |---------------------|--invalid---|--------------------------|
 663     // |------------------new_region---------------------------------|
 664     // |----bottom_region--|---intersection---|------top_region------|
 665     //                     |----old_region----|
 666     // The intersection part has all pages in place we don't need to migrate them.
 667     // Pages for the top and bottom part should be freed and then reallocated.
 668 
 669     MemRegion intersection = old_region.intersection(new_region);
 670 
 671     if (intersection.start() == NULL || intersection.end() == NULL) {
 672       intersection = MemRegion(new_region.start(), new_region.start());
 673     }
 674 
 675     if (!os::numa_has_static_binding()) {
 676       MemRegion invalid_region = ls->invalid_region().intersection(new_region);
 677       // Invalid region is a range of memory that could've possibly
 678       // been allocated on the other node. That's relevant only on Solaris where
 679       // there is no static memory binding.
 680       if (!invalid_region.is_empty()) {
 681         merge_regions(new_region, &intersection, &invalid_region);
 682         free_region(invalid_region);
 683         ls->set_invalid_region(MemRegion());
 684       }
 685     }
 686 
 687     select_tails(new_region, intersection, &bottom_region, &top_region);
 688 
 689     if (!os::numa_has_static_binding()) {
 690       // If that's a system with the first-touch policy then it's enough
 691       // to free the pages.
 692       free_region(bottom_region);
 693       free_region(top_region);
 694     } else {
 695       // In a system with static binding we have to change the bias whenever
 696       // we reshape the heap.
 697       bias_region(bottom_region, ls->lgrp_id());
 698       bias_region(top_region, ls->lgrp_id());
 699     }
 700 
 701     // Clear space (set top = bottom) but never mangle.
 702     s->initialize(new_region, SpaceDecorator::Clear, SpaceDecorator::DontMangle, MutableSpace::DontSetupPages);
 703 
 704     set_adaptation_cycles(samples_count());
 705   }
 706 }
 707 
 708 // Set the top of the whole space.
 709 // Mark the the holes in chunks below the top() as invalid.
 710 void MutableNUMASpace::set_top(HeapWord* value) {
 711   bool found_top = false;
 712   for (int i = 0; i < lgrp_spaces()->length();) {
 713     LGRPSpace *ls = lgrp_spaces()->at(i);
 714     MutableSpace *s = ls->space();
 715     HeapWord *top = MAX2((HeapWord*)round_down((intptr_t)s->top(), page_size()), s->bottom());
 716 
 717     if (s->contains(value)) {
 718       // Check if setting the chunk's top to a given value would create a hole less than
 719       // a minimal object; assuming that's not the last chunk in which case we don't care.
 720       if (i < lgrp_spaces()->length() - 1) {
 721         size_t remainder = pointer_delta(s->end(), value);
 722         const size_t min_fill_size = CollectedHeap::min_fill_size();
 723         if (remainder < min_fill_size && remainder > 0) {
 724           // Add a minimum size filler object; it will cross the chunk boundary.
 725           CollectedHeap::fill_with_object(value, min_fill_size);
 726           value += min_fill_size;
 727           assert(!s->contains(value), "Should be in the next chunk");
 728           // Restart the loop from the same chunk, since the value has moved
 729           // to the next one.
 730           continue;
 731         }
 732       }
 733 
 734       if (!os::numa_has_static_binding() && top < value && top < s->end()) {
 735         ls->add_invalid_region(MemRegion(top, value));
 736       }
 737       s->set_top(value);
 738       found_top = true;
 739     } else {
 740         if (found_top) {
 741             s->set_top(s->bottom());
 742         } else {
 743           if (!os::numa_has_static_binding() && top < s->end()) {
 744             ls->add_invalid_region(MemRegion(top, s->end()));
 745           }
 746           s->set_top(s->end());
 747         }
 748     }
 749     i++;
 750   }
 751   MutableSpace::set_top(value);
 752 }
 753 
 754 void MutableNUMASpace::clear(bool mangle_space) {
 755   MutableSpace::set_top(bottom());
 756   for (int i = 0; i < lgrp_spaces()->length(); i++) {
 757     // Never mangle NUMA spaces because the mangling will
 758     // bind the memory to a possibly unwanted lgroup.
 759     lgrp_spaces()->at(i)->space()->clear(SpaceDecorator::DontMangle);
 760   }
 761 }
 762 
 763 /*
 764    Linux supports static memory binding, therefore the most part of the
 765    logic dealing with the possible invalid page allocation is effectively
 766    disabled. Besides there is no notion of the home node in Linux. A
 767    thread is allowed to migrate freely. Although the scheduler is rather
 768    reluctant to move threads between the nodes. We check for the current
 769    node every allocation. And with a high probability a thread stays on
 770    the same node for some time allowing local access to recently allocated
 771    objects.
 772  */
 773 
 774 HeapWord* MutableNUMASpace::allocate(size_t size) {
 775   Thread* thr = Thread::current();
 776   int lgrp_id = thr->lgrp_id();
 777   if (lgrp_id == -1 || !os::numa_has_group_homing()) {
 778     lgrp_id = os::numa_get_group_id();
 779     thr->set_lgrp_id(lgrp_id);
 780   }
 781 
 782   int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals);
 783 
 784   // It is possible that a new CPU has been hotplugged and
 785   // we haven't reshaped the space accordingly.
 786   if (i == -1) {
 787     i = os::random() % lgrp_spaces()->length();
 788   }
 789 
 790   LGRPSpace* ls = lgrp_spaces()->at(i);
 791   MutableSpace *s = ls->space();
 792   HeapWord *p = s->allocate(size);
 793 
 794   if (p != NULL) {
 795     size_t remainder = s->free_in_words();
 796     if (remainder < CollectedHeap::min_fill_size() && remainder > 0) {
 797       s->set_top(s->top() - size);
 798       p = NULL;
 799     }
 800   }
 801   if (p != NULL) {
 802     if (top() < s->top()) { // Keep _top updated.
 803       MutableSpace::set_top(s->top());
 804     }
 805   }
 806   // Make the page allocation happen here if there is no static binding..
 807   if (p != NULL && !os::numa_has_static_binding()) {
 808     for (HeapWord *i = p; i < p + size; i += os::vm_page_size() >> LogHeapWordSize) {
 809       *(int*)i = 0;
 810     }
 811   }
 812   if (p == NULL) {
 813     ls->set_allocation_failed();
 814   }
 815   return p;
 816 }
 817 
 818 // This version is lock-free.
 819 HeapWord* MutableNUMASpace::cas_allocate(size_t size) {
 820   Thread* thr = Thread::current();
 821   int lgrp_id = thr->lgrp_id();
 822   if (lgrp_id == -1 || !os::numa_has_group_homing()) {
 823     lgrp_id = os::numa_get_group_id();
 824     thr->set_lgrp_id(lgrp_id);
 825   }
 826 
 827   int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals);
 828   // It is possible that a new CPU has been hotplugged and
 829   // we haven't reshaped the space accordingly.
 830   if (i == -1) {
 831     i = os::random() % lgrp_spaces()->length();
 832   }
 833   LGRPSpace *ls = lgrp_spaces()->at(i);
 834   MutableSpace *s = ls->space();
 835   HeapWord *p = s->cas_allocate(size);
 836   if (p != NULL) {
 837     size_t remainder = pointer_delta(s->end(), p + size);
 838     if (remainder < CollectedHeap::min_fill_size() && remainder > 0) {
 839       if (s->cas_deallocate(p, size)) {
 840         // We were the last to allocate and created a fragment less than
 841         // a minimal object.
 842         p = NULL;
 843       } else {
 844         guarantee(false, "Deallocation should always succeed");
 845       }
 846     }
 847   }
 848   if (p != NULL) {
 849     HeapWord* cur_top, *cur_chunk_top = p + size;
 850     while ((cur_top = top()) < cur_chunk_top) { // Keep _top updated.
 851       if (Atomic::cmpxchg_ptr(cur_chunk_top, top_addr(), cur_top) == cur_top) {
 852         break;
 853       }
 854     }
 855   }
 856 
 857   // Make the page allocation happen here if there is no static binding.
 858   if (p != NULL && !os::numa_has_static_binding() ) {
 859     for (HeapWord *i = p; i < p + size; i += os::vm_page_size() >> LogHeapWordSize) {
 860       *(int*)i = 0;
 861     }
 862   }
 863   if (p == NULL) {
 864     ls->set_allocation_failed();
 865   }
 866   return p;
 867 }
 868 
 869 void MutableNUMASpace::print_short_on(outputStream* st) const {
 870   MutableSpace::print_short_on(st);
 871   st->print(" (");
 872   for (int i = 0; i < lgrp_spaces()->length(); i++) {
 873     st->print("lgrp %d: ", lgrp_spaces()->at(i)->lgrp_id());
 874     lgrp_spaces()->at(i)->space()->print_short_on(st);
 875     if (i < lgrp_spaces()->length() - 1) {
 876       st->print(", ");
 877     }
 878   }
 879   st->print(")");
 880 }
 881 
 882 void MutableNUMASpace::print_on(outputStream* st) const {
 883   MutableSpace::print_on(st);
 884   for (int i = 0; i < lgrp_spaces()->length(); i++) {
 885     LGRPSpace *ls = lgrp_spaces()->at(i);
 886     st->print("    lgrp %d", ls->lgrp_id());
 887     ls->space()->print_on(st);
 888     if (NUMAStats) {
 889       for (int i = 0; i < lgrp_spaces()->length(); i++) {
 890         lgrp_spaces()->at(i)->accumulate_statistics(page_size());
 891       }
 892       st->print("    local/remote/unbiased/uncommitted: " SIZE_FORMAT "K/"
 893                 SIZE_FORMAT "K/" SIZE_FORMAT "K/" SIZE_FORMAT
 894                 "K, large/small pages: " SIZE_FORMAT "/" SIZE_FORMAT "\n",
 895                 ls->space_stats()->_local_space / K,
 896                 ls->space_stats()->_remote_space / K,
 897                 ls->space_stats()->_unbiased_space / K,
 898                 ls->space_stats()->_uncommited_space / K,
 899                 ls->space_stats()->_large_pages,
 900                 ls->space_stats()->_small_pages);
 901     }
 902   }
 903 }
 904 
 905 void MutableNUMASpace::verify() {
 906   // This can be called after setting an arbitrary value to the space's top,
 907   // so an object can cross the chunk boundary. We ensure the parsability
 908   // of the space and just walk the objects in linear fashion.
 909   ensure_parsability();
 910   MutableSpace::verify();
 911 }
 912 
 913 // Scan pages and gather stats about page placement and size.
 914 void MutableNUMASpace::LGRPSpace::accumulate_statistics(size_t page_size) {
 915   clear_space_stats();
 916   char *start = (char*)round_to((intptr_t) space()->bottom(), page_size);
 917   char* end = (char*)round_down((intptr_t) space()->end(), page_size);
 918   if (start < end) {
 919     for (char *p = start; p < end;) {
 920       os::page_info info;
 921       if (os::get_page_info(p, &info)) {
 922         if (info.size > 0) {
 923           if (info.size > (size_t)os::vm_page_size()) {
 924             space_stats()->_large_pages++;
 925           } else {
 926             space_stats()->_small_pages++;
 927           }
 928           if (info.lgrp_id == lgrp_id()) {
 929             space_stats()->_local_space += info.size;
 930           } else {
 931             space_stats()->_remote_space += info.size;
 932           }
 933           p += info.size;
 934         } else {
 935           p += os::vm_page_size();
 936           space_stats()->_uncommited_space += os::vm_page_size();
 937         }
 938       } else {
 939         return;
 940       }
 941     }
 942   }
 943   space_stats()->_unbiased_space = pointer_delta(start, space()->bottom(), sizeof(char)) +
 944                                    pointer_delta(space()->end(), end, sizeof(char));
 945 
 946 }
 947 
 948 // Scan page_count pages and verify if they have the right size and right placement.
 949 // If invalid pages are found they are freed in hope that subsequent reallocation
 950 // will be more successful.
 951 void MutableNUMASpace::LGRPSpace::scan_pages(size_t page_size, size_t page_count)
 952 {
 953   char* range_start = (char*)round_to((intptr_t) space()->bottom(), page_size);
 954   char* range_end = (char*)round_down((intptr_t) space()->end(), page_size);
 955 
 956   if (range_start > last_page_scanned() || last_page_scanned() >= range_end) {
 957     set_last_page_scanned(range_start);
 958   }
 959 
 960   char *scan_start = last_page_scanned();
 961   char* scan_end = MIN2(scan_start + page_size * page_count, range_end);
 962 
 963   os::page_info page_expected, page_found;
 964   page_expected.size = page_size;
 965   page_expected.lgrp_id = lgrp_id();
 966 
 967   char *s = scan_start;
 968   while (s < scan_end) {
 969     char *e = os::scan_pages(s, (char*)scan_end, &page_expected, &page_found);
 970     if (e == NULL) {
 971       break;
 972     }
 973     if (e != scan_end) {
 974       assert(e < scan_end, "e: " PTR_FORMAT " scan_end: " PTR_FORMAT, p2i(e), p2i(scan_end));
 975 
 976       if ((page_expected.size != page_size || page_expected.lgrp_id != lgrp_id())
 977           && page_expected.size != 0) {
 978         os::free_memory(s, pointer_delta(e, s, sizeof(char)), page_size);
 979       }
 980       page_expected = page_found;
 981     }
 982     s = e;
 983   }
 984 
 985   set_last_page_scanned(scan_end);
 986 }