rev 11780 : [mq]: webrev.0
rev 11781 : [mq]: webrev.1

   1 
   2 /*
   3  * Copyright (c) 2006, 2016, Oracle and/or its affiliates. All rights reserved.
   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This code is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU General Public License version 2 only, as
   8  * published by the Free Software Foundation.
   9  *
  10  * This code is distributed in the hope that it will be useful, but WITHOUT
  11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  13  * version 2 for more details (a copy is included in the LICENSE file that
  14  * accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License version
  17  * 2 along with this work; if not, write to the Free Software Foundation,
  18  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  19  *
  20  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  21  * or visit www.oracle.com if you need additional information or have any
  22  * questions.
  23  *
  24  */
  25 
  26 #include "precompiled.hpp"
  27 #include "gc/parallel/mutableNUMASpace.hpp"
  28 #include "gc/shared/collectedHeap.hpp"
  29 #include "gc/shared/spaceDecorator.hpp"
  30 #include "oops/oop.inline.hpp"
  31 #include "runtime/atomic.inline.hpp"
  32 #include "runtime/thread.inline.hpp"
  33 
  34 MutableNUMASpace::MutableNUMASpace(size_t alignment) : MutableSpace(alignment) {
  35   _lgrp_spaces = new (ResourceObj::C_HEAP, mtGC) GrowableArray<LGRPSpace*>(0, true);
  36   _page_size = os::vm_page_size();
  37   _adaptation_cycles = 0;
  38   _samples_count = 0;
  39   update_layout(true);
  40 }
  41 
  42 MutableNUMASpace::~MutableNUMASpace() {
  43   for (int i = 0; i < lgrp_spaces()->length(); i++) {
  44     delete lgrp_spaces()->at(i);
  45   }
  46   delete lgrp_spaces();
  47 }
  48 
  49 #ifndef PRODUCT
  50 void MutableNUMASpace::mangle_unused_area() {
  51   // This method should do nothing.
  52   // It can be called on a numa space during a full compaction.
  53 }
  54 void MutableNUMASpace::mangle_unused_area_complete() {
  55   // This method should do nothing.
  56   // It can be called on a numa space during a full compaction.
  57 }
  58 void MutableNUMASpace::mangle_region(MemRegion mr) {
  59   // This method should do nothing because numa spaces are not mangled.
  60 }
  61 void MutableNUMASpace::set_top_for_allocations(HeapWord* v) {
  62   assert(false, "Do not mangle MutableNUMASpace's");
  63 }
  64 void MutableNUMASpace::set_top_for_allocations() {
  65   // This method should do nothing.
  66 }
  67 void MutableNUMASpace::check_mangled_unused_area(HeapWord* limit) {
  68   // This method should do nothing.
  69 }
  70 void MutableNUMASpace::check_mangled_unused_area_complete() {
  71   // This method should do nothing.
  72 }
  73 #endif  // NOT_PRODUCT
  74 
  75 // There may be unallocated holes in the middle chunks
  76 // that should be filled with dead objects to ensure parsability.
  77 void MutableNUMASpace::ensure_parsability() {
  78   for (int i = 0; i < lgrp_spaces()->length(); i++) {
  79     LGRPSpace *ls = lgrp_spaces()->at(i);
  80     MutableSpace *s = ls->space();
  81     if (s->top() < top()) { // For all spaces preceding the one containing top()
  82       if (s->free_in_words() > 0) {
  83         intptr_t cur_top = (intptr_t)s->top();
  84         size_t words_left_to_fill = pointer_delta(s->end(), s->top());;
  85         while (words_left_to_fill > 0) {
  86           size_t words_to_fill = MIN2(words_left_to_fill, CollectedHeap::filler_array_max_size());
  87           assert(words_to_fill >= CollectedHeap::min_fill_size(),
  88                  "Remaining size (" SIZE_FORMAT ") is too small to fill (based on " SIZE_FORMAT " and " SIZE_FORMAT ")",
  89                  words_to_fill, words_left_to_fill, CollectedHeap::filler_array_max_size());
  90           CollectedHeap::fill_with_object((HeapWord*)cur_top, words_to_fill);
  91           if (!os::numa_has_static_binding()) {
  92             size_t touched_words = words_to_fill;
  93 #ifndef ASSERT
  94             if (!ZapUnusedHeapArea) {
  95               touched_words = MIN2((size_t)align_object_size(typeArrayOopDesc::header_size(T_INT)),
  96                 touched_words);
  97             }
  98 #endif
  99             MemRegion invalid;
 100             HeapWord *crossing_start = (HeapWord*)round_to(cur_top, os::vm_page_size());
 101             HeapWord *crossing_end = (HeapWord*)round_to(cur_top + touched_words, os::vm_page_size());
 102             if (crossing_start != crossing_end) {
 103               // If object header crossed a small page boundary we mark the area
 104               // as invalid rounding it to a page_size().
 105               HeapWord *start = MAX2((HeapWord*)round_down(cur_top, page_size()), s->bottom());
 106               HeapWord *end = MIN2((HeapWord*)round_to(cur_top + touched_words, page_size()), s->end());
 107               invalid = MemRegion(start, end);
 108             }
 109 
 110             ls->add_invalid_region(invalid);
 111           }
 112           cur_top = cur_top + (words_to_fill * HeapWordSize);
 113           words_left_to_fill -= words_to_fill;
 114         }
 115       }
 116     } else {
 117       if (!os::numa_has_static_binding()) {
 118 #ifdef ASSERT
 119         MemRegion invalid(s->top(), s->end());
 120         ls->add_invalid_region(invalid);
 121 #else
 122         if (ZapUnusedHeapArea) {
 123           MemRegion invalid(s->top(), s->end());
 124           ls->add_invalid_region(invalid);
 125         } else {
 126           return;
 127         }
 128 #endif
 129       } else {
 130           return;
 131       }
 132     }
 133   }
 134 }
 135 
 136 size_t MutableNUMASpace::used_in_words() const {
 137   size_t s = 0;
 138   for (int i = 0; i < lgrp_spaces()->length(); i++) {
 139     s += lgrp_spaces()->at(i)->space()->used_in_words();
 140   }
 141   return s;
 142 }
 143 
 144 size_t MutableNUMASpace::free_in_words() const {
 145   size_t s = 0;
 146   for (int i = 0; i < lgrp_spaces()->length(); i++) {
 147     s += lgrp_spaces()->at(i)->space()->free_in_words();
 148   }
 149   return s;
 150 }
 151 
 152 
 153 size_t MutableNUMASpace::tlab_capacity(Thread *thr) const {
 154   guarantee(thr != NULL, "No thread");
 155   int lgrp_id = thr->lgrp_id();
 156   if (lgrp_id == -1) {
 157     // This case can occur after the topology of the system has
 158     // changed. Thread can change their location, the new home
 159     // group will be determined during the first allocation
 160     // attempt. For now we can safely assume that all spaces
 161     // have equal size because the whole space will be reinitialized.
 162     if (lgrp_spaces()->length() > 0) {
 163       return capacity_in_bytes() / lgrp_spaces()->length();
 164     } else {
 165       assert(false, "There should be at least one locality group");
 166       return 0;
 167     }
 168   }
 169   // That's the normal case, where we know the locality group of the thread.
 170   int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals);
 171   if (i == -1) {
 172     return 0;
 173   }
 174   return lgrp_spaces()->at(i)->space()->capacity_in_bytes();
 175 }
 176 
 177 size_t MutableNUMASpace::tlab_used(Thread *thr) const {
 178   // Please see the comments for tlab_capacity().
 179   guarantee(thr != NULL, "No thread");
 180   int lgrp_id = thr->lgrp_id();
 181   if (lgrp_id == -1) {
 182     if (lgrp_spaces()->length() > 0) {
 183       return (used_in_bytes()) / lgrp_spaces()->length();
 184     } else {
 185       assert(false, "There should be at least one locality group");
 186       return 0;
 187     }
 188   }
 189   int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals);
 190   if (i == -1) {
 191     return 0;
 192   }
 193   return lgrp_spaces()->at(i)->space()->used_in_bytes();
 194 }
 195 
 196 
 197 size_t MutableNUMASpace::unsafe_max_tlab_alloc(Thread *thr) const {
 198   // Please see the comments for tlab_capacity().
 199   guarantee(thr != NULL, "No thread");
 200   int lgrp_id = thr->lgrp_id();
 201   if (lgrp_id == -1) {
 202     if (lgrp_spaces()->length() > 0) {
 203       return free_in_bytes() / lgrp_spaces()->length();
 204     } else {
 205       assert(false, "There should be at least one locality group");
 206       return 0;
 207     }
 208   }
 209   int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals);
 210   if (i == -1) {
 211     return 0;
 212   }
 213   return lgrp_spaces()->at(i)->space()->free_in_bytes();
 214 }
 215 
 216 
 217 size_t MutableNUMASpace::capacity_in_words(Thread* thr) const {
 218   guarantee(thr != NULL, "No thread");
 219   int lgrp_id = thr->lgrp_id();
 220   if (lgrp_id == -1) {
 221     if (lgrp_spaces()->length() > 0) {
 222       return capacity_in_words() / lgrp_spaces()->length();
 223     } else {
 224       assert(false, "There should be at least one locality group");
 225       return 0;
 226     }
 227   }
 228   int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals);
 229   if (i == -1) {
 230     return 0;
 231   }
 232   return lgrp_spaces()->at(i)->space()->capacity_in_words();
 233 }
 234 
 235 // Check if the NUMA topology has changed. Add and remove spaces if needed.
 236 // The update can be forced by setting the force parameter equal to true.
 237 bool MutableNUMASpace::update_layout(bool force) {
 238   // Check if the topology had changed.
 239   bool changed = os::numa_topology_changed();
 240   if (force || changed) {
 241     // Compute lgrp intersection. Add/remove spaces.
 242     int lgrp_limit = (int)os::numa_get_groups_num();
 243     int *lgrp_ids = NEW_C_HEAP_ARRAY(int, lgrp_limit, mtGC);
 244     int lgrp_num = (int)os::numa_get_leaf_groups(lgrp_ids, lgrp_limit);
 245     assert(lgrp_num > 0, "There should be at least one locality group");
 246     // Add new spaces for the new nodes
 247     for (int i = 0; i < lgrp_num; i++) {
 248       bool found = false;
 249       for (int j = 0; j < lgrp_spaces()->length(); j++) {
 250         if (lgrp_spaces()->at(j)->lgrp_id() == lgrp_ids[i]) {
 251           found = true;
 252           break;
 253         }
 254       }
 255       if (!found) {
 256         lgrp_spaces()->append(new LGRPSpace(lgrp_ids[i], alignment()));
 257       }
 258     }
 259 
 260     // Remove spaces for the removed nodes.
 261     for (int i = 0; i < lgrp_spaces()->length();) {
 262       bool found = false;
 263       for (int j = 0; j < lgrp_num; j++) {
 264         if (lgrp_spaces()->at(i)->lgrp_id() == lgrp_ids[j]) {
 265           found = true;
 266           break;
 267         }
 268       }
 269       if (!found) {
 270         delete lgrp_spaces()->at(i);
 271         lgrp_spaces()->remove_at(i);
 272       } else {
 273         i++;
 274       }
 275     }
 276 
 277     FREE_C_HEAP_ARRAY(int, lgrp_ids);
 278 
 279     if (changed) {
 280       for (JavaThread *thread = Threads::first(); thread; thread = thread->next()) {
 281         thread->set_lgrp_id(-1);
 282       }
 283     }
 284     return true;
 285   }
 286   return false;
 287 }
 288 
 289 // Bias region towards the first-touching lgrp. Set the right page sizes.
 290 void MutableNUMASpace::bias_region(MemRegion mr, int lgrp_id) {
 291   HeapWord *start = (HeapWord*)round_to((intptr_t)mr.start(), page_size());
 292   HeapWord *end = (HeapWord*)round_down((intptr_t)mr.end(), page_size());
 293   if (end > start) {
 294     MemRegion aligned_region(start, end);
 295     assert((intptr_t)aligned_region.start()     % page_size() == 0 &&
 296            (intptr_t)aligned_region.byte_size() % page_size() == 0, "Bad alignment");
 297     assert(region().contains(aligned_region), "Sanity");
 298     // First we tell the OS which page size we want in the given range. The underlying
 299     // large page can be broken down if we require small pages.
 300     os::realign_memory((char*)aligned_region.start(), aligned_region.byte_size(), page_size());
 301     // Then we uncommit the pages in the range.
 302     os::free_memory((char*)aligned_region.start(), aligned_region.byte_size(), page_size());
 303     // And make them local/first-touch biased.
 304     os::numa_make_local((char*)aligned_region.start(), aligned_region.byte_size(), lgrp_id);
 305   }
 306 }
 307 
 308 // Free all pages in the region.
 309 void MutableNUMASpace::free_region(MemRegion mr) {
 310   HeapWord *start = (HeapWord*)round_to((intptr_t)mr.start(), page_size());
 311   HeapWord *end = (HeapWord*)round_down((intptr_t)mr.end(), page_size());
 312   if (end > start) {
 313     MemRegion aligned_region(start, end);
 314     assert((intptr_t)aligned_region.start()     % page_size() == 0 &&
 315            (intptr_t)aligned_region.byte_size() % page_size() == 0, "Bad alignment");
 316     assert(region().contains(aligned_region), "Sanity");
 317     os::free_memory((char*)aligned_region.start(), aligned_region.byte_size(), page_size());
 318   }
 319 }
 320 
 321 // Update space layout. Perform adaptation.
 322 void MutableNUMASpace::update() {
 323   if (update_layout(false)) {
 324     // If the topology has changed, make all chunks zero-sized.
 325     // And clear the alloc-rate statistics.
 326     // In future we may want to handle this more gracefully in order
 327     // to avoid the reallocation of the pages as much as possible.
 328     for (int i = 0; i < lgrp_spaces()->length(); i++) {
 329       LGRPSpace *ls = lgrp_spaces()->at(i);
 330       MutableSpace *s = ls->space();
 331       s->set_end(s->bottom());
 332       s->set_top(s->bottom());
 333       ls->clear_alloc_rate();
 334     }
 335     // A NUMA space is never mangled
 336     initialize(region(),
 337                SpaceDecorator::Clear,
 338                SpaceDecorator::DontMangle);
 339   } else {
 340     bool should_initialize = false;
 341     if (!os::numa_has_static_binding()) {
 342       for (int i = 0; i < lgrp_spaces()->length(); i++) {
 343         if (!lgrp_spaces()->at(i)->invalid_region().is_empty()) {
 344           should_initialize = true;
 345           break;
 346         }
 347       }
 348     }
 349 
 350     if (should_initialize ||
 351         (UseAdaptiveNUMAChunkSizing && adaptation_cycles() < samples_count())) {
 352       // A NUMA space is never mangled
 353       initialize(region(),
 354                  SpaceDecorator::Clear,
 355                  SpaceDecorator::DontMangle);
 356     }
 357   }
 358 
 359   if (NUMAStats) {
 360     for (int i = 0; i < lgrp_spaces()->length(); i++) {
 361       lgrp_spaces()->at(i)->accumulate_statistics(page_size());
 362     }
 363   }
 364 
 365   scan_pages(NUMAPageScanRate);
 366 }
 367 
 368 // Scan pages. Free pages that have smaller size or wrong placement.
 369 void MutableNUMASpace::scan_pages(size_t page_count)
 370 {
 371   size_t pages_per_chunk = page_count / lgrp_spaces()->length();
 372   if (pages_per_chunk > 0) {
 373     for (int i = 0; i < lgrp_spaces()->length(); i++) {
 374       LGRPSpace *ls = lgrp_spaces()->at(i);
 375       ls->scan_pages(page_size(), pages_per_chunk);
 376     }
 377   }
 378 }
 379 
 380 // Accumulate statistics about the allocation rate of each lgrp.
 381 void MutableNUMASpace::accumulate_statistics() {
 382   if (UseAdaptiveNUMAChunkSizing) {
 383     for (int i = 0; i < lgrp_spaces()->length(); i++) {
 384       lgrp_spaces()->at(i)->sample();
 385     }
 386     increment_samples_count();
 387   }
 388 
 389   if (NUMAStats) {
 390     for (int i = 0; i < lgrp_spaces()->length(); i++) {
 391       lgrp_spaces()->at(i)->accumulate_statistics(page_size());
 392     }
 393   }
 394 }
 395 
 396 // Get the current size of a chunk.
 397 // This function computes the size of the chunk based on the
 398 // difference between chunk ends. This allows it to work correctly in
 399 // case the whole space is resized and during the process of adaptive
 400 // chunk resizing.
 401 size_t MutableNUMASpace::current_chunk_size(int i) {
 402   HeapWord *cur_end, *prev_end;
 403   if (i == 0) {
 404     prev_end = bottom();
 405   } else {
 406     prev_end = lgrp_spaces()->at(i - 1)->space()->end();
 407   }
 408   if (i == lgrp_spaces()->length() - 1) {
 409     cur_end = end();
 410   } else {
 411     cur_end = lgrp_spaces()->at(i)->space()->end();
 412   }
 413   if (cur_end > prev_end) {
 414     return pointer_delta(cur_end, prev_end, sizeof(char));
 415   }
 416   return 0;
 417 }
 418 
 419 // Return the default chunk size by equally diving the space.
 420 // page_size() aligned.
 421 size_t MutableNUMASpace::default_chunk_size() {
 422   return base_space_size() / lgrp_spaces()->length() * page_size();
 423 }
 424 
 425 // Produce a new chunk size. page_size() aligned.
 426 // This function is expected to be called on sequence of i's from 0 to
 427 // lgrp_spaces()->length().
 428 size_t MutableNUMASpace::adaptive_chunk_size(int i, size_t limit) {
 429   size_t pages_available = base_space_size();
 430   for (int j = 0; j < i; j++) {
 431     pages_available -= round_down(current_chunk_size(j), page_size()) / page_size();
 432   }
 433   pages_available -= lgrp_spaces()->length() - i - 1;
 434   assert(pages_available > 0, "No pages left");
 435   float alloc_rate = 0;
 436   for (int j = i; j < lgrp_spaces()->length(); j++) {
 437     alloc_rate += lgrp_spaces()->at(j)->alloc_rate()->average();
 438   }
 439   size_t chunk_size = 0;
 440   if (alloc_rate > 0) {
 441     LGRPSpace *ls = lgrp_spaces()->at(i);
 442     chunk_size = (size_t)(ls->alloc_rate()->average() / alloc_rate * pages_available) * page_size();
 443   }
 444   chunk_size = MAX2(chunk_size, page_size());
 445 
 446   if (limit > 0) {
 447     limit = round_down(limit, page_size());
 448     if (chunk_size > current_chunk_size(i)) {
 449       size_t upper_bound = pages_available * page_size();
 450       if (upper_bound > limit &&
 451           current_chunk_size(i) < upper_bound - limit) {
 452         // The resulting upper bound should not exceed the available
 453         // amount of memory (pages_available * page_size()).
 454         upper_bound = current_chunk_size(i) + limit;
 455       }
 456       chunk_size = MIN2(chunk_size, upper_bound);
 457     } else {
 458       size_t lower_bound = page_size();
 459       if (current_chunk_size(i) > limit) { // lower_bound shouldn't underflow.
 460         lower_bound = current_chunk_size(i) - limit;
 461       }
 462       chunk_size = MAX2(chunk_size, lower_bound);
 463     }
 464   }
 465   assert(chunk_size <= pages_available * page_size(), "Chunk size out of range");
 466   return chunk_size;
 467 }
 468 
 469 
 470 // Return the bottom_region and the top_region. Align them to page_size() boundary.
 471 // |------------------new_region---------------------------------|
 472 // |----bottom_region--|---intersection---|------top_region------|
 473 void MutableNUMASpace::select_tails(MemRegion new_region, MemRegion intersection,
 474                                     MemRegion* bottom_region, MemRegion *top_region) {
 475   // Is there bottom?
 476   if (new_region.start() < intersection.start()) { // Yes
 477     // Try to coalesce small pages into a large one.
 478     if (UseLargePages && page_size() >= alignment()) {
 479       HeapWord* p = (HeapWord*)round_to((intptr_t) intersection.start(), alignment());
 480       if (new_region.contains(p)
 481           && pointer_delta(p, new_region.start(), sizeof(char)) >= alignment()) {
 482         if (intersection.contains(p)) {
 483           intersection = MemRegion(p, intersection.end());
 484         } else {
 485           intersection = MemRegion(p, p);
 486         }
 487       }
 488     }
 489     *bottom_region = MemRegion(new_region.start(), intersection.start());
 490   } else {
 491     *bottom_region = MemRegion();
 492   }
 493 
 494   // Is there top?
 495   if (intersection.end() < new_region.end()) { // Yes
 496     // Try to coalesce small pages into a large one.
 497     if (UseLargePages && page_size() >= alignment()) {
 498       HeapWord* p = (HeapWord*)round_down((intptr_t) intersection.end(), alignment());
 499       if (new_region.contains(p)
 500           && pointer_delta(new_region.end(), p, sizeof(char)) >= alignment()) {
 501         if (intersection.contains(p)) {
 502           intersection = MemRegion(intersection.start(), p);
 503         } else {
 504           intersection = MemRegion(p, p);
 505         }
 506       }
 507     }
 508     *top_region = MemRegion(intersection.end(), new_region.end());
 509   } else {
 510     *top_region = MemRegion();
 511   }
 512 }
 513 
 514 // Try to merge the invalid region with the bottom or top region by decreasing
 515 // the intersection area. Return the invalid_region aligned to the page_size()
 516 // boundary if it's inside the intersection. Return non-empty invalid_region
 517 // if it lies inside the intersection (also page-aligned).
 518 // |------------------new_region---------------------------------|
 519 // |----------------|-------invalid---|--------------------------|
 520 // |----bottom_region--|---intersection---|------top_region------|
 521 void MutableNUMASpace::merge_regions(MemRegion new_region, MemRegion* intersection,
 522                                      MemRegion *invalid_region) {
 523   if (intersection->start() >= invalid_region->start() && intersection->contains(invalid_region->end())) {
 524     *intersection = MemRegion(invalid_region->end(), intersection->end());
 525     *invalid_region = MemRegion();
 526   } else
 527     if (intersection->end() <= invalid_region->end() && intersection->contains(invalid_region->start())) {
 528       *intersection = MemRegion(intersection->start(), invalid_region->start());
 529       *invalid_region = MemRegion();
 530     } else
 531       if (intersection->equals(*invalid_region) || invalid_region->contains(*intersection)) {
 532         *intersection = MemRegion(new_region.start(), new_region.start());
 533         *invalid_region = MemRegion();
 534       } else
 535         if (intersection->contains(invalid_region)) {
 536             // That's the only case we have to make an additional bias_region() call.
 537             HeapWord* start = invalid_region->start();
 538             HeapWord* end = invalid_region->end();
 539             if (UseLargePages && page_size() >= alignment()) {
 540               HeapWord *p = (HeapWord*)round_down((intptr_t) start, alignment());
 541               if (new_region.contains(p)) {
 542                 start = p;
 543               }
 544               p = (HeapWord*)round_to((intptr_t) end, alignment());
 545               if (new_region.contains(end)) {
 546                 end = p;
 547               }
 548             }
 549             if (intersection->start() > start) {
 550               *intersection = MemRegion(start, intersection->end());
 551             }
 552             if (intersection->end() < end) {
 553               *intersection = MemRegion(intersection->start(), end);
 554             }
 555             *invalid_region = MemRegion(start, end);
 556         }
 557 }
 558 
 559 void MutableNUMASpace::initialize(MemRegion mr,
 560                                   bool clear_space,
 561                                   bool mangle_space,
 562                                   bool setup_pages) {
 563   assert(clear_space, "Reallocation will destroy data!");
 564   assert(lgrp_spaces()->length() > 0, "There should be at least one space");
 565 
 566   MemRegion old_region = region(), new_region;
 567   set_bottom(mr.start());
 568   set_end(mr.end());
 569   // Must always clear the space
 570   clear(SpaceDecorator::DontMangle);
 571 
 572   // Compute chunk sizes
 573   size_t prev_page_size = page_size();
 574   set_page_size(UseLargePages ? alignment() : os::vm_page_size());
 575   HeapWord* rounded_bottom = (HeapWord*)round_to((intptr_t) bottom(), page_size());
 576   HeapWord* rounded_end = (HeapWord*)round_down((intptr_t) end(), page_size());
 577   size_t base_space_size_pages = pointer_delta(rounded_end, rounded_bottom, sizeof(char)) / page_size();
 578 
 579   // Try small pages if the chunk size is too small
 580   if (base_space_size_pages / lgrp_spaces()->length() == 0
 581       && page_size() > (size_t)os::vm_page_size()) {
 582 #ifdef LINUX
 583     // Changing the page size below can lead to freeing of memory. When using large pages
 584     // and the memory has been both reserved and committed, Linux does not support
 585     // freeing parts of it. So we fail initialization.
 586     if (UseLargePages && !os::can_commit_large_page_memory()) {
 587       vm_exit_during_initialization("Failed initializing NUMA with large pages. Too small heap size");
 588     }
 589 #endif // LINUX
 590     set_page_size(os::vm_page_size());
 591     rounded_bottom = (HeapWord*)round_to((intptr_t) bottom(), page_size());
 592     rounded_end = (HeapWord*)round_down((intptr_t) end(), page_size());
 593     base_space_size_pages = pointer_delta(rounded_end, rounded_bottom, sizeof(char)) / page_size();
 594   }
 595   guarantee(base_space_size_pages / lgrp_spaces()->length() > 0, "Space too small");
 596   set_base_space_size(base_space_size_pages);
 597 
 598   // Handle space resize
 599   MemRegion top_region, bottom_region;
 600   if (!old_region.equals(region())) {
 601     new_region = MemRegion(rounded_bottom, rounded_end);
 602     MemRegion intersection = new_region.intersection(old_region);
 603     if (intersection.start() == NULL ||
 604         intersection.end() == NULL   ||
 605         prev_page_size > page_size()) { // If the page size got smaller we have to change
 606                                         // the page size preference for the whole space.
 607       intersection = MemRegion(new_region.start(), new_region.start());
 608     }
 609     select_tails(new_region, intersection, &bottom_region, &top_region);
 610     bias_region(bottom_region, lgrp_spaces()->at(0)->lgrp_id());
 611     bias_region(top_region, lgrp_spaces()->at(lgrp_spaces()->length() - 1)->lgrp_id());
 612   }
 613 
 614   // Check if the space layout has changed significantly?
 615   // This happens when the space has been resized so that either head or tail
 616   // chunk became less than a page.
 617   bool layout_valid = UseAdaptiveNUMAChunkSizing          &&
 618                       current_chunk_size(0) > page_size() &&
 619                       current_chunk_size(lgrp_spaces()->length() - 1) > page_size();
 620 
 621 
 622   for (int i = 0; i < lgrp_spaces()->length(); i++) {
 623     LGRPSpace *ls = lgrp_spaces()->at(i);
 624     MutableSpace *s = ls->space();
 625     old_region = s->region();
 626 
 627     size_t chunk_byte_size = 0, old_chunk_byte_size = 0;
 628     if (i < lgrp_spaces()->length() - 1) {
 629       if (!UseAdaptiveNUMAChunkSizing                                ||
 630           (UseAdaptiveNUMAChunkSizing && NUMAChunkResizeWeight == 0) ||
 631            samples_count() < AdaptiveSizePolicyReadyThreshold) {
 632         // No adaptation. Divide the space equally.
 633         chunk_byte_size = default_chunk_size();
 634       } else
 635         if (!layout_valid || NUMASpaceResizeRate == 0) {
 636           // Fast adaptation. If no space resize rate is set, resize
 637           // the chunks instantly.
 638           chunk_byte_size = adaptive_chunk_size(i, 0);
 639         } else {
 640           // Slow adaptation. Resize the chunks moving no more than
 641           // NUMASpaceResizeRate bytes per collection.
 642           size_t limit = NUMASpaceResizeRate /
 643                          (lgrp_spaces()->length() * (lgrp_spaces()->length() + 1) / 2);
 644           chunk_byte_size = adaptive_chunk_size(i, MAX2(limit * (i + 1), page_size()));
 645         }
 646 
 647       assert(chunk_byte_size >= page_size(), "Chunk size too small");
 648       assert(chunk_byte_size <= capacity_in_bytes(), "Sanity check");
 649     }
 650 
 651     if (i == 0) { // Bottom chunk
 652       if (i != lgrp_spaces()->length() - 1) {
 653         new_region = MemRegion(bottom(), rounded_bottom + (chunk_byte_size >> LogHeapWordSize));
 654       } else {
 655         new_region = MemRegion(bottom(), end());
 656       }
 657     } else
 658       if (i < lgrp_spaces()->length() - 1) { // Middle chunks
 659         MutableSpace *ps = lgrp_spaces()->at(i - 1)->space();
 660         new_region = MemRegion(ps->end(),
 661                                ps->end() + (chunk_byte_size >> LogHeapWordSize));
 662       } else { // Top chunk
 663         MutableSpace *ps = lgrp_spaces()->at(i - 1)->space();
 664         new_region = MemRegion(ps->end(), end());
 665       }
 666     guarantee(region().contains(new_region), "Region invariant");
 667 
 668 
 669     // The general case:
 670     // |---------------------|--invalid---|--------------------------|
 671     // |------------------new_region---------------------------------|
 672     // |----bottom_region--|---intersection---|------top_region------|
 673     //                     |----old_region----|
 674     // The intersection part has all pages in place we don't need to migrate them.
 675     // Pages for the top and bottom part should be freed and then reallocated.
 676 
 677     MemRegion intersection = old_region.intersection(new_region);
 678 
 679     if (intersection.start() == NULL || intersection.end() == NULL) {
 680       intersection = MemRegion(new_region.start(), new_region.start());
 681     }
 682 
 683     if (!os::numa_has_static_binding()) {
 684       MemRegion invalid_region = ls->invalid_region().intersection(new_region);
 685       // Invalid region is a range of memory that could've possibly
 686       // been allocated on the other node. That's relevant only on Solaris where
 687       // there is no static memory binding.
 688       if (!invalid_region.is_empty()) {
 689         merge_regions(new_region, &intersection, &invalid_region);
 690         free_region(invalid_region);
 691         ls->set_invalid_region(MemRegion());
 692       }
 693     }
 694 
 695     select_tails(new_region, intersection, &bottom_region, &top_region);
 696 
 697     if (!os::numa_has_static_binding()) {
 698       // If that's a system with the first-touch policy then it's enough
 699       // to free the pages.
 700       free_region(bottom_region);
 701       free_region(top_region);
 702     } else {
 703       // In a system with static binding we have to change the bias whenever
 704       // we reshape the heap.
 705       bias_region(bottom_region, ls->lgrp_id());
 706       bias_region(top_region, ls->lgrp_id());
 707     }
 708 
 709     // Clear space (set top = bottom) but never mangle.
 710     s->initialize(new_region, SpaceDecorator::Clear, SpaceDecorator::DontMangle, MutableSpace::DontSetupPages);
 711 
 712     set_adaptation_cycles(samples_count());
 713   }
 714 }
 715 
 716 // Set the top of the whole space.
 717 // Mark the the holes in chunks below the top() as invalid.
 718 void MutableNUMASpace::set_top(HeapWord* value) {
 719   bool found_top = false;
 720   for (int i = 0; i < lgrp_spaces()->length();) {
 721     LGRPSpace *ls = lgrp_spaces()->at(i);
 722     MutableSpace *s = ls->space();
 723     HeapWord *top = MAX2((HeapWord*)round_down((intptr_t)s->top(), page_size()), s->bottom());
 724 
 725     if (s->contains(value)) {
 726       // Check if setting the chunk's top to a given value would create a hole less than
 727       // a minimal object; assuming that's not the last chunk in which case we don't care.
 728       if (i < lgrp_spaces()->length() - 1) {
 729         size_t remainder = pointer_delta(s->end(), value);
 730         const size_t min_fill_size = CollectedHeap::min_fill_size();
 731         if (remainder < min_fill_size && remainder > 0) {
 732           // Add a minimum size filler object; it will cross the chunk boundary.
 733           CollectedHeap::fill_with_object(value, min_fill_size);
 734           value += min_fill_size;
 735           assert(!s->contains(value), "Should be in the next chunk");
 736           // Restart the loop from the same chunk, since the value has moved
 737           // to the next one.
 738           continue;
 739         }
 740       }
 741 
 742       if (!os::numa_has_static_binding() && top < value && top < s->end()) {
 743         ls->add_invalid_region(MemRegion(top, value));
 744       }
 745       s->set_top(value);
 746       found_top = true;
 747     } else {
 748         if (found_top) {
 749             s->set_top(s->bottom());
 750         } else {
 751           if (!os::numa_has_static_binding() && top < s->end()) {
 752             ls->add_invalid_region(MemRegion(top, s->end()));
 753           }
 754           s->set_top(s->end());
 755         }
 756     }
 757     i++;
 758   }
 759   MutableSpace::set_top(value);
 760 }
 761 
 762 void MutableNUMASpace::clear(bool mangle_space) {
 763   MutableSpace::set_top(bottom());
 764   for (int i = 0; i < lgrp_spaces()->length(); i++) {
 765     // Never mangle NUMA spaces because the mangling will
 766     // bind the memory to a possibly unwanted lgroup.
 767     lgrp_spaces()->at(i)->space()->clear(SpaceDecorator::DontMangle);
 768   }
 769 }
 770 
 771 /*
 772    Linux supports static memory binding, therefore the most part of the
 773    logic dealing with the possible invalid page allocation is effectively
 774    disabled. Besides there is no notion of the home node in Linux. A
 775    thread is allowed to migrate freely. Although the scheduler is rather
 776    reluctant to move threads between the nodes. We check for the current
 777    node every allocation. And with a high probability a thread stays on
 778    the same node for some time allowing local access to recently allocated
 779    objects.
 780  */
 781 
 782 HeapWord* MutableNUMASpace::allocate(size_t size) {
 783   Thread* thr = Thread::current();
 784   int lgrp_id = thr->lgrp_id();
 785   if (lgrp_id == -1 || !os::numa_has_group_homing()) {
 786     lgrp_id = os::numa_get_group_id();
 787     thr->set_lgrp_id(lgrp_id);
 788   }
 789 
 790   int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals);
 791 
 792   // It is possible that a new CPU has been hotplugged and
 793   // we haven't reshaped the space accordingly.
 794   if (i == -1) {
 795     i = os::random() % lgrp_spaces()->length();
 796   }
 797 
 798   LGRPSpace* ls = lgrp_spaces()->at(i);
 799   MutableSpace *s = ls->space();
 800   HeapWord *p = s->allocate(size);
 801 
 802   if (p != NULL) {
 803     size_t remainder = s->free_in_words();
 804     if (remainder < CollectedHeap::min_fill_size() && remainder > 0) {
 805       s->set_top(s->top() - size);
 806       p = NULL;
 807     }
 808   }
 809   if (p != NULL) {
 810     if (top() < s->top()) { // Keep _top updated.
 811       MutableSpace::set_top(s->top());
 812     }
 813   }
 814   // Make the page allocation happen here if there is no static binding..
 815   if (p != NULL && !os::numa_has_static_binding()) {
 816     for (HeapWord *i = p; i < p + size; i += os::vm_page_size() >> LogHeapWordSize) {
 817       *(int*)i = 0;
 818     }
 819   }
 820   if (p == NULL) {
 821     ls->set_allocation_failed();
 822   }
 823   return p;
 824 }
 825 
 826 // This version is lock-free.
 827 HeapWord* MutableNUMASpace::cas_allocate(size_t size) {
 828   Thread* thr = Thread::current();
 829   int lgrp_id = thr->lgrp_id();
 830   if (lgrp_id == -1 || !os::numa_has_group_homing()) {
 831     lgrp_id = os::numa_get_group_id();
 832     thr->set_lgrp_id(lgrp_id);
 833   }
 834 
 835   int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals);
 836   // It is possible that a new CPU has been hotplugged and
 837   // we haven't reshaped the space accordingly.
 838   if (i == -1) {
 839     i = os::random() % lgrp_spaces()->length();
 840   }
 841   LGRPSpace *ls = lgrp_spaces()->at(i);
 842   MutableSpace *s = ls->space();
 843   HeapWord *p = s->cas_allocate(size);
 844   if (p != NULL) {
 845     size_t remainder = pointer_delta(s->end(), p + size);
 846     if (remainder < CollectedHeap::min_fill_size() && remainder > 0) {
 847       if (s->cas_deallocate(p, size)) {
 848         // We were the last to allocate and created a fragment less than
 849         // a minimal object.
 850         p = NULL;
 851       } else {
 852         guarantee(false, "Deallocation should always succeed");
 853       }
 854     }
 855   }
 856   if (p != NULL) {
 857     HeapWord* cur_top, *cur_chunk_top = p + size;
 858     while ((cur_top = top()) < cur_chunk_top) { // Keep _top updated.
 859       if (Atomic::cmpxchg_ptr(cur_chunk_top, top_addr(), cur_top) == cur_top) {
 860         break;
 861       }
 862     }
 863   }
 864 
 865   // Make the page allocation happen here if there is no static binding.
 866   if (p != NULL && !os::numa_has_static_binding() ) {
 867     for (HeapWord *i = p; i < p + size; i += os::vm_page_size() >> LogHeapWordSize) {
 868       *(int*)i = 0;
 869     }
 870   }
 871   if (p == NULL) {
 872     ls->set_allocation_failed();
 873   }
 874   return p;
 875 }
 876 
 877 void MutableNUMASpace::print_short_on(outputStream* st) const {
 878   MutableSpace::print_short_on(st);
 879   st->print(" (");
 880   for (int i = 0; i < lgrp_spaces()->length(); i++) {
 881     st->print("lgrp %d: ", lgrp_spaces()->at(i)->lgrp_id());
 882     lgrp_spaces()->at(i)->space()->print_short_on(st);
 883     if (i < lgrp_spaces()->length() - 1) {
 884       st->print(", ");
 885     }
 886   }
 887   st->print(")");
 888 }
 889 
 890 void MutableNUMASpace::print_on(outputStream* st) const {
 891   MutableSpace::print_on(st);
 892   for (int i = 0; i < lgrp_spaces()->length(); i++) {
 893     LGRPSpace *ls = lgrp_spaces()->at(i);
 894     st->print("    lgrp %d", ls->lgrp_id());
 895     ls->space()->print_on(st);
 896     if (NUMAStats) {
 897       for (int i = 0; i < lgrp_spaces()->length(); i++) {
 898         lgrp_spaces()->at(i)->accumulate_statistics(page_size());
 899       }
 900       st->print("    local/remote/unbiased/uncommitted: " SIZE_FORMAT "K/"
 901                 SIZE_FORMAT "K/" SIZE_FORMAT "K/" SIZE_FORMAT
 902                 "K, large/small pages: " SIZE_FORMAT "/" SIZE_FORMAT "\n",
 903                 ls->space_stats()->_local_space / K,
 904                 ls->space_stats()->_remote_space / K,
 905                 ls->space_stats()->_unbiased_space / K,
 906                 ls->space_stats()->_uncommited_space / K,
 907                 ls->space_stats()->_large_pages,
 908                 ls->space_stats()->_small_pages);
 909     }
 910   }
 911 }
 912 
 913 void MutableNUMASpace::verify() {
 914   // This can be called after setting an arbitrary value to the space's top,
 915   // so an object can cross the chunk boundary. We ensure the parsability
 916   // of the space and just walk the objects in linear fashion.
 917   ensure_parsability();
 918   MutableSpace::verify();
 919 }
 920 
 921 // Scan pages and gather stats about page placement and size.
 922 void MutableNUMASpace::LGRPSpace::accumulate_statistics(size_t page_size) {
 923   clear_space_stats();
 924   char *start = (char*)round_to((intptr_t) space()->bottom(), page_size);
 925   char* end = (char*)round_down((intptr_t) space()->end(), page_size);
 926   if (start < end) {
 927     for (char *p = start; p < end;) {
 928       os::page_info info;
 929       if (os::get_page_info(p, &info)) {
 930         if (info.size > 0) {
 931           if (info.size > (size_t)os::vm_page_size()) {
 932             space_stats()->_large_pages++;
 933           } else {
 934             space_stats()->_small_pages++;
 935           }
 936           if (info.lgrp_id == lgrp_id()) {
 937             space_stats()->_local_space += info.size;
 938           } else {
 939             space_stats()->_remote_space += info.size;
 940           }
 941           p += info.size;
 942         } else {
 943           p += os::vm_page_size();
 944           space_stats()->_uncommited_space += os::vm_page_size();
 945         }
 946       } else {
 947         return;
 948       }
 949     }
 950   }
 951   space_stats()->_unbiased_space = pointer_delta(start, space()->bottom(), sizeof(char)) +
 952                                    pointer_delta(space()->end(), end, sizeof(char));
 953 
 954 }
 955 
 956 // Scan page_count pages and verify if they have the right size and right placement.
 957 // If invalid pages are found they are freed in hope that subsequent reallocation
 958 // will be more successful.
 959 void MutableNUMASpace::LGRPSpace::scan_pages(size_t page_size, size_t page_count)
 960 {
 961   char* range_start = (char*)round_to((intptr_t) space()->bottom(), page_size);
 962   char* range_end = (char*)round_down((intptr_t) space()->end(), page_size);
 963 
 964   if (range_start > last_page_scanned() || last_page_scanned() >= range_end) {
 965     set_last_page_scanned(range_start);
 966   }
 967 
 968   char *scan_start = last_page_scanned();
 969   char* scan_end = MIN2(scan_start + page_size * page_count, range_end);
 970 
 971   os::page_info page_expected, page_found;
 972   page_expected.size = page_size;
 973   page_expected.lgrp_id = lgrp_id();
 974 
 975   char *s = scan_start;
 976   while (s < scan_end) {
 977     char *e = os::scan_pages(s, (char*)scan_end, &page_expected, &page_found);
 978     if (e == NULL) {
 979       break;
 980     }
 981     if (e != scan_end) {
 982       assert(e < scan_end, "e: " PTR_FORMAT " scan_end: " PTR_FORMAT, p2i(e), p2i(scan_end));
 983 
 984       if ((page_expected.size != page_size || page_expected.lgrp_id != lgrp_id())
 985           && page_expected.size != 0) {
 986         os::free_memory(s, pointer_delta(e, s, sizeof(char)), page_size);
 987       }
 988       page_expected = page_found;
 989     }
 990     s = e;
 991   }
 992 
 993   set_last_page_scanned(scan_end);
 994 }
--- EOF ---