rev 11780 : [mq]: webrev.0
rev 11781 : [mq]: webrev.1

   1 
   2 /*
   3  * Copyright (c) 2006, 2016, Oracle and/or its affiliates. All rights reserved.
   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This code is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU General Public License version 2 only, as
   8  * published by the Free Software Foundation.
   9  *
  10  * This code is distributed in the hope that it will be useful, but WITHOUT
  11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  13  * version 2 for more details (a copy is included in the LICENSE file that
  14  * accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License version
  17  * 2 along with this work; if not, write to the Free Software Foundation,
  18  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  19  *
  20  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  21  * or visit www.oracle.com if you need additional information or have any
  22  * questions.
  23  *
  24  */
  25 
  26 #include "precompiled.hpp"
  27 #include "gc/parallel/mutableNUMASpace.hpp"
  28 #include "gc/shared/collectedHeap.hpp"
  29 #include "gc/shared/spaceDecorator.hpp"
  30 #include "oops/oop.inline.hpp"
  31 #include "runtime/atomic.inline.hpp"
  32 #include "runtime/thread.inline.hpp"
  33 
  34 MutableNUMASpace::MutableNUMASpace(size_t alignment) : MutableSpace(alignment) {
  35   _lgrp_spaces = new (ResourceObj::C_HEAP, mtGC) GrowableArray<LGRPSpace*>(0, true);
  36   _page_size = os::vm_page_size();
  37   _adaptation_cycles = 0;
  38   _samples_count = 0;
  39   update_layout(true);
  40 }
  41 
  42 MutableNUMASpace::~MutableNUMASpace() {
  43   for (int i = 0; i < lgrp_spaces()->length(); i++) {
  44     delete lgrp_spaces()->at(i);
  45   }
  46   delete lgrp_spaces();
  47 }
  48 
  49 #ifndef PRODUCT
  50 void MutableNUMASpace::mangle_unused_area() {
  51   // This method should do nothing.
  52   // It can be called on a numa space during a full compaction.
  53 }
  54 void MutableNUMASpace::mangle_unused_area_complete() {
  55   // This method should do nothing.
  56   // It can be called on a numa space during a full compaction.
  57 }
  58 void MutableNUMASpace::mangle_region(MemRegion mr) {
  59   // This method should do nothing because numa spaces are not mangled.
  60 }
  61 void MutableNUMASpace::set_top_for_allocations(HeapWord* v) {
  62   assert(false, "Do not mangle MutableNUMASpace's");
  63 }
  64 void MutableNUMASpace::set_top_for_allocations() {
  65   // This method should do nothing.
  66 }
  67 void MutableNUMASpace::check_mangled_unused_area(HeapWord* limit) {
  68   // This method should do nothing.
  69 }
  70 void MutableNUMASpace::check_mangled_unused_area_complete() {
  71   // This method should do nothing.
  72 }
  73 #endif  // NOT_PRODUCT
  74 
  75 // There may be unallocated holes in the middle chunks
  76 // that should be filled with dead objects to ensure parsability.
  77 void MutableNUMASpace::ensure_parsability() {
  78   for (int i = 0; i < lgrp_spaces()->length(); i++) {
  79     LGRPSpace *ls = lgrp_spaces()->at(i);
  80     MutableSpace *s = ls->space();
  81     if (s->top() < top()) { // For all spaces preceding the one containing top()
  82       if (s->free_in_words() > 0) {
  83         intptr_t cur_top = (intptr_t)s->top();
  84         size_t words_left_to_fill = pointer_delta(s->end(), s->top());;
  85         while (words_left_to_fill > 0) {
  86           size_t words_to_fill = MIN2(words_left_to_fill, CollectedHeap::filler_array_max_size());
  87           assert(words_to_fill >= CollectedHeap::min_fill_size(),
  88                  "Remaining size (" SIZE_FORMAT ") is too small to fill (based on " SIZE_FORMAT " and " SIZE_FORMAT ")",
  89                  words_to_fill, words_left_to_fill, CollectedHeap::filler_array_max_size());
  90           CollectedHeap::fill_with_object((HeapWord*)cur_top, words_to_fill);
  91           if (!os::numa_has_static_binding()) {
  92             size_t touched_words = words_to_fill;
  93 #ifndef ASSERT
  94             if (!ZapUnusedHeapArea) {
  95               touched_words = MIN2((size_t)align_object_size(typeArrayOopDesc::header_size(T_INT)),
  96                 touched_words);
  97             }
  98 #endif
  99             MemRegion invalid;
 100             HeapWord *crossing_start = (HeapWord*)round_to(cur_top, os::vm_page_size());
 101             HeapWord *crossing_end = (HeapWord*)round_to(cur_top + touched_words, os::vm_page_size());
 102             if (crossing_start != crossing_end) {
 103               // If object header crossed a small page boundary we mark the area
 104               // as invalid rounding it to a page_size().
 105               HeapWord *start = MAX2((HeapWord*)round_down(cur_top, page_size()), s->bottom());
 106               HeapWord *end = MIN2((HeapWord*)round_to(cur_top + touched_words, page_size()), s->end());
 107               invalid = MemRegion(start, end);
 108             }
 109 
 110             ls->add_invalid_region(invalid);
 111           }
 112           cur_top = cur_top + (words_to_fill * HeapWordSize);
 113           words_left_to_fill -= words_to_fill;
 114         }
 115       }
 116     } else {
 117       if (!os::numa_has_static_binding()) {
 118 #ifdef ASSERT
 119         MemRegion invalid(s->top(), s->end());
 120         ls->add_invalid_region(invalid);
 121 #else
 122         if (ZapUnusedHeapArea) {
 123           MemRegion invalid(s->top(), s->end());
 124           ls->add_invalid_region(invalid);
 125         } else {
 126           return;
 127         }
 128 #endif
 129       } else {
 130           return;
 131       }
 132     }
 133   }
 134 }
 135 
 136 size_t MutableNUMASpace::used_in_words() const {
 137   size_t s = 0;
 138   for (int i = 0; i < lgrp_spaces()->length(); i++) {
 139     s += lgrp_spaces()->at(i)->space()->used_in_words();
 140   }
 141   return s;
 142 }
 143 
 144 size_t MutableNUMASpace::free_in_words() const {
 145   size_t s = 0;
 146   for (int i = 0; i < lgrp_spaces()->length(); i++) {
 147     s += lgrp_spaces()->at(i)->space()->free_in_words();
 148   }
 149   return s;
 150 }
 151 
 152 
 153 size_t MutableNUMASpace::tlab_capacity(Thread *thr) const {
 154   guarantee(thr != NULL, "No thread");
 155   int lgrp_id = thr->lgrp_id();
 156   if (lgrp_id == -1) {
 157     // This case can occur after the topology of the system has
 158     // changed. Thread can change their location, the new home
 159     // group will be determined during the first allocation
 160     // attempt. For now we can safely assume that all spaces
 161     // have equal size because the whole space will be reinitialized.
 162     if (lgrp_spaces()->length() > 0) {
 163       return capacity_in_bytes() / lgrp_spaces()->length();
 164     } else {
 165       assert(false, "There should be at least one locality group");
 166       return 0;
 167     }
 168   }
 169   // That's the normal case, where we know the locality group of the thread.
 170   int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals);
 171   if (i == -1) {
 172     return 0;
 173   }
 174   return lgrp_spaces()->at(i)->space()->capacity_in_bytes();
 175 }
 176 
 177 size_t MutableNUMASpace::tlab_used(Thread *thr) const {
 178   // Please see the comments for tlab_capacity().
 179   guarantee(thr != NULL, "No thread");
 180   int lgrp_id = thr->lgrp_id();
 181   if (lgrp_id == -1) {
 182     if (lgrp_spaces()->length() > 0) {
 183       return (used_in_bytes()) / lgrp_spaces()->length();
 184     } else {
 185       assert(false, "There should be at least one locality group");
 186       return 0;
 187     }
 188   }
 189   int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals);
 190   if (i == -1) {
 191     return 0;
 192   }
 193   return lgrp_spaces()->at(i)->space()->used_in_bytes();
 194 }
 195 
 196 
 197 size_t MutableNUMASpace::unsafe_max_tlab_alloc(Thread *thr) const {
 198   // Please see the comments for tlab_capacity().
 199   guarantee(thr != NULL, "No thread");
 200   int lgrp_id = thr->lgrp_id();
 201   if (lgrp_id == -1) {
 202     if (lgrp_spaces()->length() > 0) {
 203       return free_in_bytes() / lgrp_spaces()->length();
 204     } else {
 205       assert(false, "There should be at least one locality group");
 206       return 0;
 207     }
 208   }
 209   int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals);
 210   if (i == -1) {
 211     return 0;
 212   }
 213   return lgrp_spaces()->at(i)->space()->free_in_bytes();
 214 }
 215 
 216 
 217 size_t MutableNUMASpace::capacity_in_words(Thread* thr) const {
 218   guarantee(thr != NULL, "No thread");
 219   int lgrp_id = thr->lgrp_id();
 220   if (lgrp_id == -1) {
 221     if (lgrp_spaces()->length() > 0) {
 222       return capacity_in_words() / lgrp_spaces()->length();
 223     } else {
 224       assert(false, "There should be at least one locality group");
 225       return 0;
 226     }
 227   }
 228   int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals);
 229   if (i == -1) {
 230     return 0;
 231   }
 232   return lgrp_spaces()->at(i)->space()->capacity_in_words();
 233 }
 234 
 235 // Check if the NUMA topology has changed. Add and remove spaces if needed.
 236 // The update can be forced by setting the force parameter equal to true.
 237 bool MutableNUMASpace::update_layout(bool force) {
 238   // Check if the topology had changed.
 239   bool changed = os::numa_topology_changed();
 240   if (force || changed) {
 241     // Compute lgrp intersection. Add/remove spaces.
 242     int lgrp_limit = (int)os::numa_get_groups_num();
 243     int *lgrp_ids = NEW_C_HEAP_ARRAY(int, lgrp_limit, mtGC);
 244     int lgrp_num = (int)os::numa_get_leaf_groups(lgrp_ids, lgrp_limit);
 245     assert(lgrp_num > 0, "There should be at least one locality group");
 246     // Add new spaces for the new nodes
 247     for (int i = 0; i < lgrp_num; i++) {
 248       bool found = false;
 249       for (int j = 0; j < lgrp_spaces()->length(); j++) {
 250         if (lgrp_spaces()->at(j)->lgrp_id() == lgrp_ids[i]) {
 251           found = true;
 252           break;
 253         }
 254       }
 255       if (!found) {
 256         lgrp_spaces()->append(new LGRPSpace(lgrp_ids[i], alignment()));
 257       }
 258     }
 259 
 260     // Remove spaces for the removed nodes.
 261     for (int i = 0; i < lgrp_spaces()->length();) {
 262       bool found = false;
 263       for (int j = 0; j < lgrp_num; j++) {
 264         if (lgrp_spaces()->at(i)->lgrp_id() == lgrp_ids[j]) {
 265           found = true;
 266           break;
 267         }
 268       }
 269       if (!found) {
 270         delete lgrp_spaces()->at(i);
 271         lgrp_spaces()->remove_at(i);
 272       } else {
 273         i++;
 274       }
 275     }
 276 
 277     FREE_C_HEAP_ARRAY(int, lgrp_ids);
 278 
 279     if (changed) {
 280       for (JavaThread *thread = Threads::first(); thread; thread = thread->next()) {
 281         thread->set_lgrp_id(-1);
 282       }
 283     }
 284     return true;
 285   }
 286   return false;
 287 }
 288 
 289 // Bias region towards the first-touching lgrp. Set the right page sizes.
 290 void MutableNUMASpace::bias_region(MemRegion mr, int lgrp_id) {
 291   HeapWord *start = (HeapWord*)round_to((intptr_t)mr.start(), page_size());
 292   HeapWord *end = (HeapWord*)round_down((intptr_t)mr.end(), page_size());
 293   if (end > start) {
 294     MemRegion aligned_region(start, end);
 295     assert((intptr_t)aligned_region.start()     % page_size() == 0 &&
 296            (intptr_t)aligned_region.byte_size() % page_size() == 0, "Bad alignment");
 297     assert(region().contains(aligned_region), "Sanity");
 298     // First we tell the OS which page size we want in the given range. The underlying
 299     // large page can be broken down if we require small pages.
 300     os::realign_memory((char*)aligned_region.start(), aligned_region.byte_size(), page_size());
 301     // Then we uncommit the pages in the range.
 302     os::free_memory((char*)aligned_region.start(), aligned_region.byte_size(), page_size());
 303     // And make them local/first-touch biased.
 304     os::numa_make_local((char*)aligned_region.start(), aligned_region.byte_size(), lgrp_id);
 305   }
 306 }
 307 
 308 // Free all pages in the region.
 309 void MutableNUMASpace::free_region(MemRegion mr) {
 310   HeapWord *start = (HeapWord*)round_to((intptr_t)mr.start(), page_size());
 311   HeapWord *end = (HeapWord*)round_down((intptr_t)mr.end(), page_size());
 312   if (end > start) {
 313     MemRegion aligned_region(start, end);
 314     assert((intptr_t)aligned_region.start()     % page_size() == 0 &&
 315            (intptr_t)aligned_region.byte_size() % page_size() == 0, "Bad alignment");
 316     assert(region().contains(aligned_region), "Sanity");
 317     os::free_memory((char*)aligned_region.start(), aligned_region.byte_size(), page_size());
 318   }
 319 }
 320 
 321 // Update space layout. Perform adaptation.
 322 void MutableNUMASpace::update() {
 323   if (update_layout(false)) {
 324     // If the topology has changed, make all chunks zero-sized.
 325     // And clear the alloc-rate statistics.
 326     // In future we may want to handle this more gracefully in order
 327     // to avoid the reallocation of the pages as much as possible.
 328     for (int i = 0; i < lgrp_spaces()->length(); i++) {
 329       LGRPSpace *ls = lgrp_spaces()->at(i);
 330       MutableSpace *s = ls->space();
 331       s->set_end(s->bottom());
 332       s->set_top(s->bottom());
 333       ls->clear_alloc_rate();
 334     }
 335     // A NUMA space is never mangled
 336     initialize(region(),
 337                SpaceDecorator::Clear,
 338                SpaceDecorator::DontMangle);
 339   } else {
 340     bool should_initialize = false;
 341     if (!os::numa_has_static_binding()) {
 342       for (int i = 0; i < lgrp_spaces()->length(); i++) {
 343         if (!lgrp_spaces()->at(i)->invalid_region().is_empty()) {
 344           should_initialize = true;
 345           break;
 346         }
 347       }
 348     }
 349 
 350     if (should_initialize ||
 351         (UseAdaptiveNUMAChunkSizing && adaptation_cycles() < samples_count())) {
 352       // A NUMA space is never mangled
 353       initialize(region(),
 354                  SpaceDecorator::Clear,
 355                  SpaceDecorator::DontMangle);
 356     }
 357   }
 358 
 359   if (NUMAStats) {
 360     for (int i = 0; i < lgrp_spaces()->length(); i++) {
 361       lgrp_spaces()->at(i)->accumulate_statistics(page_size());
 362     }
 363   }
 364 
 365   scan_pages(NUMAPageScanRate);
 366 }
 367 
 368 // Scan pages. Free pages that have smaller size or wrong placement.
 369 void MutableNUMASpace::scan_pages(size_t page_count)
 370 {
 371   size_t pages_per_chunk = page_count / lgrp_spaces()->length();
 372   if (pages_per_chunk > 0) {
 373     for (int i = 0; i < lgrp_spaces()->length(); i++) {
 374       LGRPSpace *ls = lgrp_spaces()->at(i);
 375       ls->scan_pages(page_size(), pages_per_chunk);
 376     }
 377   }
 378 }
 379 
 380 // Accumulate statistics about the allocation rate of each lgrp.
 381 void MutableNUMASpace::accumulate_statistics() {
 382   if (UseAdaptiveNUMAChunkSizing) {
 383     for (int i = 0; i < lgrp_spaces()->length(); i++) {
 384       lgrp_spaces()->at(i)->sample();
 385     }
 386     increment_samples_count();
 387   }
 388 
 389   if (NUMAStats) {
 390     for (int i = 0; i < lgrp_spaces()->length(); i++) {
 391       lgrp_spaces()->at(i)->accumulate_statistics(page_size());
 392     }
 393   }
 394 }
 395 
 396 // Get the current size of a chunk.
 397 // This function computes the size of the chunk based on the
 398 // difference between chunk ends. This allows it to work correctly in
 399 // case the whole space is resized and during the process of adaptive
 400 // chunk resizing.
 401 size_t MutableNUMASpace::current_chunk_size(int i) {
 402   HeapWord *cur_end, *prev_end;
 403   if (i == 0) {
 404     prev_end = bottom();
 405   } else {
 406     prev_end = lgrp_spaces()->at(i - 1)->space()->end();
 407   }
 408   if (i == lgrp_spaces()->length() - 1) {
 409     cur_end = end();
 410   } else {
 411     cur_end = lgrp_spaces()->at(i)->space()->end();
 412   }
 413   if (cur_end > prev_end) {
 414     return pointer_delta(cur_end, prev_end, sizeof(char));
 415   }
 416   return 0;
 417 }
 418 
 419 // Return the default chunk size by equally diving the space.
 420 // page_size() aligned.
 421 size_t MutableNUMASpace::default_chunk_size() {
 422   return base_space_size() / lgrp_spaces()->length() * page_size();
 423 }
 424 
 425 // Produce a new chunk size. page_size() aligned.
 426 // This function is expected to be called on sequence of i's from 0 to
 427 // lgrp_spaces()->length().
 428 size_t MutableNUMASpace::adaptive_chunk_size(int i, size_t limit) {
 429   size_t pages_available = base_space_size();
 430   for (int j = 0; j < i; j++) {
 431     pages_available -= round_down(current_chunk_size(j), page_size()) / page_size();
 432   }
 433   pages_available -= lgrp_spaces()->length() - i - 1;
 434   assert(pages_available > 0, "No pages left");
 435   float alloc_rate = 0;
 436   for (int j = i; j < lgrp_spaces()->length(); j++) {
 437     alloc_rate += lgrp_spaces()->at(j)->alloc_rate()->average();
 438   }
 439   size_t chunk_size = 0;
 440   if (alloc_rate > 0) {
 441     LGRPSpace *ls = lgrp_spaces()->at(i);
 442     chunk_size = (size_t)(ls->alloc_rate()->average() / alloc_rate * pages_available) * page_size();
 443   }
 444   chunk_size = MAX2(chunk_size, page_size());
 445 
 446   if (limit > 0) {
 447     limit = round_down(limit, page_size());
 448     if (chunk_size > current_chunk_size(i)) {
 449       size_t upper_bound = pages_available * page_size();
 450       if (upper_bound > limit &&
 451           current_chunk_size(i) < upper_bound - limit) {
 452         // The resulting upper bound should not exceed the available
 453         // amount of memory (pages_available * page_size()).
 454         upper_bound = current_chunk_size(i) + limit;
 455       }
 456       chunk_size = MIN2(chunk_size, upper_bound);
 457     } else {
 458       size_t lower_bound = page_size();
 459       if (current_chunk_size(i) > limit) { // lower_bound shouldn't underflow.
 460         lower_bound = current_chunk_size(i) - limit;
 461       }
 462       chunk_size = MAX2(chunk_size, lower_bound);
 463     }
 464   }
 465   assert(chunk_size <= pages_available * page_size(), "Chunk size out of range");
 466   return chunk_size;
 467 }
 468 
 469 
 470 // Return the bottom_region and the top_region. Align them to page_size() boundary.
 471 // |------------------new_region---------------------------------|
 472 // |----bottom_region--|---intersection---|------top_region------|
 473 void MutableNUMASpace::select_tails(MemRegion new_region, MemRegion intersection,
 474                                     MemRegion* bottom_region, MemRegion *top_region) {
 475   // Is there bottom?
 476   if (new_region.start() < intersection.start()) { // Yes
 477     // Try to coalesce small pages into a large one.
 478     if (UseLargePages && page_size() >= alignment()) {
 479       HeapWord* p = (HeapWord*)round_to((intptr_t) intersection.start(), alignment());
 480       if (new_region.contains(p)
 481           && pointer_delta(p, new_region.start(), sizeof(char)) >= alignment()) {
 482         if (intersection.contains(p)) {
 483           intersection = MemRegion(p, intersection.end());
 484         } else {
 485           intersection = MemRegion(p, p);
 486         }
 487       }
 488     }
 489     *bottom_region = MemRegion(new_region.start(), intersection.start());
 490   } else {
 491     *bottom_region = MemRegion();
 492   }
 493 
 494   // Is there top?
 495   if (intersection.end() < new_region.end()) { // Yes
 496     // Try to coalesce small pages into a large one.
 497     if (UseLargePages && page_size() >= alignment()) {
 498       HeapWord* p = (HeapWord*)round_down((intptr_t) intersection.end(), alignment());
 499       if (new_region.contains(p)
 500           && pointer_delta(new_region.end(), p, sizeof(char)) >= alignment()) {
 501         if (intersection.contains(p)) {
 502           intersection = MemRegion(intersection.start(), p);
 503         } else {
 504           intersection = MemRegion(p, p);
 505         }
 506       }
 507     }
 508     *top_region = MemRegion(intersection.end(), new_region.end());
 509   } else {
 510     *top_region = MemRegion();
 511   }
 512 }
 513 
 514 // Try to merge the invalid region with the bottom or top region by decreasing
 515 // the intersection area. Return the invalid_region aligned to the page_size()
 516 // boundary if it's inside the intersection. Return non-empty invalid_region
 517 // if it lies inside the intersection (also page-aligned).
 518 // |------------------new_region---------------------------------|
 519 // |----------------|-------invalid---|--------------------------|
 520 // |----bottom_region--|---intersection---|------top_region------|
 521 void MutableNUMASpace::merge_regions(MemRegion new_region, MemRegion* intersection,
 522                                      MemRegion *invalid_region) {
 523   if (intersection->start() >= invalid_region->start() && intersection->contains(invalid_region->end())) {
 524     *intersection = MemRegion(invalid_region->end(), intersection->end());
 525     *invalid_region = MemRegion();
 526   } else
 527     if (intersection->end() <= invalid_region->end() && intersection->contains(invalid_region->start())) {
 528       *intersection = MemRegion(intersection->start(), invalid_region->start());
 529       *invalid_region = MemRegion();
 530     } else
 531       if (intersection->equals(*invalid_region) || invalid_region->contains(*intersection)) {
 532         *intersection = MemRegion(new_region.start(), new_region.start());
 533         *invalid_region = MemRegion();
 534       } else
 535         if (intersection->contains(invalid_region)) {
 536             // That's the only case we have to make an additional bias_region() call.
 537             HeapWord* start = invalid_region->start();
 538             HeapWord* end = invalid_region->end();
 539             if (UseLargePages && page_size() >= alignment()) {
 540               HeapWord *p = (HeapWord*)round_down((intptr_t) start, alignment());
 541               if (new_region.contains(p)) {
 542                 start = p;
 543               }
 544               p = (HeapWord*)round_to((intptr_t) end, alignment());
 545               if (new_region.contains(end)) {
 546                 end = p;
 547               }
 548             }
 549             if (intersection->start() > start) {
 550               *intersection = MemRegion(start, intersection->end());
 551             }
 552             if (intersection->end() < end) {
 553               *intersection = MemRegion(intersection->start(), end);
 554             }
 555             *invalid_region = MemRegion(start, end);
 556         }
 557 }
 558 
 559 void MutableNUMASpace::initialize(MemRegion mr,
 560                                   bool clear_space,
 561                                   bool mangle_space,
 562                                   bool setup_pages) {
 563   assert(clear_space, "Reallocation will destroy data!");
 564   assert(lgrp_spaces()->length() > 0, "There should be at least one space");
 565 
 566   MemRegion old_region = region(), new_region;
 567   set_bottom(mr.start());
 568   set_end(mr.end());
 569   // Must always clear the space
 570   clear(SpaceDecorator::DontMangle);
 571 
 572   // Compute chunk sizes
 573   size_t prev_page_size = page_size();
 574   set_page_size(UseLargePages ? alignment() : os::vm_page_size());
 575   HeapWord* rounded_bottom = (HeapWord*)round_to((intptr_t) bottom(), page_size());
 576   HeapWord* rounded_end = (HeapWord*)round_down((intptr_t) end(), page_size());
 577   size_t base_space_size_pages = pointer_delta(rounded_end, rounded_bottom, sizeof(char)) / page_size();
 578 
 579   // Try small pages if the chunk size is too small
 580   if (base_space_size_pages / lgrp_spaces()->length() == 0
 581       && page_size() > (size_t)os::vm_page_size()) {
 582 #ifdef LINUX
 583     // If we are using pin region, we cannot change the page size to default size
 584     // as we could free memory which is not expected for pin region in Linux.

 585     if (UseLargePages && !os::can_commit_large_page_memory()) {
 586       vm_exit_during_initialization("Failed initializing NUMA. Too small heap size");
 587     }
 588 #endif // LINUX
 589     set_page_size(os::vm_page_size());
 590     rounded_bottom = (HeapWord*)round_to((intptr_t) bottom(), page_size());
 591     rounded_end = (HeapWord*)round_down((intptr_t) end(), page_size());
 592     base_space_size_pages = pointer_delta(rounded_end, rounded_bottom, sizeof(char)) / page_size();
 593   }
 594   guarantee(base_space_size_pages / lgrp_spaces()->length() > 0, "Space too small");
 595   set_base_space_size(base_space_size_pages);
 596 
 597   // Handle space resize
 598   MemRegion top_region, bottom_region;
 599   if (!old_region.equals(region())) {
 600     new_region = MemRegion(rounded_bottom, rounded_end);
 601     MemRegion intersection = new_region.intersection(old_region);
 602     if (intersection.start() == NULL ||
 603         intersection.end() == NULL   ||
 604         prev_page_size > page_size()) { // If the page size got smaller we have to change
 605                                         // the page size preference for the whole space.
 606       intersection = MemRegion(new_region.start(), new_region.start());
 607     }
 608     select_tails(new_region, intersection, &bottom_region, &top_region);
 609     bias_region(bottom_region, lgrp_spaces()->at(0)->lgrp_id());
 610     bias_region(top_region, lgrp_spaces()->at(lgrp_spaces()->length() - 1)->lgrp_id());
 611   }
 612 
 613   // Check if the space layout has changed significantly?
 614   // This happens when the space has been resized so that either head or tail
 615   // chunk became less than a page.
 616   bool layout_valid = UseAdaptiveNUMAChunkSizing          &&
 617                       current_chunk_size(0) > page_size() &&
 618                       current_chunk_size(lgrp_spaces()->length() - 1) > page_size();
 619 
 620 
 621   for (int i = 0; i < lgrp_spaces()->length(); i++) {
 622     LGRPSpace *ls = lgrp_spaces()->at(i);
 623     MutableSpace *s = ls->space();
 624     old_region = s->region();
 625 
 626     size_t chunk_byte_size = 0, old_chunk_byte_size = 0;
 627     if (i < lgrp_spaces()->length() - 1) {
 628       if (!UseAdaptiveNUMAChunkSizing                                ||
 629           (UseAdaptiveNUMAChunkSizing && NUMAChunkResizeWeight == 0) ||
 630            samples_count() < AdaptiveSizePolicyReadyThreshold) {
 631         // No adaptation. Divide the space equally.
 632         chunk_byte_size = default_chunk_size();
 633       } else
 634         if (!layout_valid || NUMASpaceResizeRate == 0) {
 635           // Fast adaptation. If no space resize rate is set, resize
 636           // the chunks instantly.
 637           chunk_byte_size = adaptive_chunk_size(i, 0);
 638         } else {
 639           // Slow adaptation. Resize the chunks moving no more than
 640           // NUMASpaceResizeRate bytes per collection.
 641           size_t limit = NUMASpaceResizeRate /
 642                          (lgrp_spaces()->length() * (lgrp_spaces()->length() + 1) / 2);
 643           chunk_byte_size = adaptive_chunk_size(i, MAX2(limit * (i + 1), page_size()));
 644         }
 645 
 646       assert(chunk_byte_size >= page_size(), "Chunk size too small");
 647       assert(chunk_byte_size <= capacity_in_bytes(), "Sanity check");
 648     }
 649 
 650     if (i == 0) { // Bottom chunk
 651       if (i != lgrp_spaces()->length() - 1) {
 652         new_region = MemRegion(bottom(), rounded_bottom + (chunk_byte_size >> LogHeapWordSize));
 653       } else {
 654         new_region = MemRegion(bottom(), end());
 655       }
 656     } else
 657       if (i < lgrp_spaces()->length() - 1) { // Middle chunks
 658         MutableSpace *ps = lgrp_spaces()->at(i - 1)->space();
 659         new_region = MemRegion(ps->end(),
 660                                ps->end() + (chunk_byte_size >> LogHeapWordSize));
 661       } else { // Top chunk
 662         MutableSpace *ps = lgrp_spaces()->at(i - 1)->space();
 663         new_region = MemRegion(ps->end(), end());
 664       }
 665     guarantee(region().contains(new_region), "Region invariant");
 666 
 667 
 668     // The general case:
 669     // |---------------------|--invalid---|--------------------------|
 670     // |------------------new_region---------------------------------|
 671     // |----bottom_region--|---intersection---|------top_region------|
 672     //                     |----old_region----|
 673     // The intersection part has all pages in place we don't need to migrate them.
 674     // Pages for the top and bottom part should be freed and then reallocated.
 675 
 676     MemRegion intersection = old_region.intersection(new_region);
 677 
 678     if (intersection.start() == NULL || intersection.end() == NULL) {
 679       intersection = MemRegion(new_region.start(), new_region.start());
 680     }
 681 
 682     if (!os::numa_has_static_binding()) {
 683       MemRegion invalid_region = ls->invalid_region().intersection(new_region);
 684       // Invalid region is a range of memory that could've possibly
 685       // been allocated on the other node. That's relevant only on Solaris where
 686       // there is no static memory binding.
 687       if (!invalid_region.is_empty()) {
 688         merge_regions(new_region, &intersection, &invalid_region);
 689         free_region(invalid_region);
 690         ls->set_invalid_region(MemRegion());
 691       }
 692     }
 693 
 694     select_tails(new_region, intersection, &bottom_region, &top_region);
 695 
 696     if (!os::numa_has_static_binding()) {
 697       // If that's a system with the first-touch policy then it's enough
 698       // to free the pages.
 699       free_region(bottom_region);
 700       free_region(top_region);
 701     } else {
 702       // In a system with static binding we have to change the bias whenever
 703       // we reshape the heap.
 704       bias_region(bottom_region, ls->lgrp_id());
 705       bias_region(top_region, ls->lgrp_id());
 706     }
 707 
 708     // Clear space (set top = bottom) but never mangle.
 709     s->initialize(new_region, SpaceDecorator::Clear, SpaceDecorator::DontMangle, MutableSpace::DontSetupPages);
 710 
 711     set_adaptation_cycles(samples_count());
 712   }
 713 }
 714 
 715 // Set the top of the whole space.
 716 // Mark the the holes in chunks below the top() as invalid.
 717 void MutableNUMASpace::set_top(HeapWord* value) {
 718   bool found_top = false;
 719   for (int i = 0; i < lgrp_spaces()->length();) {
 720     LGRPSpace *ls = lgrp_spaces()->at(i);
 721     MutableSpace *s = ls->space();
 722     HeapWord *top = MAX2((HeapWord*)round_down((intptr_t)s->top(), page_size()), s->bottom());
 723 
 724     if (s->contains(value)) {
 725       // Check if setting the chunk's top to a given value would create a hole less than
 726       // a minimal object; assuming that's not the last chunk in which case we don't care.
 727       if (i < lgrp_spaces()->length() - 1) {
 728         size_t remainder = pointer_delta(s->end(), value);
 729         const size_t min_fill_size = CollectedHeap::min_fill_size();
 730         if (remainder < min_fill_size && remainder > 0) {
 731           // Add a minimum size filler object; it will cross the chunk boundary.
 732           CollectedHeap::fill_with_object(value, min_fill_size);
 733           value += min_fill_size;
 734           assert(!s->contains(value), "Should be in the next chunk");
 735           // Restart the loop from the same chunk, since the value has moved
 736           // to the next one.
 737           continue;
 738         }
 739       }
 740 
 741       if (!os::numa_has_static_binding() && top < value && top < s->end()) {
 742         ls->add_invalid_region(MemRegion(top, value));
 743       }
 744       s->set_top(value);
 745       found_top = true;
 746     } else {
 747         if (found_top) {
 748             s->set_top(s->bottom());
 749         } else {
 750           if (!os::numa_has_static_binding() && top < s->end()) {
 751             ls->add_invalid_region(MemRegion(top, s->end()));
 752           }
 753           s->set_top(s->end());
 754         }
 755     }
 756     i++;
 757   }
 758   MutableSpace::set_top(value);
 759 }
 760 
 761 void MutableNUMASpace::clear(bool mangle_space) {
 762   MutableSpace::set_top(bottom());
 763   for (int i = 0; i < lgrp_spaces()->length(); i++) {
 764     // Never mangle NUMA spaces because the mangling will
 765     // bind the memory to a possibly unwanted lgroup.
 766     lgrp_spaces()->at(i)->space()->clear(SpaceDecorator::DontMangle);
 767   }
 768 }
 769 
 770 /*
 771    Linux supports static memory binding, therefore the most part of the
 772    logic dealing with the possible invalid page allocation is effectively
 773    disabled. Besides there is no notion of the home node in Linux. A
 774    thread is allowed to migrate freely. Although the scheduler is rather
 775    reluctant to move threads between the nodes. We check for the current
 776    node every allocation. And with a high probability a thread stays on
 777    the same node for some time allowing local access to recently allocated
 778    objects.
 779  */
 780 
 781 HeapWord* MutableNUMASpace::allocate(size_t size) {
 782   Thread* thr = Thread::current();
 783   int lgrp_id = thr->lgrp_id();
 784   if (lgrp_id == -1 || !os::numa_has_group_homing()) {
 785     lgrp_id = os::numa_get_group_id();
 786     thr->set_lgrp_id(lgrp_id);
 787   }
 788 
 789   int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals);
 790 
 791   // It is possible that a new CPU has been hotplugged and
 792   // we haven't reshaped the space accordingly.
 793   if (i == -1) {
 794     i = os::random() % lgrp_spaces()->length();
 795   }
 796 
 797   LGRPSpace* ls = lgrp_spaces()->at(i);
 798   MutableSpace *s = ls->space();
 799   HeapWord *p = s->allocate(size);
 800 
 801   if (p != NULL) {
 802     size_t remainder = s->free_in_words();
 803     if (remainder < CollectedHeap::min_fill_size() && remainder > 0) {
 804       s->set_top(s->top() - size);
 805       p = NULL;
 806     }
 807   }
 808   if (p != NULL) {
 809     if (top() < s->top()) { // Keep _top updated.
 810       MutableSpace::set_top(s->top());
 811     }
 812   }
 813   // Make the page allocation happen here if there is no static binding..
 814   if (p != NULL && !os::numa_has_static_binding()) {
 815     for (HeapWord *i = p; i < p + size; i += os::vm_page_size() >> LogHeapWordSize) {
 816       *(int*)i = 0;
 817     }
 818   }
 819   if (p == NULL) {
 820     ls->set_allocation_failed();
 821   }
 822   return p;
 823 }
 824 
 825 // This version is lock-free.
 826 HeapWord* MutableNUMASpace::cas_allocate(size_t size) {
 827   Thread* thr = Thread::current();
 828   int lgrp_id = thr->lgrp_id();
 829   if (lgrp_id == -1 || !os::numa_has_group_homing()) {
 830     lgrp_id = os::numa_get_group_id();
 831     thr->set_lgrp_id(lgrp_id);
 832   }
 833 
 834   int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals);
 835   // It is possible that a new CPU has been hotplugged and
 836   // we haven't reshaped the space accordingly.
 837   if (i == -1) {
 838     i = os::random() % lgrp_spaces()->length();
 839   }
 840   LGRPSpace *ls = lgrp_spaces()->at(i);
 841   MutableSpace *s = ls->space();
 842   HeapWord *p = s->cas_allocate(size);
 843   if (p != NULL) {
 844     size_t remainder = pointer_delta(s->end(), p + size);
 845     if (remainder < CollectedHeap::min_fill_size() && remainder > 0) {
 846       if (s->cas_deallocate(p, size)) {
 847         // We were the last to allocate and created a fragment less than
 848         // a minimal object.
 849         p = NULL;
 850       } else {
 851         guarantee(false, "Deallocation should always succeed");
 852       }
 853     }
 854   }
 855   if (p != NULL) {
 856     HeapWord* cur_top, *cur_chunk_top = p + size;
 857     while ((cur_top = top()) < cur_chunk_top) { // Keep _top updated.
 858       if (Atomic::cmpxchg_ptr(cur_chunk_top, top_addr(), cur_top) == cur_top) {
 859         break;
 860       }
 861     }
 862   }
 863 
 864   // Make the page allocation happen here if there is no static binding.
 865   if (p != NULL && !os::numa_has_static_binding() ) {
 866     for (HeapWord *i = p; i < p + size; i += os::vm_page_size() >> LogHeapWordSize) {
 867       *(int*)i = 0;
 868     }
 869   }
 870   if (p == NULL) {
 871     ls->set_allocation_failed();
 872   }
 873   return p;
 874 }
 875 
 876 void MutableNUMASpace::print_short_on(outputStream* st) const {
 877   MutableSpace::print_short_on(st);
 878   st->print(" (");
 879   for (int i = 0; i < lgrp_spaces()->length(); i++) {
 880     st->print("lgrp %d: ", lgrp_spaces()->at(i)->lgrp_id());
 881     lgrp_spaces()->at(i)->space()->print_short_on(st);
 882     if (i < lgrp_spaces()->length() - 1) {
 883       st->print(", ");
 884     }
 885   }
 886   st->print(")");
 887 }
 888 
 889 void MutableNUMASpace::print_on(outputStream* st) const {
 890   MutableSpace::print_on(st);
 891   for (int i = 0; i < lgrp_spaces()->length(); i++) {
 892     LGRPSpace *ls = lgrp_spaces()->at(i);
 893     st->print("    lgrp %d", ls->lgrp_id());
 894     ls->space()->print_on(st);
 895     if (NUMAStats) {
 896       for (int i = 0; i < lgrp_spaces()->length(); i++) {
 897         lgrp_spaces()->at(i)->accumulate_statistics(page_size());
 898       }
 899       st->print("    local/remote/unbiased/uncommitted: " SIZE_FORMAT "K/"
 900                 SIZE_FORMAT "K/" SIZE_FORMAT "K/" SIZE_FORMAT
 901                 "K, large/small pages: " SIZE_FORMAT "/" SIZE_FORMAT "\n",
 902                 ls->space_stats()->_local_space / K,
 903                 ls->space_stats()->_remote_space / K,
 904                 ls->space_stats()->_unbiased_space / K,
 905                 ls->space_stats()->_uncommited_space / K,
 906                 ls->space_stats()->_large_pages,
 907                 ls->space_stats()->_small_pages);
 908     }
 909   }
 910 }
 911 
 912 void MutableNUMASpace::verify() {
 913   // This can be called after setting an arbitrary value to the space's top,
 914   // so an object can cross the chunk boundary. We ensure the parsability
 915   // of the space and just walk the objects in linear fashion.
 916   ensure_parsability();
 917   MutableSpace::verify();
 918 }
 919 
 920 // Scan pages and gather stats about page placement and size.
 921 void MutableNUMASpace::LGRPSpace::accumulate_statistics(size_t page_size) {
 922   clear_space_stats();
 923   char *start = (char*)round_to((intptr_t) space()->bottom(), page_size);
 924   char* end = (char*)round_down((intptr_t) space()->end(), page_size);
 925   if (start < end) {
 926     for (char *p = start; p < end;) {
 927       os::page_info info;
 928       if (os::get_page_info(p, &info)) {
 929         if (info.size > 0) {
 930           if (info.size > (size_t)os::vm_page_size()) {
 931             space_stats()->_large_pages++;
 932           } else {
 933             space_stats()->_small_pages++;
 934           }
 935           if (info.lgrp_id == lgrp_id()) {
 936             space_stats()->_local_space += info.size;
 937           } else {
 938             space_stats()->_remote_space += info.size;
 939           }
 940           p += info.size;
 941         } else {
 942           p += os::vm_page_size();
 943           space_stats()->_uncommited_space += os::vm_page_size();
 944         }
 945       } else {
 946         return;
 947       }
 948     }
 949   }
 950   space_stats()->_unbiased_space = pointer_delta(start, space()->bottom(), sizeof(char)) +
 951                                    pointer_delta(space()->end(), end, sizeof(char));
 952 
 953 }
 954 
 955 // Scan page_count pages and verify if they have the right size and right placement.
 956 // If invalid pages are found they are freed in hope that subsequent reallocation
 957 // will be more successful.
 958 void MutableNUMASpace::LGRPSpace::scan_pages(size_t page_size, size_t page_count)
 959 {
 960   char* range_start = (char*)round_to((intptr_t) space()->bottom(), page_size);
 961   char* range_end = (char*)round_down((intptr_t) space()->end(), page_size);
 962 
 963   if (range_start > last_page_scanned() || last_page_scanned() >= range_end) {
 964     set_last_page_scanned(range_start);
 965   }
 966 
 967   char *scan_start = last_page_scanned();
 968   char* scan_end = MIN2(scan_start + page_size * page_count, range_end);
 969 
 970   os::page_info page_expected, page_found;
 971   page_expected.size = page_size;
 972   page_expected.lgrp_id = lgrp_id();
 973 
 974   char *s = scan_start;
 975   while (s < scan_end) {
 976     char *e = os::scan_pages(s, (char*)scan_end, &page_expected, &page_found);
 977     if (e == NULL) {
 978       break;
 979     }
 980     if (e != scan_end) {
 981       assert(e < scan_end, "e: " PTR_FORMAT " scan_end: " PTR_FORMAT, p2i(e), p2i(scan_end));
 982 
 983       if ((page_expected.size != page_size || page_expected.lgrp_id != lgrp_id())
 984           && page_expected.size != 0) {
 985         os::free_memory(s, pointer_delta(e, s, sizeof(char)), page_size);
 986       }
 987       page_expected = page_found;
 988     }
 989     s = e;
 990   }
 991 
 992   set_last_page_scanned(scan_end);
 993 }
--- EOF ---