1 /*
   2  * Copyright (c) 2011, 2016, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 #include "precompiled.hpp"
  25 #include "gc/shared/collectedHeap.hpp"
  26 #include "gc/shared/collectorPolicy.hpp"
  27 #include "gc/shared/gcLocker.hpp"
  28 #include "logging/log.hpp"
  29 #include "memory/allocation.hpp"
  30 #include "memory/binaryTreeDictionary.hpp"
  31 #include "memory/filemap.hpp"
  32 #include "memory/freeList.hpp"
  33 #include "memory/metachunk.hpp"
  34 #include "memory/metaspace.hpp"
  35 #include "memory/metaspaceGCThresholdUpdater.hpp"
  36 #include "memory/metaspaceShared.hpp"
  37 #include "memory/metaspaceTracer.hpp"
  38 #include "memory/resourceArea.hpp"
  39 #include "memory/universe.hpp"
  40 #include "runtime/atomic.inline.hpp"
  41 #include "runtime/globals.hpp"
  42 #include "runtime/init.hpp"
  43 #include "runtime/java.hpp"
  44 #include "runtime/mutex.hpp"
  45 #include "runtime/orderAccess.inline.hpp"
  46 #include "services/memTracker.hpp"
  47 #include "services/memoryService.hpp"
  48 #include "utilities/copy.hpp"
  49 #include "utilities/debug.hpp"
  50 #include "utilities/macros.hpp"
  51 
  52 typedef BinaryTreeDictionary<Metablock, FreeList<Metablock> > BlockTreeDictionary;
  53 typedef BinaryTreeDictionary<Metachunk, FreeList<Metachunk> > ChunkTreeDictionary;
  54 
  55 // Set this constant to enable slow integrity checking of the free chunk lists
  56 const bool metaspace_slow_verify = false;
  57 
  58 size_t const allocation_from_dictionary_limit = 4 * K;
  59 
  60 MetaWord* last_allocated = 0;
  61 
  62 size_t Metaspace::_compressed_class_space_size;
  63 const MetaspaceTracer* Metaspace::_tracer = NULL;
  64 
  65 // Used in declarations in SpaceManager and ChunkManager
  66 enum ChunkIndex {
  67   ZeroIndex = 0,
  68   SpecializedIndex = ZeroIndex,
  69   SmallIndex = SpecializedIndex + 1,
  70   MediumIndex = SmallIndex + 1,
  71   HumongousIndex = MediumIndex + 1,
  72   NumberOfFreeLists = 3,
  73   NumberOfInUseLists = 4
  74 };
  75 
  76 enum ChunkSizes {    // in words.
  77   ClassSpecializedChunk = 128,
  78   SpecializedChunk = 128,
  79   ClassSmallChunk = 256,
  80   SmallChunk = 512,
  81   ClassMediumChunk = 4 * K,
  82   MediumChunk = 8 * K
  83 };
  84 
  85 static ChunkIndex next_chunk_index(ChunkIndex i) {
  86   assert(i < NumberOfInUseLists, "Out of bound");
  87   return (ChunkIndex) (i+1);
  88 }
  89 
  90 volatile intptr_t MetaspaceGC::_capacity_until_GC = 0;
  91 uint MetaspaceGC::_shrink_factor = 0;
  92 bool MetaspaceGC::_should_concurrent_collect = false;
  93 
  94 typedef class FreeList<Metachunk> ChunkList;
  95 
  96 // Manages the global free lists of chunks.
  97 class ChunkManager : public CHeapObj<mtInternal> {
  98   friend class TestVirtualSpaceNodeTest;
  99 
 100   // Free list of chunks of different sizes.
 101   //   SpecializedChunk
 102   //   SmallChunk
 103   //   MediumChunk
 104   //   HumongousChunk
 105   ChunkList _free_chunks[NumberOfFreeLists];
 106 
 107   //   HumongousChunk
 108   ChunkTreeDictionary _humongous_dictionary;
 109 
 110   // ChunkManager in all lists of this type
 111   size_t _free_chunks_total;
 112   size_t _free_chunks_count;
 113 
 114   void dec_free_chunks_total(size_t v) {
 115     assert(_free_chunks_count > 0 &&
 116              _free_chunks_total > 0,
 117              "About to go negative");
 118     Atomic::add_ptr(-1, &_free_chunks_count);
 119     jlong minus_v = (jlong) - (jlong) v;
 120     Atomic::add_ptr(minus_v, &_free_chunks_total);
 121   }
 122 
 123   // Debug support
 124 
 125   size_t sum_free_chunks();
 126   size_t sum_free_chunks_count();
 127 
 128   void locked_verify_free_chunks_total();
 129   void slow_locked_verify_free_chunks_total() {
 130     if (metaspace_slow_verify) {
 131       locked_verify_free_chunks_total();
 132     }
 133   }
 134   void locked_verify_free_chunks_count();
 135   void slow_locked_verify_free_chunks_count() {
 136     if (metaspace_slow_verify) {
 137       locked_verify_free_chunks_count();
 138     }
 139   }
 140   void verify_free_chunks_count();
 141 
 142  public:
 143 
 144   ChunkManager(size_t specialized_size, size_t small_size, size_t medium_size)
 145       : _free_chunks_total(0), _free_chunks_count(0) {
 146     _free_chunks[SpecializedIndex].set_size(specialized_size);
 147     _free_chunks[SmallIndex].set_size(small_size);
 148     _free_chunks[MediumIndex].set_size(medium_size);
 149   }
 150 
 151   // add or delete (return) a chunk to the global freelist.
 152   Metachunk* chunk_freelist_allocate(size_t word_size);
 153 
 154   // Map a size to a list index assuming that there are lists
 155   // for special, small, medium, and humongous chunks.
 156   static ChunkIndex list_index(size_t size);
 157 
 158   // Remove the chunk from its freelist.  It is
 159   // expected to be on one of the _free_chunks[] lists.
 160   void remove_chunk(Metachunk* chunk);
 161 
 162   // Add the simple linked list of chunks to the freelist of chunks
 163   // of type index.
 164   void return_chunks(ChunkIndex index, Metachunk* chunks);
 165 
 166   // Total of the space in the free chunks list
 167   size_t free_chunks_total_words();
 168   size_t free_chunks_total_bytes();
 169 
 170   // Number of chunks in the free chunks list
 171   size_t free_chunks_count();
 172 
 173   void inc_free_chunks_total(size_t v, size_t count = 1) {
 174     Atomic::add_ptr(count, &_free_chunks_count);
 175     Atomic::add_ptr(v, &_free_chunks_total);
 176   }
 177   ChunkTreeDictionary* humongous_dictionary() {
 178     return &_humongous_dictionary;
 179   }
 180 
 181   ChunkList* free_chunks(ChunkIndex index);
 182 
 183   // Returns the list for the given chunk word size.
 184   ChunkList* find_free_chunks_list(size_t word_size);
 185 
 186   // Remove from a list by size.  Selects list based on size of chunk.
 187   Metachunk* free_chunks_get(size_t chunk_word_size);
 188 
 189 #define index_bounds_check(index)                                         \
 190   assert(index == SpecializedIndex ||                                     \
 191          index == SmallIndex ||                                           \
 192          index == MediumIndex ||                                          \
 193          index == HumongousIndex, "Bad index: %d", (int) index)
 194 
 195   size_t num_free_chunks(ChunkIndex index) const {
 196     index_bounds_check(index);
 197 
 198     if (index == HumongousIndex) {
 199       return _humongous_dictionary.total_free_blocks();
 200     }
 201 
 202     ssize_t count = _free_chunks[index].count();
 203     return count == -1 ? 0 : (size_t) count;
 204   }
 205 
 206   size_t size_free_chunks_in_bytes(ChunkIndex index) const {
 207     index_bounds_check(index);
 208 
 209     size_t word_size = 0;
 210     if (index == HumongousIndex) {
 211       word_size = _humongous_dictionary.total_size();
 212     } else {
 213       const size_t size_per_chunk_in_words = _free_chunks[index].size();
 214       word_size = size_per_chunk_in_words * num_free_chunks(index);
 215     }
 216 
 217     return word_size * BytesPerWord;
 218   }
 219 
 220   MetaspaceChunkFreeListSummary chunk_free_list_summary() const {
 221     return MetaspaceChunkFreeListSummary(num_free_chunks(SpecializedIndex),
 222                                          num_free_chunks(SmallIndex),
 223                                          num_free_chunks(MediumIndex),
 224                                          num_free_chunks(HumongousIndex),
 225                                          size_free_chunks_in_bytes(SpecializedIndex),
 226                                          size_free_chunks_in_bytes(SmallIndex),
 227                                          size_free_chunks_in_bytes(MediumIndex),
 228                                          size_free_chunks_in_bytes(HumongousIndex));
 229   }
 230 
 231   // Debug support
 232   void verify();
 233   void slow_verify() {
 234     if (metaspace_slow_verify) {
 235       verify();
 236     }
 237   }
 238   void locked_verify();
 239   void slow_locked_verify() {
 240     if (metaspace_slow_verify) {
 241       locked_verify();
 242     }
 243   }
 244   void verify_free_chunks_total();
 245 
 246   void locked_print_free_chunks(outputStream* st);
 247   void locked_print_sum_free_chunks(outputStream* st);
 248 
 249   void print_on(outputStream* st) const;
 250 };
 251 
 252 // Used to manage the free list of Metablocks (a block corresponds
 253 // to the allocation of a quantum of metadata).
 254 class BlockFreelist VALUE_OBJ_CLASS_SPEC {
 255   BlockTreeDictionary* const _dictionary;
 256 
 257   // Only allocate and split from freelist if the size of the allocation
 258   // is at least 1/4th the size of the available block.
 259   const static int WasteMultiplier = 4;
 260 
 261   // Accessors
 262   BlockTreeDictionary* dictionary() const { return _dictionary; }
 263 
 264  public:
 265   BlockFreelist();
 266   ~BlockFreelist();
 267 
 268   // Get and return a block to the free list
 269   MetaWord* get_block(size_t word_size);
 270   void return_block(MetaWord* p, size_t word_size);
 271 
 272   size_t total_size() { return dictionary()->total_size(); }
 273 
 274   void print_on(outputStream* st) const;
 275 };
 276 
 277 // A VirtualSpaceList node.
 278 class VirtualSpaceNode : public CHeapObj<mtClass> {
 279   friend class VirtualSpaceList;
 280 
 281   // Link to next VirtualSpaceNode
 282   VirtualSpaceNode* _next;
 283 
 284   // total in the VirtualSpace
 285   MemRegion _reserved;
 286   ReservedSpace _rs;
 287   VirtualSpace _virtual_space;
 288   MetaWord* _top;
 289   // count of chunks contained in this VirtualSpace
 290   uintx _container_count;
 291 
 292   // Convenience functions to access the _virtual_space
 293   char* low()  const { return virtual_space()->low(); }
 294   char* high() const { return virtual_space()->high(); }
 295 
 296   // The first Metachunk will be allocated at the bottom of the
 297   // VirtualSpace
 298   Metachunk* first_chunk() { return (Metachunk*) bottom(); }
 299 
 300   // Committed but unused space in the virtual space
 301   size_t free_words_in_vs() const;
 302  public:
 303 
 304   VirtualSpaceNode(size_t byte_size);
 305   VirtualSpaceNode(ReservedSpace rs) : _top(NULL), _next(NULL), _rs(rs), _container_count(0) {}
 306   ~VirtualSpaceNode();
 307 
 308   // Convenience functions for logical bottom and end
 309   MetaWord* bottom() const { return (MetaWord*) _virtual_space.low(); }
 310   MetaWord* end() const { return (MetaWord*) _virtual_space.high(); }
 311 
 312   bool contains(const void* ptr) { return ptr >= low() && ptr < high(); }
 313 
 314   size_t reserved_words() const  { return _virtual_space.reserved_size() / BytesPerWord; }
 315   size_t committed_words() const { return _virtual_space.actual_committed_size() / BytesPerWord; }
 316 
 317   bool is_pre_committed() const { return _virtual_space.special(); }
 318 
 319   // address of next available space in _virtual_space;
 320   // Accessors
 321   VirtualSpaceNode* next() { return _next; }
 322   void set_next(VirtualSpaceNode* v) { _next = v; }
 323 
 324   void set_reserved(MemRegion const v) { _reserved = v; }
 325   void set_top(MetaWord* v) { _top = v; }
 326 
 327   // Accessors
 328   MemRegion* reserved() { return &_reserved; }
 329   VirtualSpace* virtual_space() const { return (VirtualSpace*) &_virtual_space; }
 330 
 331   // Returns true if "word_size" is available in the VirtualSpace
 332   bool is_available(size_t word_size) { return word_size <= pointer_delta(end(), _top, sizeof(MetaWord)); }
 333 
 334   MetaWord* top() const { return _top; }
 335   void inc_top(size_t word_size) { _top += word_size; }
 336 
 337   uintx container_count() { return _container_count; }
 338   void inc_container_count();
 339   void dec_container_count();
 340 #ifdef ASSERT
 341   uintx container_count_slow();
 342   void verify_container_count();
 343 #endif
 344 
 345   // used and capacity in this single entry in the list
 346   size_t used_words_in_vs() const;
 347   size_t capacity_words_in_vs() const;
 348 
 349   bool initialize();
 350 
 351   // get space from the virtual space
 352   Metachunk* take_from_committed(size_t chunk_word_size);
 353 
 354   // Allocate a chunk from the virtual space and return it.
 355   Metachunk* get_chunk_vs(size_t chunk_word_size);
 356 
 357   // Expands/shrinks the committed space in a virtual space.  Delegates
 358   // to Virtualspace
 359   bool expand_by(size_t min_words, size_t preferred_words);
 360 
 361   // In preparation for deleting this node, remove all the chunks
 362   // in the node from any freelist.
 363   void purge(ChunkManager* chunk_manager);
 364 
 365   // If an allocation doesn't fit in the current node a new node is created.
 366   // Allocate chunks out of the remaining committed space in this node
 367   // to avoid wasting that memory.
 368   // This always adds up because all the chunk sizes are multiples of
 369   // the smallest chunk size.
 370   void retire(ChunkManager* chunk_manager);
 371 
 372 #ifdef ASSERT
 373   // Debug support
 374   void mangle();
 375 #endif
 376 
 377   void print_on(outputStream* st) const;
 378 };
 379 
 380 #define assert_is_ptr_aligned(ptr, alignment) \
 381   assert(is_ptr_aligned(ptr, alignment),      \
 382          PTR_FORMAT " is not aligned to "     \
 383          SIZE_FORMAT, p2i(ptr), alignment)
 384 
 385 #define assert_is_size_aligned(size, alignment) \
 386   assert(is_size_aligned(size, alignment),      \
 387          SIZE_FORMAT " is not aligned to "      \
 388          SIZE_FORMAT, size, alignment)
 389 
 390 
 391 // Decide if large pages should be committed when the memory is reserved.
 392 static bool should_commit_large_pages_when_reserving(size_t bytes) {
 393   if (UseLargePages && UseLargePagesInMetaspace && !os::can_commit_large_page_memory()) {
 394     size_t words = bytes / BytesPerWord;
 395     bool is_class = false; // We never reserve large pages for the class space.
 396     if (MetaspaceGC::can_expand(words, is_class) &&
 397         MetaspaceGC::allowed_expansion() >= words) {
 398       return true;
 399     }
 400   }
 401 
 402   return false;
 403 }
 404 
 405   // byte_size is the size of the associated virtualspace.
 406 VirtualSpaceNode::VirtualSpaceNode(size_t bytes) : _top(NULL), _next(NULL), _rs(), _container_count(0) {
 407   assert_is_size_aligned(bytes, Metaspace::reserve_alignment());
 408 
 409 #if INCLUDE_CDS
 410   // This allocates memory with mmap.  For DumpSharedspaces, try to reserve
 411   // configurable address, generally at the top of the Java heap so other
 412   // memory addresses don't conflict.
 413   if (DumpSharedSpaces) {
 414     bool large_pages = false; // No large pages when dumping the CDS archive.
 415     char* shared_base = (char*)align_ptr_up((char*)SharedBaseAddress, Metaspace::reserve_alignment());
 416 
 417     _rs = ReservedSpace(bytes, Metaspace::reserve_alignment(), large_pages, shared_base);
 418     if (_rs.is_reserved()) {
 419       assert(shared_base == 0 || _rs.base() == shared_base, "should match");
 420     } else {
 421       // Get a mmap region anywhere if the SharedBaseAddress fails.
 422       _rs = ReservedSpace(bytes, Metaspace::reserve_alignment(), large_pages);
 423     }
 424     MetaspaceShared::initialize_shared_rs(&_rs);
 425   } else
 426 #endif
 427   {
 428     bool large_pages = should_commit_large_pages_when_reserving(bytes);
 429 
 430     _rs = ReservedSpace(bytes, Metaspace::reserve_alignment(), large_pages);
 431   }
 432 
 433   if (_rs.is_reserved()) {
 434     assert(_rs.base() != NULL, "Catch if we get a NULL address");
 435     assert(_rs.size() != 0, "Catch if we get a 0 size");
 436     assert_is_ptr_aligned(_rs.base(), Metaspace::reserve_alignment());
 437     assert_is_size_aligned(_rs.size(), Metaspace::reserve_alignment());
 438 
 439     MemTracker::record_virtual_memory_type((address)_rs.base(), mtClass);
 440   }
 441 }
 442 
 443 void VirtualSpaceNode::purge(ChunkManager* chunk_manager) {
 444   Metachunk* chunk = first_chunk();
 445   Metachunk* invalid_chunk = (Metachunk*) top();
 446   while (chunk < invalid_chunk ) {
 447     assert(chunk->is_tagged_free(), "Should be tagged free");
 448     MetaWord* next = ((MetaWord*)chunk) + chunk->word_size();
 449     chunk_manager->remove_chunk(chunk);
 450     assert(chunk->next() == NULL &&
 451            chunk->prev() == NULL,
 452            "Was not removed from its list");
 453     chunk = (Metachunk*) next;
 454   }
 455 }
 456 
 457 #ifdef ASSERT
 458 uintx VirtualSpaceNode::container_count_slow() {
 459   uintx count = 0;
 460   Metachunk* chunk = first_chunk();
 461   Metachunk* invalid_chunk = (Metachunk*) top();
 462   while (chunk < invalid_chunk ) {
 463     MetaWord* next = ((MetaWord*)chunk) + chunk->word_size();
 464     // Don't count the chunks on the free lists.  Those are
 465     // still part of the VirtualSpaceNode but not currently
 466     // counted.
 467     if (!chunk->is_tagged_free()) {
 468       count++;
 469     }
 470     chunk = (Metachunk*) next;
 471   }
 472   return count;
 473 }
 474 #endif
 475 
 476 // List of VirtualSpaces for metadata allocation.
 477 class VirtualSpaceList : public CHeapObj<mtClass> {
 478   friend class VirtualSpaceNode;
 479 
 480   enum VirtualSpaceSizes {
 481     VirtualSpaceSize = 256 * K
 482   };
 483 
 484   // Head of the list
 485   VirtualSpaceNode* _virtual_space_list;
 486   // virtual space currently being used for allocations
 487   VirtualSpaceNode* _current_virtual_space;
 488 
 489   // Is this VirtualSpaceList used for the compressed class space
 490   bool _is_class;
 491 
 492   // Sum of reserved and committed memory in the virtual spaces
 493   size_t _reserved_words;
 494   size_t _committed_words;
 495 
 496   // Number of virtual spaces
 497   size_t _virtual_space_count;
 498 
 499   ~VirtualSpaceList();
 500 
 501   VirtualSpaceNode* virtual_space_list() const { return _virtual_space_list; }
 502 
 503   void set_virtual_space_list(VirtualSpaceNode* v) {
 504     _virtual_space_list = v;
 505   }
 506   void set_current_virtual_space(VirtualSpaceNode* v) {
 507     _current_virtual_space = v;
 508   }
 509 
 510   void link_vs(VirtualSpaceNode* new_entry);
 511 
 512   // Get another virtual space and add it to the list.  This
 513   // is typically prompted by a failed attempt to allocate a chunk
 514   // and is typically followed by the allocation of a chunk.
 515   bool create_new_virtual_space(size_t vs_word_size);
 516 
 517   // Chunk up the unused committed space in the current
 518   // virtual space and add the chunks to the free list.
 519   void retire_current_virtual_space();
 520 
 521  public:
 522   VirtualSpaceList(size_t word_size);
 523   VirtualSpaceList(ReservedSpace rs);
 524 
 525   size_t free_bytes();
 526 
 527   Metachunk* get_new_chunk(size_t word_size,
 528                            size_t grow_chunks_by_words,
 529                            size_t medium_chunk_bunch);
 530 
 531   bool expand_node_by(VirtualSpaceNode* node,
 532                       size_t min_words,
 533                       size_t preferred_words);
 534 
 535   bool expand_by(size_t min_words,
 536                  size_t preferred_words);
 537 
 538   VirtualSpaceNode* current_virtual_space() {
 539     return _current_virtual_space;
 540   }
 541 
 542   bool is_class() const { return _is_class; }
 543 
 544   bool initialization_succeeded() { return _virtual_space_list != NULL; }
 545 
 546   size_t reserved_words()  { return _reserved_words; }
 547   size_t reserved_bytes()  { return reserved_words() * BytesPerWord; }
 548   size_t committed_words() { return _committed_words; }
 549   size_t committed_bytes() { return committed_words() * BytesPerWord; }
 550 
 551   void inc_reserved_words(size_t v);
 552   void dec_reserved_words(size_t v);
 553   void inc_committed_words(size_t v);
 554   void dec_committed_words(size_t v);
 555   void inc_virtual_space_count();
 556   void dec_virtual_space_count();
 557 
 558   bool contains(const void* ptr);
 559 
 560   // Unlink empty VirtualSpaceNodes and free it.
 561   void purge(ChunkManager* chunk_manager);
 562 
 563   void print_on(outputStream* st) const;
 564 
 565   class VirtualSpaceListIterator : public StackObj {
 566     VirtualSpaceNode* _virtual_spaces;
 567    public:
 568     VirtualSpaceListIterator(VirtualSpaceNode* virtual_spaces) :
 569       _virtual_spaces(virtual_spaces) {}
 570 
 571     bool repeat() {
 572       return _virtual_spaces != NULL;
 573     }
 574 
 575     VirtualSpaceNode* get_next() {
 576       VirtualSpaceNode* result = _virtual_spaces;
 577       if (_virtual_spaces != NULL) {
 578         _virtual_spaces = _virtual_spaces->next();
 579       }
 580       return result;
 581     }
 582   };
 583 };
 584 
 585 class Metadebug : AllStatic {
 586   // Debugging support for Metaspaces
 587   static int _allocation_fail_alot_count;
 588 
 589  public:
 590 
 591   static void init_allocation_fail_alot_count();
 592 #ifdef ASSERT
 593   static bool test_metadata_failure();
 594 #endif
 595 };
 596 
 597 int Metadebug::_allocation_fail_alot_count = 0;
 598 
 599 //  SpaceManager - used by Metaspace to handle allocations
 600 class SpaceManager : public CHeapObj<mtClass> {
 601   friend class Metaspace;
 602   friend class Metadebug;
 603 
 604  private:
 605 
 606   // protects allocations
 607   Mutex* const _lock;
 608 
 609   // Type of metadata allocated.
 610   Metaspace::MetadataType _mdtype;
 611 
 612   // List of chunks in use by this SpaceManager.  Allocations
 613   // are done from the current chunk.  The list is used for deallocating
 614   // chunks when the SpaceManager is freed.
 615   Metachunk* _chunks_in_use[NumberOfInUseLists];
 616   Metachunk* _current_chunk;
 617 
 618   // Maximum number of small chunks to allocate to a SpaceManager
 619   static uint const _small_chunk_limit;
 620 
 621   // Sum of all space in allocated chunks
 622   size_t _allocated_blocks_words;
 623 
 624   // Sum of all allocated chunks
 625   size_t _allocated_chunks_words;
 626   size_t _allocated_chunks_count;
 627 
 628   // Free lists of blocks are per SpaceManager since they
 629   // are assumed to be in chunks in use by the SpaceManager
 630   // and all chunks in use by a SpaceManager are freed when
 631   // the class loader using the SpaceManager is collected.
 632   BlockFreelist _block_freelists;
 633 
 634   // protects virtualspace and chunk expansions
 635   static const char*  _expand_lock_name;
 636   static const int    _expand_lock_rank;
 637   static Mutex* const _expand_lock;
 638 
 639  private:
 640   // Accessors
 641   Metachunk* chunks_in_use(ChunkIndex index) const { return _chunks_in_use[index]; }
 642   void set_chunks_in_use(ChunkIndex index, Metachunk* v) {
 643     _chunks_in_use[index] = v;
 644   }
 645 
 646   BlockFreelist* block_freelists() const {
 647     return (BlockFreelist*) &_block_freelists;
 648   }
 649 
 650   Metaspace::MetadataType mdtype() { return _mdtype; }
 651 
 652   VirtualSpaceList* vs_list()   const { return Metaspace::get_space_list(_mdtype); }
 653   ChunkManager* chunk_manager() const { return Metaspace::get_chunk_manager(_mdtype); }
 654 
 655   Metachunk* current_chunk() const { return _current_chunk; }
 656   void set_current_chunk(Metachunk* v) {
 657     _current_chunk = v;
 658   }
 659 
 660   Metachunk* find_current_chunk(size_t word_size);
 661 
 662   // Add chunk to the list of chunks in use
 663   void add_chunk(Metachunk* v, bool make_current);
 664   void retire_current_chunk();
 665 
 666   Mutex* lock() const { return _lock; }
 667 
 668   const char* chunk_size_name(ChunkIndex index) const;
 669 
 670  protected:
 671   void initialize();
 672 
 673  public:
 674   SpaceManager(Metaspace::MetadataType mdtype,
 675                Mutex* lock);
 676   ~SpaceManager();
 677 
 678   enum ChunkMultiples {
 679     MediumChunkMultiple = 4
 680   };
 681 
 682   bool is_class() { return _mdtype == Metaspace::ClassType; }
 683 
 684   // Accessors
 685   size_t specialized_chunk_size() { return (size_t) is_class() ? ClassSpecializedChunk : SpecializedChunk; }
 686   size_t small_chunk_size()       { return (size_t) is_class() ? ClassSmallChunk : SmallChunk; }
 687   size_t medium_chunk_size()      { return (size_t) is_class() ? ClassMediumChunk : MediumChunk; }
 688   size_t medium_chunk_bunch()     { return medium_chunk_size() * MediumChunkMultiple; }
 689 
 690   size_t smallest_chunk_size()  { return specialized_chunk_size(); }
 691 
 692   size_t allocated_blocks_words() const { return _allocated_blocks_words; }
 693   size_t allocated_blocks_bytes() const { return _allocated_blocks_words * BytesPerWord; }
 694   size_t allocated_chunks_words() const { return _allocated_chunks_words; }
 695   size_t allocated_chunks_bytes() const { return _allocated_chunks_words * BytesPerWord; }
 696   size_t allocated_chunks_count() const { return _allocated_chunks_count; }
 697 
 698   bool is_humongous(size_t word_size) { return word_size > medium_chunk_size(); }
 699 
 700   static Mutex* expand_lock() { return _expand_lock; }
 701 
 702   // Increment the per Metaspace and global running sums for Metachunks
 703   // by the given size.  This is used when a Metachunk to added to
 704   // the in-use list.
 705   void inc_size_metrics(size_t words);
 706   // Increment the per Metaspace and global running sums Metablocks by the given
 707   // size.  This is used when a Metablock is allocated.
 708   void inc_used_metrics(size_t words);
 709   // Delete the portion of the running sums for this SpaceManager. That is,
 710   // the globals running sums for the Metachunks and Metablocks are
 711   // decremented for all the Metachunks in-use by this SpaceManager.
 712   void dec_total_from_size_metrics();
 713 
 714   // Set the sizes for the initial chunks.
 715   void get_initial_chunk_sizes(Metaspace::MetaspaceType type,
 716                                size_t* chunk_word_size,
 717                                size_t* class_chunk_word_size);
 718 
 719   size_t sum_capacity_in_chunks_in_use() const;
 720   size_t sum_used_in_chunks_in_use() const;
 721   size_t sum_free_in_chunks_in_use() const;
 722   size_t sum_waste_in_chunks_in_use() const;
 723   size_t sum_waste_in_chunks_in_use(ChunkIndex index ) const;
 724 
 725   size_t sum_count_in_chunks_in_use();
 726   size_t sum_count_in_chunks_in_use(ChunkIndex i);
 727 
 728   Metachunk* get_new_chunk(size_t word_size, size_t grow_chunks_by_words);
 729 
 730   // Block allocation and deallocation.
 731   // Allocates a block from the current chunk
 732   MetaWord* allocate(size_t word_size);
 733   // Allocates a block from a small chunk
 734   MetaWord* get_small_chunk_and_allocate(size_t word_size);
 735 
 736   // Helper for allocations
 737   MetaWord* allocate_work(size_t word_size);
 738 
 739   // Returns a block to the per manager freelist
 740   void deallocate(MetaWord* p, size_t word_size);
 741 
 742   // Based on the allocation size and a minimum chunk size,
 743   // returned chunk size (for expanding space for chunk allocation).
 744   size_t calc_chunk_size(size_t allocation_word_size);
 745 
 746   // Called when an allocation from the current chunk fails.
 747   // Gets a new chunk (may require getting a new virtual space),
 748   // and allocates from that chunk.
 749   MetaWord* grow_and_allocate(size_t word_size);
 750 
 751   // Notify memory usage to MemoryService.
 752   void track_metaspace_memory_usage();
 753 
 754   // debugging support.
 755 
 756   void dump(outputStream* const out) const;
 757   void print_on(outputStream* st) const;
 758   void locked_print_chunks_in_use_on(outputStream* st) const;
 759 
 760   void verify();
 761   void verify_chunk_size(Metachunk* chunk);
 762   NOT_PRODUCT(void mangle_freed_chunks();)
 763 #ifdef ASSERT
 764   void verify_allocated_blocks_words();
 765 #endif
 766 
 767   size_t get_raw_word_size(size_t word_size) {
 768     size_t byte_size = word_size * BytesPerWord;
 769 
 770     size_t raw_bytes_size = MAX2(byte_size, sizeof(Metablock));
 771     raw_bytes_size = align_size_up(raw_bytes_size, Metachunk::object_alignment());
 772 
 773     size_t raw_word_size = raw_bytes_size / BytesPerWord;
 774     assert(raw_word_size * BytesPerWord == raw_bytes_size, "Size problem");
 775 
 776     return raw_word_size;
 777   }
 778 };
 779 
 780 uint const SpaceManager::_small_chunk_limit = 4;
 781 
 782 const char* SpaceManager::_expand_lock_name =
 783   "SpaceManager chunk allocation lock";
 784 const int SpaceManager::_expand_lock_rank = Monitor::leaf - 1;
 785 Mutex* const SpaceManager::_expand_lock =
 786   new Mutex(SpaceManager::_expand_lock_rank,
 787             SpaceManager::_expand_lock_name,
 788             Mutex::_allow_vm_block_flag,
 789             Monitor::_safepoint_check_never);
 790 
 791 void VirtualSpaceNode::inc_container_count() {
 792   assert_lock_strong(SpaceManager::expand_lock());
 793   _container_count++;
 794 }
 795 
 796 void VirtualSpaceNode::dec_container_count() {
 797   assert_lock_strong(SpaceManager::expand_lock());
 798   _container_count--;
 799 }
 800 
 801 #ifdef ASSERT
 802 void VirtualSpaceNode::verify_container_count() {
 803   assert(_container_count == container_count_slow(),
 804          "Inconsistency in container_count _container_count " UINTX_FORMAT
 805          " container_count_slow() " UINTX_FORMAT, _container_count, container_count_slow());
 806 }
 807 #endif
 808 
 809 // BlockFreelist methods
 810 
 811 BlockFreelist::BlockFreelist() : _dictionary(new BlockTreeDictionary()) {}
 812 
 813 BlockFreelist::~BlockFreelist() {
 814   LogHandle(gc, metaspace, freelist) log;
 815   if (log.is_trace()) {
 816     ResourceMark rm;
 817     dictionary()->print_free_lists(log.trace_stream());
 818   }
 819   delete _dictionary;
 820 }
 821 
 822 void BlockFreelist::return_block(MetaWord* p, size_t word_size) {
 823   Metablock* free_chunk = ::new (p) Metablock(word_size);
 824   dictionary()->return_chunk(free_chunk);
 825 }
 826 
 827 MetaWord* BlockFreelist::get_block(size_t word_size) {
 828   if (word_size < TreeChunk<Metablock, FreeList<Metablock> >::min_size()) {
 829     // Dark matter.  Too small for dictionary.
 830     return NULL;
 831   }
 832 
 833   Metablock* free_block =
 834     dictionary()->get_chunk(word_size, FreeBlockDictionary<Metablock>::atLeast);
 835   if (free_block == NULL) {
 836     return NULL;
 837   }
 838 
 839   const size_t block_size = free_block->size();
 840   if (block_size > WasteMultiplier * word_size) {
 841     return_block((MetaWord*)free_block, block_size);
 842     return NULL;
 843   }
 844 
 845   MetaWord* new_block = (MetaWord*)free_block;
 846   assert(block_size >= word_size, "Incorrect size of block from freelist");
 847   const size_t unused = block_size - word_size;
 848   if (unused >= TreeChunk<Metablock, FreeList<Metablock> >::min_size()) {
 849     return_block(new_block + word_size, unused);
 850   }
 851 
 852   return new_block;
 853 }
 854 
 855 void BlockFreelist::print_on(outputStream* st) const {
 856   dictionary()->print_free_lists(st);
 857 }
 858 
 859 // VirtualSpaceNode methods
 860 
 861 VirtualSpaceNode::~VirtualSpaceNode() {
 862   _rs.release();
 863 #ifdef ASSERT
 864   size_t word_size = sizeof(*this) / BytesPerWord;
 865   Copy::fill_to_words((HeapWord*) this, word_size, 0xf1f1f1f1);
 866 #endif
 867 }
 868 
 869 size_t VirtualSpaceNode::used_words_in_vs() const {
 870   return pointer_delta(top(), bottom(), sizeof(MetaWord));
 871 }
 872 
 873 // Space committed in the VirtualSpace
 874 size_t VirtualSpaceNode::capacity_words_in_vs() const {
 875   return pointer_delta(end(), bottom(), sizeof(MetaWord));
 876 }
 877 
 878 size_t VirtualSpaceNode::free_words_in_vs() const {
 879   return pointer_delta(end(), top(), sizeof(MetaWord));
 880 }
 881 
 882 // Allocates the chunk from the virtual space only.
 883 // This interface is also used internally for debugging.  Not all
 884 // chunks removed here are necessarily used for allocation.
 885 Metachunk* VirtualSpaceNode::take_from_committed(size_t chunk_word_size) {
 886   // Bottom of the new chunk
 887   MetaWord* chunk_limit = top();
 888   assert(chunk_limit != NULL, "Not safe to call this method");
 889 
 890   // The virtual spaces are always expanded by the
 891   // commit granularity to enforce the following condition.
 892   // Without this the is_available check will not work correctly.
 893   assert(_virtual_space.committed_size() == _virtual_space.actual_committed_size(),
 894       "The committed memory doesn't match the expanded memory.");
 895 
 896   if (!is_available(chunk_word_size)) {
 897     LogHandle(gc, metaspace, freelist) log;
 898     log.debug("VirtualSpaceNode::take_from_committed() not available " SIZE_FORMAT " words ", chunk_word_size);
 899     // Dump some information about the virtual space that is nearly full
 900     ResourceMark rm;
 901     print_on(log.debug_stream());
 902     return NULL;
 903   }
 904 
 905   // Take the space  (bump top on the current virtual space).
 906   inc_top(chunk_word_size);
 907 
 908   // Initialize the chunk
 909   Metachunk* result = ::new (chunk_limit) Metachunk(chunk_word_size, this);
 910   return result;
 911 }
 912 
 913 
 914 // Expand the virtual space (commit more of the reserved space)
 915 bool VirtualSpaceNode::expand_by(size_t min_words, size_t preferred_words) {
 916   size_t min_bytes = min_words * BytesPerWord;
 917   size_t preferred_bytes = preferred_words * BytesPerWord;
 918 
 919   size_t uncommitted = virtual_space()->reserved_size() - virtual_space()->actual_committed_size();
 920 
 921   if (uncommitted < min_bytes) {
 922     return false;
 923   }
 924 
 925   size_t commit = MIN2(preferred_bytes, uncommitted);
 926   bool result = virtual_space()->expand_by(commit, false);
 927 
 928   assert(result, "Failed to commit memory");
 929 
 930   return result;
 931 }
 932 
 933 Metachunk* VirtualSpaceNode::get_chunk_vs(size_t chunk_word_size) {
 934   assert_lock_strong(SpaceManager::expand_lock());
 935   Metachunk* result = take_from_committed(chunk_word_size);
 936   if (result != NULL) {
 937     inc_container_count();
 938   }
 939   return result;
 940 }
 941 
 942 bool VirtualSpaceNode::initialize() {
 943 
 944   if (!_rs.is_reserved()) {
 945     return false;
 946   }
 947 
 948   // These are necessary restriction to make sure that the virtual space always
 949   // grows in steps of Metaspace::commit_alignment(). If both base and size are
 950   // aligned only the middle alignment of the VirtualSpace is used.
 951   assert_is_ptr_aligned(_rs.base(), Metaspace::commit_alignment());
 952   assert_is_size_aligned(_rs.size(), Metaspace::commit_alignment());
 953 
 954   // ReservedSpaces marked as special will have the entire memory
 955   // pre-committed. Setting a committed size will make sure that
 956   // committed_size and actual_committed_size agrees.
 957   size_t pre_committed_size = _rs.special() ? _rs.size() : 0;
 958 
 959   bool result = virtual_space()->initialize_with_granularity(_rs, pre_committed_size,
 960                                             Metaspace::commit_alignment());
 961   if (result) {
 962     assert(virtual_space()->committed_size() == virtual_space()->actual_committed_size(),
 963         "Checking that the pre-committed memory was registered by the VirtualSpace");
 964 
 965     set_top((MetaWord*)virtual_space()->low());
 966     set_reserved(MemRegion((HeapWord*)_rs.base(),
 967                  (HeapWord*)(_rs.base() + _rs.size())));
 968 
 969     assert(reserved()->start() == (HeapWord*) _rs.base(),
 970            "Reserved start was not set properly " PTR_FORMAT
 971            " != " PTR_FORMAT, p2i(reserved()->start()), p2i(_rs.base()));
 972     assert(reserved()->word_size() == _rs.size() / BytesPerWord,
 973            "Reserved size was not set properly " SIZE_FORMAT
 974            " != " SIZE_FORMAT, reserved()->word_size(),
 975            _rs.size() / BytesPerWord);
 976   }
 977 
 978   return result;
 979 }
 980 
 981 void VirtualSpaceNode::print_on(outputStream* st) const {
 982   size_t used = used_words_in_vs();
 983   size_t capacity = capacity_words_in_vs();
 984   VirtualSpace* vs = virtual_space();
 985   st->print_cr("   space @ " PTR_FORMAT " " SIZE_FORMAT "K, " SIZE_FORMAT_W(3) "%% used "
 986            "[" PTR_FORMAT ", " PTR_FORMAT ", "
 987            PTR_FORMAT ", " PTR_FORMAT ")",
 988            p2i(vs), capacity / K,
 989            capacity == 0 ? 0 : used * 100 / capacity,
 990            p2i(bottom()), p2i(top()), p2i(end()),
 991            p2i(vs->high_boundary()));
 992 }
 993 
 994 #ifdef ASSERT
 995 void VirtualSpaceNode::mangle() {
 996   size_t word_size = capacity_words_in_vs();
 997   Copy::fill_to_words((HeapWord*) low(), word_size, 0xf1f1f1f1);
 998 }
 999 #endif // ASSERT
1000 
1001 // VirtualSpaceList methods
1002 // Space allocated from the VirtualSpace
1003 
1004 VirtualSpaceList::~VirtualSpaceList() {
1005   VirtualSpaceListIterator iter(virtual_space_list());
1006   while (iter.repeat()) {
1007     VirtualSpaceNode* vsl = iter.get_next();
1008     delete vsl;
1009   }
1010 }
1011 
1012 void VirtualSpaceList::inc_reserved_words(size_t v) {
1013   assert_lock_strong(SpaceManager::expand_lock());
1014   _reserved_words = _reserved_words + v;
1015 }
1016 void VirtualSpaceList::dec_reserved_words(size_t v) {
1017   assert_lock_strong(SpaceManager::expand_lock());
1018   _reserved_words = _reserved_words - v;
1019 }
1020 
1021 #define assert_committed_below_limit()                        \
1022   assert(MetaspaceAux::committed_bytes() <= MaxMetaspaceSize, \
1023          "Too much committed memory. Committed: " SIZE_FORMAT \
1024          " limit (MaxMetaspaceSize): " SIZE_FORMAT,           \
1025          MetaspaceAux::committed_bytes(), MaxMetaspaceSize);
1026 
1027 void VirtualSpaceList::inc_committed_words(size_t v) {
1028   assert_lock_strong(SpaceManager::expand_lock());
1029   _committed_words = _committed_words + v;
1030 
1031   assert_committed_below_limit();
1032 }
1033 void VirtualSpaceList::dec_committed_words(size_t v) {
1034   assert_lock_strong(SpaceManager::expand_lock());
1035   _committed_words = _committed_words - v;
1036 
1037   assert_committed_below_limit();
1038 }
1039 
1040 void VirtualSpaceList::inc_virtual_space_count() {
1041   assert_lock_strong(SpaceManager::expand_lock());
1042   _virtual_space_count++;
1043 }
1044 void VirtualSpaceList::dec_virtual_space_count() {
1045   assert_lock_strong(SpaceManager::expand_lock());
1046   _virtual_space_count--;
1047 }
1048 
1049 void ChunkManager::remove_chunk(Metachunk* chunk) {
1050   size_t word_size = chunk->word_size();
1051   ChunkIndex index = list_index(word_size);
1052   if (index != HumongousIndex) {
1053     free_chunks(index)->remove_chunk(chunk);
1054   } else {
1055     humongous_dictionary()->remove_chunk(chunk);
1056   }
1057 
1058   // Chunk is being removed from the chunks free list.
1059   dec_free_chunks_total(chunk->word_size());
1060 }
1061 
1062 // Walk the list of VirtualSpaceNodes and delete
1063 // nodes with a 0 container_count.  Remove Metachunks in
1064 // the node from their respective freelists.
1065 void VirtualSpaceList::purge(ChunkManager* chunk_manager) {
1066   assert(SafepointSynchronize::is_at_safepoint(), "must be called at safepoint for contains to work");
1067   assert_lock_strong(SpaceManager::expand_lock());
1068   // Don't use a VirtualSpaceListIterator because this
1069   // list is being changed and a straightforward use of an iterator is not safe.
1070   VirtualSpaceNode* purged_vsl = NULL;
1071   VirtualSpaceNode* prev_vsl = virtual_space_list();
1072   VirtualSpaceNode* next_vsl = prev_vsl;
1073   while (next_vsl != NULL) {
1074     VirtualSpaceNode* vsl = next_vsl;
1075     DEBUG_ONLY(vsl->verify_container_count();)
1076     next_vsl = vsl->next();
1077     // Don't free the current virtual space since it will likely
1078     // be needed soon.
1079     if (vsl->container_count() == 0 && vsl != current_virtual_space()) {
1080       // Unlink it from the list
1081       if (prev_vsl == vsl) {
1082         // This is the case of the current node being the first node.
1083         assert(vsl == virtual_space_list(), "Expected to be the first node");
1084         set_virtual_space_list(vsl->next());
1085       } else {
1086         prev_vsl->set_next(vsl->next());
1087       }
1088 
1089       vsl->purge(chunk_manager);
1090       dec_reserved_words(vsl->reserved_words());
1091       dec_committed_words(vsl->committed_words());
1092       dec_virtual_space_count();
1093       purged_vsl = vsl;
1094       delete vsl;
1095     } else {
1096       prev_vsl = vsl;
1097     }
1098   }
1099 #ifdef ASSERT
1100   if (purged_vsl != NULL) {
1101     // List should be stable enough to use an iterator here.
1102     VirtualSpaceListIterator iter(virtual_space_list());
1103     while (iter.repeat()) {
1104       VirtualSpaceNode* vsl = iter.get_next();
1105       assert(vsl != purged_vsl, "Purge of vsl failed");
1106     }
1107   }
1108 #endif
1109 }
1110 
1111 
1112 // This function looks at the mmap regions in the metaspace without locking.
1113 // The chunks are added with store ordering and not deleted except for at
1114 // unloading time during a safepoint.
1115 bool VirtualSpaceList::contains(const void* ptr) {
1116   // List should be stable enough to use an iterator here because removing virtual
1117   // space nodes is only allowed at a safepoint.
1118   VirtualSpaceListIterator iter(virtual_space_list());
1119   while (iter.repeat()) {
1120     VirtualSpaceNode* vsn = iter.get_next();
1121     if (vsn->contains(ptr)) {
1122       return true;
1123     }
1124   }
1125   return false;
1126 }
1127 
1128 void VirtualSpaceList::retire_current_virtual_space() {
1129   assert_lock_strong(SpaceManager::expand_lock());
1130 
1131   VirtualSpaceNode* vsn = current_virtual_space();
1132 
1133   ChunkManager* cm = is_class() ? Metaspace::chunk_manager_class() :
1134                                   Metaspace::chunk_manager_metadata();
1135 
1136   vsn->retire(cm);
1137 }
1138 
1139 void VirtualSpaceNode::retire(ChunkManager* chunk_manager) {
1140   DEBUG_ONLY(verify_container_count();)
1141   for (int i = (int)MediumIndex; i >= (int)ZeroIndex; --i) {
1142     ChunkIndex index = (ChunkIndex)i;
1143     size_t chunk_size = chunk_manager->free_chunks(index)->size();
1144 
1145     while (free_words_in_vs() >= chunk_size) {
1146       Metachunk* chunk = get_chunk_vs(chunk_size);
1147       assert(chunk != NULL, "allocation should have been successful");
1148 
1149       chunk_manager->return_chunks(index, chunk);
1150       chunk_manager->inc_free_chunks_total(chunk_size);
1151     }
1152     DEBUG_ONLY(verify_container_count();)
1153   }
1154   assert(free_words_in_vs() == 0, "should be empty now");
1155 }
1156 
1157 VirtualSpaceList::VirtualSpaceList(size_t word_size) :
1158                                    _is_class(false),
1159                                    _virtual_space_list(NULL),
1160                                    _current_virtual_space(NULL),
1161                                    _reserved_words(0),
1162                                    _committed_words(0),
1163                                    _virtual_space_count(0) {
1164   MutexLockerEx cl(SpaceManager::expand_lock(),
1165                    Mutex::_no_safepoint_check_flag);
1166   create_new_virtual_space(word_size);
1167 }
1168 
1169 VirtualSpaceList::VirtualSpaceList(ReservedSpace rs) :
1170                                    _is_class(true),
1171                                    _virtual_space_list(NULL),
1172                                    _current_virtual_space(NULL),
1173                                    _reserved_words(0),
1174                                    _committed_words(0),
1175                                    _virtual_space_count(0) {
1176   MutexLockerEx cl(SpaceManager::expand_lock(),
1177                    Mutex::_no_safepoint_check_flag);
1178   VirtualSpaceNode* class_entry = new VirtualSpaceNode(rs);
1179   bool succeeded = class_entry->initialize();
1180   if (succeeded) {
1181     link_vs(class_entry);
1182   }
1183 }
1184 
1185 size_t VirtualSpaceList::free_bytes() {
1186   return virtual_space_list()->free_words_in_vs() * BytesPerWord;
1187 }
1188 
1189 // Allocate another meta virtual space and add it to the list.
1190 bool VirtualSpaceList::create_new_virtual_space(size_t vs_word_size) {
1191   assert_lock_strong(SpaceManager::expand_lock());
1192 
1193   if (is_class()) {
1194     assert(false, "We currently don't support more than one VirtualSpace for"
1195                   " the compressed class space. The initialization of the"
1196                   " CCS uses another code path and should not hit this path.");
1197     return false;
1198   }
1199 
1200   if (vs_word_size == 0) {
1201     assert(false, "vs_word_size should always be at least _reserve_alignment large.");
1202     return false;
1203   }
1204 
1205   // Reserve the space
1206   size_t vs_byte_size = vs_word_size * BytesPerWord;
1207   assert_is_size_aligned(vs_byte_size, Metaspace::reserve_alignment());
1208 
1209   // Allocate the meta virtual space and initialize it.
1210   VirtualSpaceNode* new_entry = new VirtualSpaceNode(vs_byte_size);
1211   if (!new_entry->initialize()) {
1212     delete new_entry;
1213     return false;
1214   } else {
1215     assert(new_entry->reserved_words() == vs_word_size,
1216         "Reserved memory size differs from requested memory size");
1217     // ensure lock-free iteration sees fully initialized node
1218     OrderAccess::storestore();
1219     link_vs(new_entry);
1220     return true;
1221   }
1222 }
1223 
1224 void VirtualSpaceList::link_vs(VirtualSpaceNode* new_entry) {
1225   if (virtual_space_list() == NULL) {
1226       set_virtual_space_list(new_entry);
1227   } else {
1228     current_virtual_space()->set_next(new_entry);
1229   }
1230   set_current_virtual_space(new_entry);
1231   inc_reserved_words(new_entry->reserved_words());
1232   inc_committed_words(new_entry->committed_words());
1233   inc_virtual_space_count();
1234 #ifdef ASSERT
1235   new_entry->mangle();
1236 #endif
1237   if (log_is_enabled(Trace, gc, metaspace)) {
1238     LogHandle(gc, metaspace) log;
1239     VirtualSpaceNode* vsl = current_virtual_space();
1240     ResourceMark rm;
1241     vsl->print_on(log.trace_stream());
1242   }
1243 }
1244 
1245 bool VirtualSpaceList::expand_node_by(VirtualSpaceNode* node,
1246                                       size_t min_words,
1247                                       size_t preferred_words) {
1248   size_t before = node->committed_words();
1249 
1250   bool result = node->expand_by(min_words, preferred_words);
1251 
1252   size_t after = node->committed_words();
1253 
1254   // after and before can be the same if the memory was pre-committed.
1255   assert(after >= before, "Inconsistency");
1256   inc_committed_words(after - before);
1257 
1258   return result;
1259 }
1260 
1261 bool VirtualSpaceList::expand_by(size_t min_words, size_t preferred_words) {
1262   assert_is_size_aligned(min_words,       Metaspace::commit_alignment_words());
1263   assert_is_size_aligned(preferred_words, Metaspace::commit_alignment_words());
1264   assert(min_words <= preferred_words, "Invalid arguments");
1265 
1266   if (!MetaspaceGC::can_expand(min_words, this->is_class())) {
1267     return  false;
1268   }
1269 
1270   size_t allowed_expansion_words = MetaspaceGC::allowed_expansion();
1271   if (allowed_expansion_words < min_words) {
1272     return false;
1273   }
1274 
1275   size_t max_expansion_words = MIN2(preferred_words, allowed_expansion_words);
1276 
1277   // Commit more memory from the the current virtual space.
1278   bool vs_expanded = expand_node_by(current_virtual_space(),
1279                                     min_words,
1280                                     max_expansion_words);
1281   if (vs_expanded) {
1282     return true;
1283   }
1284   retire_current_virtual_space();
1285 
1286   // Get another virtual space.
1287   size_t grow_vs_words = MAX2((size_t)VirtualSpaceSize, preferred_words);
1288   grow_vs_words = align_size_up(grow_vs_words, Metaspace::reserve_alignment_words());
1289 
1290   if (create_new_virtual_space(grow_vs_words)) {
1291     if (current_virtual_space()->is_pre_committed()) {
1292       // The memory was pre-committed, so we are done here.
1293       assert(min_words <= current_virtual_space()->committed_words(),
1294           "The new VirtualSpace was pre-committed, so it"
1295           "should be large enough to fit the alloc request.");
1296       return true;
1297     }
1298 
1299     return expand_node_by(current_virtual_space(),
1300                           min_words,
1301                           max_expansion_words);
1302   }
1303 
1304   return false;
1305 }
1306 
1307 Metachunk* VirtualSpaceList::get_new_chunk(size_t word_size,
1308                                            size_t grow_chunks_by_words,
1309                                            size_t medium_chunk_bunch) {
1310 
1311   // Allocate a chunk out of the current virtual space.
1312   Metachunk* next = current_virtual_space()->get_chunk_vs(grow_chunks_by_words);
1313 
1314   if (next != NULL) {
1315     return next;
1316   }
1317 
1318   // The expand amount is currently only determined by the requested sizes
1319   // and not how much committed memory is left in the current virtual space.
1320 
1321   size_t min_word_size       = align_size_up(grow_chunks_by_words, Metaspace::commit_alignment_words());
1322   size_t preferred_word_size = align_size_up(medium_chunk_bunch,   Metaspace::commit_alignment_words());
1323   if (min_word_size >= preferred_word_size) {
1324     // Can happen when humongous chunks are allocated.
1325     preferred_word_size = min_word_size;
1326   }
1327 
1328   bool expanded = expand_by(min_word_size, preferred_word_size);
1329   if (expanded) {
1330     next = current_virtual_space()->get_chunk_vs(grow_chunks_by_words);
1331     assert(next != NULL, "The allocation was expected to succeed after the expansion");
1332   }
1333 
1334    return next;
1335 }
1336 
1337 void VirtualSpaceList::print_on(outputStream* st) const {
1338   VirtualSpaceListIterator iter(virtual_space_list());
1339   while (iter.repeat()) {
1340     VirtualSpaceNode* node = iter.get_next();
1341     node->print_on(st);
1342   }
1343 }
1344 
1345 // MetaspaceGC methods
1346 
1347 // VM_CollectForMetadataAllocation is the vm operation used to GC.
1348 // Within the VM operation after the GC the attempt to allocate the metadata
1349 // should succeed.  If the GC did not free enough space for the metaspace
1350 // allocation, the HWM is increased so that another virtualspace will be
1351 // allocated for the metadata.  With perm gen the increase in the perm
1352 // gen had bounds, MinMetaspaceExpansion and MaxMetaspaceExpansion.  The
1353 // metaspace policy uses those as the small and large steps for the HWM.
1354 //
1355 // After the GC the compute_new_size() for MetaspaceGC is called to
1356 // resize the capacity of the metaspaces.  The current implementation
1357 // is based on the flags MinMetaspaceFreeRatio and MaxMetaspaceFreeRatio used
1358 // to resize the Java heap by some GC's.  New flags can be implemented
1359 // if really needed.  MinMetaspaceFreeRatio is used to calculate how much
1360 // free space is desirable in the metaspace capacity to decide how much
1361 // to increase the HWM.  MaxMetaspaceFreeRatio is used to decide how much
1362 // free space is desirable in the metaspace capacity before decreasing
1363 // the HWM.
1364 
1365 // Calculate the amount to increase the high water mark (HWM).
1366 // Increase by a minimum amount (MinMetaspaceExpansion) so that
1367 // another expansion is not requested too soon.  If that is not
1368 // enough to satisfy the allocation, increase by MaxMetaspaceExpansion.
1369 // If that is still not enough, expand by the size of the allocation
1370 // plus some.
1371 size_t MetaspaceGC::delta_capacity_until_GC(size_t bytes) {
1372   size_t min_delta = MinMetaspaceExpansion;
1373   size_t max_delta = MaxMetaspaceExpansion;
1374   size_t delta = align_size_up(bytes, Metaspace::commit_alignment());
1375 
1376   if (delta <= min_delta) {
1377     delta = min_delta;
1378   } else if (delta <= max_delta) {
1379     // Don't want to hit the high water mark on the next
1380     // allocation so make the delta greater than just enough
1381     // for this allocation.
1382     delta = max_delta;
1383   } else {
1384     // This allocation is large but the next ones are probably not
1385     // so increase by the minimum.
1386     delta = delta + min_delta;
1387   }
1388 
1389   assert_is_size_aligned(delta, Metaspace::commit_alignment());
1390 
1391   return delta;
1392 }
1393 
1394 size_t MetaspaceGC::capacity_until_GC() {
1395   size_t value = (size_t)OrderAccess::load_ptr_acquire(&_capacity_until_GC);
1396   assert(value >= MetaspaceSize, "Not initialized properly?");
1397   return value;
1398 }
1399 
1400 bool MetaspaceGC::inc_capacity_until_GC(size_t v, size_t* new_cap_until_GC, size_t* old_cap_until_GC) {
1401   assert_is_size_aligned(v, Metaspace::commit_alignment());
1402 
1403   size_t capacity_until_GC = (size_t) _capacity_until_GC;
1404   size_t new_value = capacity_until_GC + v;
1405 
1406   if (new_value < capacity_until_GC) {
1407     // The addition wrapped around, set new_value to aligned max value.
1408     new_value = align_size_down(max_uintx, Metaspace::commit_alignment());
1409   }
1410 
1411   intptr_t expected = (intptr_t) capacity_until_GC;
1412   intptr_t actual = Atomic::cmpxchg_ptr((intptr_t) new_value, &_capacity_until_GC, expected);
1413 
1414   if (expected != actual) {
1415     return false;
1416   }
1417 
1418   if (new_cap_until_GC != NULL) {
1419     *new_cap_until_GC = new_value;
1420   }
1421   if (old_cap_until_GC != NULL) {
1422     *old_cap_until_GC = capacity_until_GC;
1423   }
1424   return true;
1425 }
1426 
1427 size_t MetaspaceGC::dec_capacity_until_GC(size_t v) {
1428   assert_is_size_aligned(v, Metaspace::commit_alignment());
1429 
1430   return (size_t)Atomic::add_ptr(-(intptr_t)v, &_capacity_until_GC);
1431 }
1432 
1433 void MetaspaceGC::initialize() {
1434   // Set the high-water mark to MaxMetapaceSize during VM initializaton since
1435   // we can't do a GC during initialization.
1436   _capacity_until_GC = MaxMetaspaceSize;
1437 }
1438 
1439 void MetaspaceGC::post_initialize() {
1440   // Reset the high-water mark once the VM initialization is done.
1441   _capacity_until_GC = MAX2(MetaspaceAux::committed_bytes(), MetaspaceSize);
1442 }
1443 
1444 bool MetaspaceGC::can_expand(size_t word_size, bool is_class) {
1445   // Check if the compressed class space is full.
1446   if (is_class && Metaspace::using_class_space()) {
1447     size_t class_committed = MetaspaceAux::committed_bytes(Metaspace::ClassType);
1448     if (class_committed + word_size * BytesPerWord > CompressedClassSpaceSize) {
1449       return false;
1450     }
1451   }
1452 
1453   // Check if the user has imposed a limit on the metaspace memory.
1454   size_t committed_bytes = MetaspaceAux::committed_bytes();
1455   if (committed_bytes + word_size * BytesPerWord > MaxMetaspaceSize) {
1456     return false;
1457   }
1458 
1459   return true;
1460 }
1461 
1462 size_t MetaspaceGC::allowed_expansion() {
1463   size_t committed_bytes = MetaspaceAux::committed_bytes();
1464   size_t capacity_until_gc = capacity_until_GC();
1465 
1466   assert(capacity_until_gc >= committed_bytes,
1467          "capacity_until_gc: " SIZE_FORMAT " < committed_bytes: " SIZE_FORMAT,
1468          capacity_until_gc, committed_bytes);
1469 
1470   size_t left_until_max  = MaxMetaspaceSize - committed_bytes;
1471   size_t left_until_GC = capacity_until_gc - committed_bytes;
1472   size_t left_to_commit = MIN2(left_until_GC, left_until_max);
1473 
1474   return left_to_commit / BytesPerWord;
1475 }
1476 
1477 void MetaspaceGC::compute_new_size() {
1478   assert(_shrink_factor <= 100, "invalid shrink factor");
1479   uint current_shrink_factor = _shrink_factor;
1480   _shrink_factor = 0;
1481 
1482   // Using committed_bytes() for used_after_gc is an overestimation, since the
1483   // chunk free lists are included in committed_bytes() and the memory in an
1484   // un-fragmented chunk free list is available for future allocations.
1485   // However, if the chunk free lists becomes fragmented, then the memory may
1486   // not be available for future allocations and the memory is therefore "in use".
1487   // Including the chunk free lists in the definition of "in use" is therefore
1488   // necessary. Not including the chunk free lists can cause capacity_until_GC to
1489   // shrink below committed_bytes() and this has caused serious bugs in the past.
1490   const size_t used_after_gc = MetaspaceAux::committed_bytes();
1491   const size_t capacity_until_GC = MetaspaceGC::capacity_until_GC();
1492 
1493   const double minimum_free_percentage = MinMetaspaceFreeRatio / 100.0;
1494   const double maximum_used_percentage = 1.0 - minimum_free_percentage;
1495 
1496   const double min_tmp = used_after_gc / maximum_used_percentage;
1497   size_t minimum_desired_capacity =
1498     (size_t)MIN2(min_tmp, double(max_uintx));
1499   // Don't shrink less than the initial generation size
1500   minimum_desired_capacity = MAX2(minimum_desired_capacity,
1501                                   MetaspaceSize);
1502 
1503   log_trace(gc, metaspace)("MetaspaceGC::compute_new_size: ");
1504   log_trace(gc, metaspace)("    minimum_free_percentage: %6.2f  maximum_used_percentage: %6.2f",
1505                            minimum_free_percentage, maximum_used_percentage);
1506   log_trace(gc, metaspace)("     used_after_gc       : %6.1fKB", used_after_gc / (double) K);
1507 
1508 
1509   size_t shrink_bytes = 0;
1510   if (capacity_until_GC < minimum_desired_capacity) {
1511     // If we have less capacity below the metaspace HWM, then
1512     // increment the HWM.
1513     size_t expand_bytes = minimum_desired_capacity - capacity_until_GC;
1514     expand_bytes = align_size_up(expand_bytes, Metaspace::commit_alignment());
1515     // Don't expand unless it's significant
1516     if (expand_bytes >= MinMetaspaceExpansion) {
1517       size_t new_capacity_until_GC = 0;
1518       bool succeeded = MetaspaceGC::inc_capacity_until_GC(expand_bytes, &new_capacity_until_GC);
1519       assert(succeeded, "Should always succesfully increment HWM when at safepoint");
1520 
1521       Metaspace::tracer()->report_gc_threshold(capacity_until_GC,
1522                                                new_capacity_until_GC,
1523                                                MetaspaceGCThresholdUpdater::ComputeNewSize);
1524       log_trace(gc, metaspace)("    expanding:  minimum_desired_capacity: %6.1fKB  expand_bytes: %6.1fKB  MinMetaspaceExpansion: %6.1fKB  new metaspace HWM:  %6.1fKB",
1525                                minimum_desired_capacity / (double) K,
1526                                expand_bytes / (double) K,
1527                                MinMetaspaceExpansion / (double) K,
1528                                new_capacity_until_GC / (double) K);
1529     }
1530     return;
1531   }
1532 
1533   // No expansion, now see if we want to shrink
1534   // We would never want to shrink more than this
1535   assert(capacity_until_GC >= minimum_desired_capacity,
1536          SIZE_FORMAT " >= " SIZE_FORMAT,
1537          capacity_until_GC, minimum_desired_capacity);
1538   size_t max_shrink_bytes = capacity_until_GC - minimum_desired_capacity;
1539 
1540   // Should shrinking be considered?
1541   if (MaxMetaspaceFreeRatio < 100) {
1542     const double maximum_free_percentage = MaxMetaspaceFreeRatio / 100.0;
1543     const double minimum_used_percentage = 1.0 - maximum_free_percentage;
1544     const double max_tmp = used_after_gc / minimum_used_percentage;
1545     size_t maximum_desired_capacity = (size_t)MIN2(max_tmp, double(max_uintx));
1546     maximum_desired_capacity = MAX2(maximum_desired_capacity,
1547                                     MetaspaceSize);
1548     log_trace(gc, metaspace)("    maximum_free_percentage: %6.2f  minimum_used_percentage: %6.2f",
1549                              maximum_free_percentage, minimum_used_percentage);
1550     log_trace(gc, metaspace)("    minimum_desired_capacity: %6.1fKB  maximum_desired_capacity: %6.1fKB",
1551                              minimum_desired_capacity / (double) K, maximum_desired_capacity / (double) K);
1552 
1553     assert(minimum_desired_capacity <= maximum_desired_capacity,
1554            "sanity check");
1555 
1556     if (capacity_until_GC > maximum_desired_capacity) {
1557       // Capacity too large, compute shrinking size
1558       shrink_bytes = capacity_until_GC - maximum_desired_capacity;
1559       // We don't want shrink all the way back to initSize if people call
1560       // System.gc(), because some programs do that between "phases" and then
1561       // we'd just have to grow the heap up again for the next phase.  So we
1562       // damp the shrinking: 0% on the first call, 10% on the second call, 40%
1563       // on the third call, and 100% by the fourth call.  But if we recompute
1564       // size without shrinking, it goes back to 0%.
1565       shrink_bytes = shrink_bytes / 100 * current_shrink_factor;
1566 
1567       shrink_bytes = align_size_down(shrink_bytes, Metaspace::commit_alignment());
1568 
1569       assert(shrink_bytes <= max_shrink_bytes,
1570              "invalid shrink size " SIZE_FORMAT " not <= " SIZE_FORMAT,
1571              shrink_bytes, max_shrink_bytes);
1572       if (current_shrink_factor == 0) {
1573         _shrink_factor = 10;
1574       } else {
1575         _shrink_factor = MIN2(current_shrink_factor * 4, (uint) 100);
1576       }
1577       log_trace(gc, metaspace)("    shrinking:  initSize: %.1fK  maximum_desired_capacity: %.1fK",
1578                                MetaspaceSize / (double) K, maximum_desired_capacity / (double) K);
1579       log_trace(gc, metaspace)("    shrink_bytes: %.1fK  current_shrink_factor: %d  new shrink factor: %d  MinMetaspaceExpansion: %.1fK",
1580                                shrink_bytes / (double) K, current_shrink_factor, _shrink_factor, MinMetaspaceExpansion / (double) K);
1581     }
1582   }
1583 
1584   // Don't shrink unless it's significant
1585   if (shrink_bytes >= MinMetaspaceExpansion &&
1586       ((capacity_until_GC - shrink_bytes) >= MetaspaceSize)) {
1587     size_t new_capacity_until_GC = MetaspaceGC::dec_capacity_until_GC(shrink_bytes);
1588     Metaspace::tracer()->report_gc_threshold(capacity_until_GC,
1589                                              new_capacity_until_GC,
1590                                              MetaspaceGCThresholdUpdater::ComputeNewSize);
1591   }
1592 }
1593 
1594 // Metadebug methods
1595 
1596 void Metadebug::init_allocation_fail_alot_count() {
1597   if (MetadataAllocationFailALot) {
1598     _allocation_fail_alot_count =
1599       1+(long)((double)MetadataAllocationFailALotInterval*os::random()/(max_jint+1.0));
1600   }
1601 }
1602 
1603 #ifdef ASSERT
1604 bool Metadebug::test_metadata_failure() {
1605   if (MetadataAllocationFailALot &&
1606       Threads::is_vm_complete()) {
1607     if (_allocation_fail_alot_count > 0) {
1608       _allocation_fail_alot_count--;
1609     } else {
1610       log_trace(gc, metaspace, freelist)("Metadata allocation failing for MetadataAllocationFailALot");
1611       init_allocation_fail_alot_count();
1612       return true;
1613     }
1614   }
1615   return false;
1616 }
1617 #endif
1618 
1619 // ChunkManager methods
1620 
1621 size_t ChunkManager::free_chunks_total_words() {
1622   return _free_chunks_total;
1623 }
1624 
1625 size_t ChunkManager::free_chunks_total_bytes() {
1626   return free_chunks_total_words() * BytesPerWord;
1627 }
1628 
1629 size_t ChunkManager::free_chunks_count() {
1630 #ifdef ASSERT
1631   if (!UseConcMarkSweepGC && !SpaceManager::expand_lock()->is_locked()) {
1632     MutexLockerEx cl(SpaceManager::expand_lock(),
1633                      Mutex::_no_safepoint_check_flag);
1634     // This lock is only needed in debug because the verification
1635     // of the _free_chunks_totals walks the list of free chunks
1636     slow_locked_verify_free_chunks_count();
1637   }
1638 #endif
1639   return _free_chunks_count;
1640 }
1641 
1642 void ChunkManager::locked_verify_free_chunks_total() {
1643   assert_lock_strong(SpaceManager::expand_lock());
1644   assert(sum_free_chunks() == _free_chunks_total,
1645          "_free_chunks_total " SIZE_FORMAT " is not the"
1646          " same as sum " SIZE_FORMAT, _free_chunks_total,
1647          sum_free_chunks());
1648 }
1649 
1650 void ChunkManager::verify_free_chunks_total() {
1651   MutexLockerEx cl(SpaceManager::expand_lock(),
1652                      Mutex::_no_safepoint_check_flag);
1653   locked_verify_free_chunks_total();
1654 }
1655 
1656 void ChunkManager::locked_verify_free_chunks_count() {
1657   assert_lock_strong(SpaceManager::expand_lock());
1658   assert(sum_free_chunks_count() == _free_chunks_count,
1659          "_free_chunks_count " SIZE_FORMAT " is not the"
1660          " same as sum " SIZE_FORMAT, _free_chunks_count,
1661          sum_free_chunks_count());
1662 }
1663 
1664 void ChunkManager::verify_free_chunks_count() {
1665 #ifdef ASSERT
1666   MutexLockerEx cl(SpaceManager::expand_lock(),
1667                      Mutex::_no_safepoint_check_flag);
1668   locked_verify_free_chunks_count();
1669 #endif
1670 }
1671 
1672 void ChunkManager::verify() {
1673   MutexLockerEx cl(SpaceManager::expand_lock(),
1674                      Mutex::_no_safepoint_check_flag);
1675   locked_verify();
1676 }
1677 
1678 void ChunkManager::locked_verify() {
1679   locked_verify_free_chunks_count();
1680   locked_verify_free_chunks_total();
1681 }
1682 
1683 void ChunkManager::locked_print_free_chunks(outputStream* st) {
1684   assert_lock_strong(SpaceManager::expand_lock());
1685   st->print_cr("Free chunk total " SIZE_FORMAT "  count " SIZE_FORMAT,
1686                 _free_chunks_total, _free_chunks_count);
1687 }
1688 
1689 void ChunkManager::locked_print_sum_free_chunks(outputStream* st) {
1690   assert_lock_strong(SpaceManager::expand_lock());
1691   st->print_cr("Sum free chunk total " SIZE_FORMAT "  count " SIZE_FORMAT,
1692                 sum_free_chunks(), sum_free_chunks_count());
1693 }
1694 ChunkList* ChunkManager::free_chunks(ChunkIndex index) {
1695   return &_free_chunks[index];
1696 }
1697 
1698 // These methods that sum the free chunk lists are used in printing
1699 // methods that are used in product builds.
1700 size_t ChunkManager::sum_free_chunks() {
1701   assert_lock_strong(SpaceManager::expand_lock());
1702   size_t result = 0;
1703   for (ChunkIndex i = ZeroIndex; i < NumberOfFreeLists; i = next_chunk_index(i)) {
1704     ChunkList* list = free_chunks(i);
1705 
1706     if (list == NULL) {
1707       continue;
1708     }
1709 
1710     result = result + list->count() * list->size();
1711   }
1712   result = result + humongous_dictionary()->total_size();
1713   return result;
1714 }
1715 
1716 size_t ChunkManager::sum_free_chunks_count() {
1717   assert_lock_strong(SpaceManager::expand_lock());
1718   size_t count = 0;
1719   for (ChunkIndex i = ZeroIndex; i < NumberOfFreeLists; i = next_chunk_index(i)) {
1720     ChunkList* list = free_chunks(i);
1721     if (list == NULL) {
1722       continue;
1723     }
1724     count = count + list->count();
1725   }
1726   count = count + humongous_dictionary()->total_free_blocks();
1727   return count;
1728 }
1729 
1730 ChunkList* ChunkManager::find_free_chunks_list(size_t word_size) {
1731   ChunkIndex index = list_index(word_size);
1732   assert(index < HumongousIndex, "No humongous list");
1733   return free_chunks(index);
1734 }
1735 
1736 Metachunk* ChunkManager::free_chunks_get(size_t word_size) {
1737   assert_lock_strong(SpaceManager::expand_lock());
1738 
1739   slow_locked_verify();
1740 
1741   Metachunk* chunk = NULL;
1742   if (list_index(word_size) != HumongousIndex) {
1743     ChunkList* free_list = find_free_chunks_list(word_size);
1744     assert(free_list != NULL, "Sanity check");
1745 
1746     chunk = free_list->head();
1747 
1748     if (chunk == NULL) {
1749       return NULL;
1750     }
1751 
1752     // Remove the chunk as the head of the list.
1753     free_list->remove_chunk(chunk);
1754 
1755     log_trace(gc, metaspace, freelist)("ChunkManager::free_chunks_get: free_list " PTR_FORMAT " head " PTR_FORMAT " size " SIZE_FORMAT,
1756                                        p2i(free_list), p2i(chunk), chunk->word_size());
1757   } else {
1758     chunk = humongous_dictionary()->get_chunk(
1759       word_size,
1760       FreeBlockDictionary<Metachunk>::atLeast);
1761 
1762     if (chunk == NULL) {
1763       return NULL;
1764     }
1765 
1766     log_debug(gc, metaspace, alloc)("Free list allocate humongous chunk size " SIZE_FORMAT " for requested size " SIZE_FORMAT " waste " SIZE_FORMAT,
1767                                     chunk->word_size(), word_size, chunk->word_size() - word_size);
1768   }
1769 
1770   // Chunk is being removed from the chunks free list.
1771   dec_free_chunks_total(chunk->word_size());
1772 
1773   // Remove it from the links to this freelist
1774   chunk->set_next(NULL);
1775   chunk->set_prev(NULL);
1776 #ifdef ASSERT
1777   // Chunk is no longer on any freelist. Setting to false make container_count_slow()
1778   // work.
1779   chunk->set_is_tagged_free(false);
1780 #endif
1781   chunk->container()->inc_container_count();
1782 
1783   slow_locked_verify();
1784   return chunk;
1785 }
1786 
1787 Metachunk* ChunkManager::chunk_freelist_allocate(size_t word_size) {
1788   assert_lock_strong(SpaceManager::expand_lock());
1789   slow_locked_verify();
1790 
1791   // Take from the beginning of the list
1792   Metachunk* chunk = free_chunks_get(word_size);
1793   if (chunk == NULL) {
1794     return NULL;
1795   }
1796 
1797   assert((word_size <= chunk->word_size()) ||
1798          list_index(chunk->word_size() == HumongousIndex),
1799          "Non-humongous variable sized chunk");
1800   LogHandle(gc, metaspace, freelist) log;
1801   if (log.is_debug()) {
1802     size_t list_count;
1803     if (list_index(word_size) < HumongousIndex) {
1804       ChunkList* list = find_free_chunks_list(word_size);
1805       list_count = list->count();
1806     } else {
1807       list_count = humongous_dictionary()->total_count();
1808     }
1809     log.debug("ChunkManager::chunk_freelist_allocate: " PTR_FORMAT " chunk " PTR_FORMAT "  size " SIZE_FORMAT " count " SIZE_FORMAT " ",
1810                p2i(this), p2i(chunk), chunk->word_size(), list_count);
1811     ResourceMark rm;
1812     locked_print_free_chunks(log.debug_stream());
1813   }
1814 
1815   return chunk;
1816 }
1817 
1818 void ChunkManager::print_on(outputStream* out) const {
1819   const_cast<ChunkManager *>(this)->humongous_dictionary()->report_statistics(out);
1820 }
1821 
1822 // SpaceManager methods
1823 
1824 void SpaceManager::get_initial_chunk_sizes(Metaspace::MetaspaceType type,
1825                                            size_t* chunk_word_size,
1826                                            size_t* class_chunk_word_size) {
1827   switch (type) {
1828   case Metaspace::BootMetaspaceType:
1829     *chunk_word_size = Metaspace::first_chunk_word_size();
1830     *class_chunk_word_size = Metaspace::first_class_chunk_word_size();
1831     break;
1832   case Metaspace::ROMetaspaceType:
1833     *chunk_word_size = SharedReadOnlySize / wordSize;
1834     *class_chunk_word_size = ClassSpecializedChunk;
1835     break;
1836   case Metaspace::ReadWriteMetaspaceType:
1837     *chunk_word_size = SharedReadWriteSize / wordSize;
1838     *class_chunk_word_size = ClassSpecializedChunk;
1839     break;
1840   case Metaspace::AnonymousMetaspaceType:
1841   case Metaspace::ReflectionMetaspaceType:
1842     *chunk_word_size = SpecializedChunk;
1843     *class_chunk_word_size = ClassSpecializedChunk;
1844     break;
1845   default:
1846     *chunk_word_size = SmallChunk;
1847     *class_chunk_word_size = ClassSmallChunk;
1848     break;
1849   }
1850   assert(*chunk_word_size != 0 && *class_chunk_word_size != 0,
1851          "Initial chunks sizes bad: data  " SIZE_FORMAT
1852          " class " SIZE_FORMAT,
1853          *chunk_word_size, *class_chunk_word_size);
1854 }
1855 
1856 size_t SpaceManager::sum_free_in_chunks_in_use() const {
1857   MutexLockerEx cl(lock(), Mutex::_no_safepoint_check_flag);
1858   size_t free = 0;
1859   for (ChunkIndex i = ZeroIndex; i < NumberOfInUseLists; i = next_chunk_index(i)) {
1860     Metachunk* chunk = chunks_in_use(i);
1861     while (chunk != NULL) {
1862       free += chunk->free_word_size();
1863       chunk = chunk->next();
1864     }
1865   }
1866   return free;
1867 }
1868 
1869 size_t SpaceManager::sum_waste_in_chunks_in_use() const {
1870   MutexLockerEx cl(lock(), Mutex::_no_safepoint_check_flag);
1871   size_t result = 0;
1872   for (ChunkIndex i = ZeroIndex; i < NumberOfInUseLists; i = next_chunk_index(i)) {
1873    result += sum_waste_in_chunks_in_use(i);
1874   }
1875 
1876   return result;
1877 }
1878 
1879 size_t SpaceManager::sum_waste_in_chunks_in_use(ChunkIndex index) const {
1880   size_t result = 0;
1881   Metachunk* chunk = chunks_in_use(index);
1882   // Count the free space in all the chunk but not the
1883   // current chunk from which allocations are still being done.
1884   while (chunk != NULL) {
1885     if (chunk != current_chunk()) {
1886       result += chunk->free_word_size();
1887     }
1888     chunk = chunk->next();
1889   }
1890   return result;
1891 }
1892 
1893 size_t SpaceManager::sum_capacity_in_chunks_in_use() const {
1894   // For CMS use "allocated_chunks_words()" which does not need the
1895   // Metaspace lock.  For the other collectors sum over the
1896   // lists.  Use both methods as a check that "allocated_chunks_words()"
1897   // is correct.  That is, sum_capacity_in_chunks() is too expensive
1898   // to use in the product and allocated_chunks_words() should be used
1899   // but allow for  checking that allocated_chunks_words() returns the same
1900   // value as sum_capacity_in_chunks_in_use() which is the definitive
1901   // answer.
1902   if (UseConcMarkSweepGC) {
1903     return allocated_chunks_words();
1904   } else {
1905     MutexLockerEx cl(lock(), Mutex::_no_safepoint_check_flag);
1906     size_t sum = 0;
1907     for (ChunkIndex i = ZeroIndex; i < NumberOfInUseLists; i = next_chunk_index(i)) {
1908       Metachunk* chunk = chunks_in_use(i);
1909       while (chunk != NULL) {
1910         sum += chunk->word_size();
1911         chunk = chunk->next();
1912       }
1913     }
1914   return sum;
1915   }
1916 }
1917 
1918 size_t SpaceManager::sum_count_in_chunks_in_use() {
1919   size_t count = 0;
1920   for (ChunkIndex i = ZeroIndex; i < NumberOfInUseLists; i = next_chunk_index(i)) {
1921     count = count + sum_count_in_chunks_in_use(i);
1922   }
1923 
1924   return count;
1925 }
1926 
1927 size_t SpaceManager::sum_count_in_chunks_in_use(ChunkIndex i) {
1928   size_t count = 0;
1929   Metachunk* chunk = chunks_in_use(i);
1930   while (chunk != NULL) {
1931     count++;
1932     chunk = chunk->next();
1933   }
1934   return count;
1935 }
1936 
1937 
1938 size_t SpaceManager::sum_used_in_chunks_in_use() const {
1939   MutexLockerEx cl(lock(), Mutex::_no_safepoint_check_flag);
1940   size_t used = 0;
1941   for (ChunkIndex i = ZeroIndex; i < NumberOfInUseLists; i = next_chunk_index(i)) {
1942     Metachunk* chunk = chunks_in_use(i);
1943     while (chunk != NULL) {
1944       used += chunk->used_word_size();
1945       chunk = chunk->next();
1946     }
1947   }
1948   return used;
1949 }
1950 
1951 void SpaceManager::locked_print_chunks_in_use_on(outputStream* st) const {
1952 
1953   for (ChunkIndex i = ZeroIndex; i < NumberOfInUseLists; i = next_chunk_index(i)) {
1954     Metachunk* chunk = chunks_in_use(i);
1955     st->print("SpaceManager: %s " PTR_FORMAT,
1956                  chunk_size_name(i), p2i(chunk));
1957     if (chunk != NULL) {
1958       st->print_cr(" free " SIZE_FORMAT,
1959                    chunk->free_word_size());
1960     } else {
1961       st->cr();
1962     }
1963   }
1964 
1965   chunk_manager()->locked_print_free_chunks(st);
1966   chunk_manager()->locked_print_sum_free_chunks(st);
1967 }
1968 
1969 size_t SpaceManager::calc_chunk_size(size_t word_size) {
1970 
1971   // Decide between a small chunk and a medium chunk.  Up to
1972   // _small_chunk_limit small chunks can be allocated.
1973   // After that a medium chunk is preferred.
1974   size_t chunk_word_size;
1975   if (chunks_in_use(MediumIndex) == NULL &&
1976       sum_count_in_chunks_in_use(SmallIndex) < _small_chunk_limit) {
1977     chunk_word_size = (size_t) small_chunk_size();
1978     if (word_size + Metachunk::overhead() > small_chunk_size()) {
1979       chunk_word_size = medium_chunk_size();
1980     }
1981   } else {
1982     chunk_word_size = medium_chunk_size();
1983   }
1984 
1985   // Might still need a humongous chunk.  Enforce
1986   // humongous allocations sizes to be aligned up to
1987   // the smallest chunk size.
1988   size_t if_humongous_sized_chunk =
1989     align_size_up(word_size + Metachunk::overhead(),
1990                   smallest_chunk_size());
1991   chunk_word_size =
1992     MAX2((size_t) chunk_word_size, if_humongous_sized_chunk);
1993 
1994   assert(!SpaceManager::is_humongous(word_size) ||
1995          chunk_word_size == if_humongous_sized_chunk,
1996          "Size calculation is wrong, word_size " SIZE_FORMAT
1997          " chunk_word_size " SIZE_FORMAT,
1998          word_size, chunk_word_size);
1999   LogHandle(gc, metaspace, alloc) log;
2000   if (log.is_debug() && SpaceManager::is_humongous(word_size)) {
2001     log.debug("Metadata humongous allocation:");
2002     log.debug("  word_size " PTR_FORMAT, word_size);
2003     log.debug("  chunk_word_size " PTR_FORMAT, chunk_word_size);
2004     log.debug("    chunk overhead " PTR_FORMAT, Metachunk::overhead());
2005   }
2006   return chunk_word_size;
2007 }
2008 
2009 void SpaceManager::track_metaspace_memory_usage() {
2010   if (is_init_completed()) {
2011     if (is_class()) {
2012       MemoryService::track_compressed_class_memory_usage();
2013     }
2014     MemoryService::track_metaspace_memory_usage();
2015   }
2016 }
2017 
2018 MetaWord* SpaceManager::grow_and_allocate(size_t word_size) {
2019   assert(vs_list()->current_virtual_space() != NULL,
2020          "Should have been set");
2021   assert(current_chunk() == NULL ||
2022          current_chunk()->allocate(word_size) == NULL,
2023          "Don't need to expand");
2024   MutexLockerEx cl(SpaceManager::expand_lock(), Mutex::_no_safepoint_check_flag);
2025 
2026   if (log_is_enabled(Trace, gc, metaspace, freelist)) {
2027     size_t words_left = 0;
2028     size_t words_used = 0;
2029     if (current_chunk() != NULL) {
2030       words_left = current_chunk()->free_word_size();
2031       words_used = current_chunk()->used_word_size();
2032     }
2033     log_trace(gc, metaspace, freelist)("SpaceManager::grow_and_allocate for " SIZE_FORMAT " words " SIZE_FORMAT " words used " SIZE_FORMAT " words left",
2034                                        word_size, words_used, words_left);
2035   }
2036 
2037   // Get another chunk
2038   size_t grow_chunks_by_words = calc_chunk_size(word_size);
2039   Metachunk* next = get_new_chunk(word_size, grow_chunks_by_words);
2040 
2041   MetaWord* mem = NULL;
2042 
2043   // If a chunk was available, add it to the in-use chunk list
2044   // and do an allocation from it.
2045   if (next != NULL) {
2046     // Add to this manager's list of chunks in use.
2047     add_chunk(next, false);
2048     mem = next->allocate(word_size);
2049   }
2050 
2051   // Track metaspace memory usage statistic.
2052   track_metaspace_memory_usage();
2053 
2054   return mem;
2055 }
2056 
2057 void SpaceManager::print_on(outputStream* st) const {
2058 
2059   for (ChunkIndex i = ZeroIndex;
2060        i < NumberOfInUseLists ;
2061        i = next_chunk_index(i) ) {
2062     st->print_cr("  chunks_in_use " PTR_FORMAT " chunk size " SIZE_FORMAT,
2063                  p2i(chunks_in_use(i)),
2064                  chunks_in_use(i) == NULL ? 0 : chunks_in_use(i)->word_size());
2065   }
2066   st->print_cr("    waste:  Small " SIZE_FORMAT " Medium " SIZE_FORMAT
2067                " Humongous " SIZE_FORMAT,
2068                sum_waste_in_chunks_in_use(SmallIndex),
2069                sum_waste_in_chunks_in_use(MediumIndex),
2070                sum_waste_in_chunks_in_use(HumongousIndex));
2071   // block free lists
2072   if (block_freelists() != NULL) {
2073     st->print_cr("total in block free lists " SIZE_FORMAT,
2074       block_freelists()->total_size());
2075   }
2076 }
2077 
2078 SpaceManager::SpaceManager(Metaspace::MetadataType mdtype,
2079                            Mutex* lock) :
2080   _mdtype(mdtype),
2081   _allocated_blocks_words(0),
2082   _allocated_chunks_words(0),
2083   _allocated_chunks_count(0),
2084   _lock(lock)
2085 {
2086   initialize();
2087 }
2088 
2089 void SpaceManager::inc_size_metrics(size_t words) {
2090   assert_lock_strong(SpaceManager::expand_lock());
2091   // Total of allocated Metachunks and allocated Metachunks count
2092   // for each SpaceManager
2093   _allocated_chunks_words = _allocated_chunks_words + words;
2094   _allocated_chunks_count++;
2095   // Global total of capacity in allocated Metachunks
2096   MetaspaceAux::inc_capacity(mdtype(), words);
2097   // Global total of allocated Metablocks.
2098   // used_words_slow() includes the overhead in each
2099   // Metachunk so include it in the used when the
2100   // Metachunk is first added (so only added once per
2101   // Metachunk).
2102   MetaspaceAux::inc_used(mdtype(), Metachunk::overhead());
2103 }
2104 
2105 void SpaceManager::inc_used_metrics(size_t words) {
2106   // Add to the per SpaceManager total
2107   Atomic::add_ptr(words, &_allocated_blocks_words);
2108   // Add to the global total
2109   MetaspaceAux::inc_used(mdtype(), words);
2110 }
2111 
2112 void SpaceManager::dec_total_from_size_metrics() {
2113   MetaspaceAux::dec_capacity(mdtype(), allocated_chunks_words());
2114   MetaspaceAux::dec_used(mdtype(), allocated_blocks_words());
2115   // Also deduct the overhead per Metachunk
2116   MetaspaceAux::dec_used(mdtype(), allocated_chunks_count() * Metachunk::overhead());
2117 }
2118 
2119 void SpaceManager::initialize() {
2120   Metadebug::init_allocation_fail_alot_count();
2121   for (ChunkIndex i = ZeroIndex; i < NumberOfInUseLists; i = next_chunk_index(i)) {
2122     _chunks_in_use[i] = NULL;
2123   }
2124   _current_chunk = NULL;
2125   log_trace(gc, metaspace, freelist)("SpaceManager(): " PTR_FORMAT, p2i(this));
2126 }
2127 
2128 void ChunkManager::return_chunks(ChunkIndex index, Metachunk* chunks) {
2129   if (chunks == NULL) {
2130     return;
2131   }
2132   ChunkList* list = free_chunks(index);
2133   assert(list->size() == chunks->word_size(), "Mismatch in chunk sizes");
2134   assert_lock_strong(SpaceManager::expand_lock());
2135   Metachunk* cur = chunks;
2136 
2137   // This returns chunks one at a time.  If a new
2138   // class List can be created that is a base class
2139   // of FreeList then something like FreeList::prepend()
2140   // can be used in place of this loop
2141   while (cur != NULL) {
2142     assert(cur->container() != NULL, "Container should have been set");
2143     cur->container()->dec_container_count();
2144     // Capture the next link before it is changed
2145     // by the call to return_chunk_at_head();
2146     Metachunk* next = cur->next();
2147     DEBUG_ONLY(cur->set_is_tagged_free(true);)
2148     NOT_PRODUCT(cur->mangle(badMetaWordVal);)
2149     list->return_chunk_at_head(cur);
2150     cur = next;
2151   }
2152 }
2153 
2154 SpaceManager::~SpaceManager() {
2155   // This call this->_lock which can't be done while holding expand_lock()
2156   assert(sum_capacity_in_chunks_in_use() == allocated_chunks_words(),
2157          "sum_capacity_in_chunks_in_use() " SIZE_FORMAT
2158          " allocated_chunks_words() " SIZE_FORMAT,
2159          sum_capacity_in_chunks_in_use(), allocated_chunks_words());
2160 
2161   MutexLockerEx fcl(SpaceManager::expand_lock(),
2162                     Mutex::_no_safepoint_check_flag);
2163 
2164   chunk_manager()->slow_locked_verify();
2165 
2166   dec_total_from_size_metrics();
2167 
2168   LogHandle(gc, metaspace, freelist) log;
2169   if (log.is_trace()) {
2170     log.trace("~SpaceManager(): " PTR_FORMAT, p2i(this));
2171     ResourceMark rm;
2172     locked_print_chunks_in_use_on(log.trace_stream());
2173   }
2174 



2175   // Have to update before the chunks_in_use lists are emptied
2176   // below.
2177   chunk_manager()->inc_free_chunks_total(allocated_chunks_words(),
2178                                          sum_count_in_chunks_in_use());
2179 
2180   // Add all the chunks in use by this space manager
2181   // to the global list of free chunks.
2182 
2183   // Follow each list of chunks-in-use and add them to the
2184   // free lists.  Each list is NULL terminated.
2185 
2186   for (ChunkIndex i = ZeroIndex; i < HumongousIndex; i = next_chunk_index(i)) {
2187     log.trace("returned " SIZE_FORMAT " %s chunks to freelist", sum_count_in_chunks_in_use(i), chunk_size_name(i));
2188     Metachunk* chunks = chunks_in_use(i);
2189     chunk_manager()->return_chunks(i, chunks);
2190     set_chunks_in_use(i, NULL);
2191     log.trace("updated freelist count " SSIZE_FORMAT " %s", chunk_manager()->free_chunks(i)->count(), chunk_size_name(i));
2192     assert(i != HumongousIndex, "Humongous chunks are handled explicitly later");
2193   }
2194 
2195   // The medium chunk case may be optimized by passing the head and
2196   // tail of the medium chunk list to add_at_head().  The tail is often
2197   // the current chunk but there are probably exceptions.
2198 
2199   // Humongous chunks
2200   log.trace("returned " SIZE_FORMAT " %s humongous chunks to dictionary",
2201             sum_count_in_chunks_in_use(HumongousIndex), chunk_size_name(HumongousIndex));
2202   log.trace("Humongous chunk dictionary: ");
2203   // Humongous chunks are never the current chunk.
2204   Metachunk* humongous_chunks = chunks_in_use(HumongousIndex);
2205 
2206   while (humongous_chunks != NULL) {
2207     DEBUG_ONLY(humongous_chunks->set_is_tagged_free(true);)
2208     NOT_PRODUCT(humongous_chunks->mangle(badMetaWordVal);)

2209     log.trace(PTR_FORMAT " (" SIZE_FORMAT ") ", p2i(humongous_chunks), humongous_chunks->word_size());
2210     assert(humongous_chunks->word_size() == (size_t)
2211            align_size_up(humongous_chunks->word_size(),
2212                              smallest_chunk_size()),
2213            "Humongous chunk size is wrong: word size " SIZE_FORMAT
2214            " granularity " SIZE_FORMAT,
2215            humongous_chunks->word_size(), smallest_chunk_size());
2216     Metachunk* next_humongous_chunks = humongous_chunks->next();
2217     humongous_chunks->container()->dec_container_count();
2218     chunk_manager()->humongous_dictionary()->return_chunk(humongous_chunks);
2219     humongous_chunks = next_humongous_chunks;
2220   }
2221   log.trace("updated dictionary count " SIZE_FORMAT " %s", chunk_manager()->humongous_dictionary()->total_count(), chunk_size_name(HumongousIndex));
2222   chunk_manager()->slow_locked_verify();
2223 }
2224 
2225 const char* SpaceManager::chunk_size_name(ChunkIndex index) const {
2226   switch (index) {
2227     case SpecializedIndex:
2228       return "Specialized";
2229     case SmallIndex:
2230       return "Small";
2231     case MediumIndex:
2232       return "Medium";
2233     case HumongousIndex:
2234       return "Humongous";
2235     default:
2236       return NULL;
2237   }
2238 }
2239 
2240 ChunkIndex ChunkManager::list_index(size_t size) {
2241   switch (size) {
2242     case SpecializedChunk:
2243       assert(SpecializedChunk == ClassSpecializedChunk,
2244              "Need branch for ClassSpecializedChunk");
2245       return SpecializedIndex;
2246     case SmallChunk:
2247     case ClassSmallChunk:
2248       return SmallIndex;
2249     case MediumChunk:
2250     case ClassMediumChunk:
2251       return MediumIndex;
2252     default:
2253       assert(size > MediumChunk || size > ClassMediumChunk,
2254              "Not a humongous chunk");
2255       return HumongousIndex;
2256   }
2257 }
2258 
2259 void SpaceManager::deallocate(MetaWord* p, size_t word_size) {
2260   assert_lock_strong(_lock);
2261   size_t raw_word_size = get_raw_word_size(word_size);
2262   size_t min_size = TreeChunk<Metablock, FreeList<Metablock> >::min_size();
2263   assert(raw_word_size >= min_size,
2264          "Should not deallocate dark matter " SIZE_FORMAT "<" SIZE_FORMAT, word_size, min_size);
2265   block_freelists()->return_block(p, raw_word_size);
2266 }
2267 
2268 // Adds a chunk to the list of chunks in use.
2269 void SpaceManager::add_chunk(Metachunk* new_chunk, bool make_current) {
2270 
2271   assert(new_chunk != NULL, "Should not be NULL");
2272   assert(new_chunk->next() == NULL, "Should not be on a list");
2273 
2274   new_chunk->reset_empty();
2275 
2276   // Find the correct list and and set the current
2277   // chunk for that list.
2278   ChunkIndex index = ChunkManager::list_index(new_chunk->word_size());
2279 
2280   if (index != HumongousIndex) {
2281     retire_current_chunk();
2282     set_current_chunk(new_chunk);
2283     new_chunk->set_next(chunks_in_use(index));
2284     set_chunks_in_use(index, new_chunk);
2285   } else {
2286     // For null class loader data and DumpSharedSpaces, the first chunk isn't
2287     // small, so small will be null.  Link this first chunk as the current
2288     // chunk.
2289     if (make_current) {
2290       // Set as the current chunk but otherwise treat as a humongous chunk.
2291       set_current_chunk(new_chunk);
2292     }
2293     // Link at head.  The _current_chunk only points to a humongous chunk for
2294     // the null class loader metaspace (class and data virtual space managers)
2295     // any humongous chunks so will not point to the tail
2296     // of the humongous chunks list.
2297     new_chunk->set_next(chunks_in_use(HumongousIndex));
2298     set_chunks_in_use(HumongousIndex, new_chunk);
2299 
2300     assert(new_chunk->word_size() > medium_chunk_size(), "List inconsistency");
2301   }
2302 
2303   // Add to the running sum of capacity
2304   inc_size_metrics(new_chunk->word_size());
2305 
2306   assert(new_chunk->is_empty(), "Not ready for reuse");
2307   LogHandle(gc, metaspace, freelist) log;
2308   if (log.is_trace()) {
2309     log.trace("SpaceManager::add_chunk: " SIZE_FORMAT ") ", sum_count_in_chunks_in_use());
2310     ResourceMark rm;
2311     outputStream* out = log.trace_stream();
2312     new_chunk->print_on(out);
2313     chunk_manager()->locked_print_free_chunks(out);
2314   }
2315 }
2316 
2317 void SpaceManager::retire_current_chunk() {
2318   if (current_chunk() != NULL) {
2319     size_t remaining_words = current_chunk()->free_word_size();
2320     if (remaining_words >= TreeChunk<Metablock, FreeList<Metablock> >::min_size()) {
2321       block_freelists()->return_block(current_chunk()->allocate(remaining_words), remaining_words);
2322       inc_used_metrics(remaining_words);
2323     }
2324   }
2325 }
2326 
2327 Metachunk* SpaceManager::get_new_chunk(size_t word_size,
2328                                        size_t grow_chunks_by_words) {
2329   // Get a chunk from the chunk freelist
2330   Metachunk* next = chunk_manager()->chunk_freelist_allocate(grow_chunks_by_words);
2331 
2332   if (next == NULL) {
2333     next = vs_list()->get_new_chunk(word_size,
2334                                     grow_chunks_by_words,
2335                                     medium_chunk_bunch());
2336   }
2337 
2338   LogHandle(gc, metaspace, alloc) log;
2339   if (log.is_debug() && next != NULL &&
2340       SpaceManager::is_humongous(next->word_size())) {
2341     log.debug("  new humongous chunk word size " PTR_FORMAT, next->word_size());
2342   }
2343 
2344   return next;
2345 }
2346 
2347 /*
2348  * The policy is to allocate up to _small_chunk_limit small chunks
2349  * after which only medium chunks are allocated.  This is done to
2350  * reduce fragmentation.  In some cases, this can result in a lot
2351  * of small chunks being allocated to the point where it's not
2352  * possible to expand.  If this happens, there may be no medium chunks
2353  * available and OOME would be thrown.  Instead of doing that,
2354  * if the allocation request size fits in a small chunk, an attempt
2355  * will be made to allocate a small chunk.
2356  */
2357 MetaWord* SpaceManager::get_small_chunk_and_allocate(size_t word_size) {
2358   size_t raw_word_size = get_raw_word_size(word_size);
2359 
2360   if (raw_word_size + Metachunk::overhead() > small_chunk_size()) {
2361     return NULL;
2362   }
2363 
2364   MutexLockerEx cl(lock(), Mutex::_no_safepoint_check_flag);
2365   MutexLockerEx cl1(expand_lock(), Mutex::_no_safepoint_check_flag);
2366 
2367   Metachunk* chunk = chunk_manager()->chunk_freelist_allocate(small_chunk_size());
2368 
2369   MetaWord* mem = NULL;
2370 
2371   if (chunk != NULL) {
2372     // Add chunk to the in-use chunk list and do an allocation from it.
2373     // Add to this manager's list of chunks in use.
2374     add_chunk(chunk, false);
2375     mem = chunk->allocate(raw_word_size);
2376 
2377     inc_used_metrics(raw_word_size);
2378 
2379     // Track metaspace memory usage statistic.
2380     track_metaspace_memory_usage();
2381   }
2382 
2383   return mem;
2384 }
2385 
2386 MetaWord* SpaceManager::allocate(size_t word_size) {
2387   MutexLockerEx cl(lock(), Mutex::_no_safepoint_check_flag);
2388 
2389   size_t raw_word_size = get_raw_word_size(word_size);
2390   BlockFreelist* fl =  block_freelists();
2391   MetaWord* p = NULL;
2392   // Allocation from the dictionary is expensive in the sense that
2393   // the dictionary has to be searched for a size.  Don't allocate
2394   // from the dictionary until it starts to get fat.  Is this
2395   // a reasonable policy?  Maybe an skinny dictionary is fast enough
2396   // for allocations.  Do some profiling.  JJJ
2397   if (fl->total_size() > allocation_from_dictionary_limit) {
2398     p = fl->get_block(raw_word_size);
2399   }
2400   if (p == NULL) {
2401     p = allocate_work(raw_word_size);
2402   }
2403 
2404   return p;
2405 }
2406 
2407 // Returns the address of spaced allocated for "word_size".
2408 // This methods does not know about blocks (Metablocks)
2409 MetaWord* SpaceManager::allocate_work(size_t word_size) {
2410   assert_lock_strong(_lock);
2411 #ifdef ASSERT
2412   if (Metadebug::test_metadata_failure()) {
2413     return NULL;
2414   }
2415 #endif
2416   // Is there space in the current chunk?
2417   MetaWord* result = NULL;
2418 
2419   // For DumpSharedSpaces, only allocate out of the current chunk which is
2420   // never null because we gave it the size we wanted.   Caller reports out
2421   // of memory if this returns null.
2422   if (DumpSharedSpaces) {
2423     assert(current_chunk() != NULL, "should never happen");
2424     inc_used_metrics(word_size);
2425     return current_chunk()->allocate(word_size); // caller handles null result
2426   }
2427 
2428   if (current_chunk() != NULL) {
2429     result = current_chunk()->allocate(word_size);
2430   }
2431 
2432   if (result == NULL) {
2433     result = grow_and_allocate(word_size);
2434   }
2435 
2436   if (result != NULL) {
2437     inc_used_metrics(word_size);
2438     assert(result != (MetaWord*) chunks_in_use(MediumIndex),
2439            "Head of the list is being allocated");
2440   }
2441 
2442   return result;
2443 }
2444 
2445 void SpaceManager::verify() {
2446   // If there are blocks in the dictionary, then
2447   // verification of chunks does not work since
2448   // being in the dictionary alters a chunk.
2449   if (block_freelists()->total_size() == 0) {
2450     for (ChunkIndex i = ZeroIndex; i < NumberOfInUseLists; i = next_chunk_index(i)) {
2451       Metachunk* curr = chunks_in_use(i);
2452       while (curr != NULL) {
2453         curr->verify();
2454         verify_chunk_size(curr);
2455         curr = curr->next();
2456       }
2457     }
2458   }
2459 }
2460 
2461 void SpaceManager::verify_chunk_size(Metachunk* chunk) {
2462   assert(is_humongous(chunk->word_size()) ||
2463          chunk->word_size() == medium_chunk_size() ||
2464          chunk->word_size() == small_chunk_size() ||
2465          chunk->word_size() == specialized_chunk_size(),
2466          "Chunk size is wrong");
2467   return;
2468 }
2469 
2470 #ifdef ASSERT
2471 void SpaceManager::verify_allocated_blocks_words() {
2472   // Verification is only guaranteed at a safepoint.
2473   assert(SafepointSynchronize::is_at_safepoint() || !Universe::is_fully_initialized(),
2474     "Verification can fail if the applications is running");
2475   assert(allocated_blocks_words() == sum_used_in_chunks_in_use(),
2476          "allocation total is not consistent " SIZE_FORMAT
2477          " vs " SIZE_FORMAT,
2478          allocated_blocks_words(), sum_used_in_chunks_in_use());
2479 }
2480 
2481 #endif
2482 
2483 void SpaceManager::dump(outputStream* const out) const {
2484   size_t curr_total = 0;
2485   size_t waste = 0;
2486   uint i = 0;
2487   size_t used = 0;
2488   size_t capacity = 0;
2489 
2490   // Add up statistics for all chunks in this SpaceManager.
2491   for (ChunkIndex index = ZeroIndex;
2492        index < NumberOfInUseLists;
2493        index = next_chunk_index(index)) {
2494     for (Metachunk* curr = chunks_in_use(index);
2495          curr != NULL;
2496          curr = curr->next()) {
2497       out->print("%d) ", i++);
2498       curr->print_on(out);
2499       curr_total += curr->word_size();
2500       used += curr->used_word_size();
2501       capacity += curr->word_size();
2502       waste += curr->free_word_size() + curr->overhead();;
2503     }
2504   }
2505 
2506   if (log_is_enabled(Trace, gc, metaspace, freelist)) {
2507     block_freelists()->print_on(out);
2508   }
2509 
2510   size_t free = current_chunk() == NULL ? 0 : current_chunk()->free_word_size();
2511   // Free space isn't wasted.
2512   waste -= free;
2513 
2514   out->print_cr("total of all chunks "  SIZE_FORMAT " used " SIZE_FORMAT
2515                 " free " SIZE_FORMAT " capacity " SIZE_FORMAT
2516                 " waste " SIZE_FORMAT, curr_total, used, free, capacity, waste);
2517 }
2518 
2519 #ifndef PRODUCT
2520 void SpaceManager::mangle_freed_chunks() {
2521   for (ChunkIndex index = ZeroIndex;
2522        index < NumberOfInUseLists;
2523        index = next_chunk_index(index)) {
2524     for (Metachunk* curr = chunks_in_use(index);
2525          curr != NULL;
2526          curr = curr->next()) {
2527       curr->mangle();
2528     }
2529   }
2530 }
2531 #endif // PRODUCT
2532 
2533 // MetaspaceAux
2534 
2535 
2536 size_t MetaspaceAux::_capacity_words[] = {0, 0};
2537 size_t MetaspaceAux::_used_words[] = {0, 0};
2538 
2539 size_t MetaspaceAux::free_bytes(Metaspace::MetadataType mdtype) {
2540   VirtualSpaceList* list = Metaspace::get_space_list(mdtype);
2541   return list == NULL ? 0 : list->free_bytes();
2542 }
2543 
2544 size_t MetaspaceAux::free_bytes() {
2545   return free_bytes(Metaspace::ClassType) + free_bytes(Metaspace::NonClassType);
2546 }
2547 
2548 void MetaspaceAux::dec_capacity(Metaspace::MetadataType mdtype, size_t words) {
2549   assert_lock_strong(SpaceManager::expand_lock());
2550   assert(words <= capacity_words(mdtype),
2551          "About to decrement below 0: words " SIZE_FORMAT
2552          " is greater than _capacity_words[%u] " SIZE_FORMAT,
2553          words, mdtype, capacity_words(mdtype));
2554   _capacity_words[mdtype] -= words;
2555 }
2556 
2557 void MetaspaceAux::inc_capacity(Metaspace::MetadataType mdtype, size_t words) {
2558   assert_lock_strong(SpaceManager::expand_lock());
2559   // Needs to be atomic
2560   _capacity_words[mdtype] += words;
2561 }
2562 
2563 void MetaspaceAux::dec_used(Metaspace::MetadataType mdtype, size_t words) {
2564   assert(words <= used_words(mdtype),
2565          "About to decrement below 0: words " SIZE_FORMAT
2566          " is greater than _used_words[%u] " SIZE_FORMAT,
2567          words, mdtype, used_words(mdtype));
2568   // For CMS deallocation of the Metaspaces occurs during the
2569   // sweep which is a concurrent phase.  Protection by the expand_lock()
2570   // is not enough since allocation is on a per Metaspace basis
2571   // and protected by the Metaspace lock.
2572   jlong minus_words = (jlong) - (jlong) words;
2573   Atomic::add_ptr(minus_words, &_used_words[mdtype]);
2574 }
2575 
2576 void MetaspaceAux::inc_used(Metaspace::MetadataType mdtype, size_t words) {
2577   // _used_words tracks allocations for
2578   // each piece of metadata.  Those allocations are
2579   // generally done concurrently by different application
2580   // threads so must be done atomically.
2581   Atomic::add_ptr(words, &_used_words[mdtype]);
2582 }
2583 
2584 size_t MetaspaceAux::used_bytes_slow(Metaspace::MetadataType mdtype) {
2585   size_t used = 0;
2586   ClassLoaderDataGraphMetaspaceIterator iter;
2587   while (iter.repeat()) {
2588     Metaspace* msp = iter.get_next();
2589     // Sum allocated_blocks_words for each metaspace
2590     if (msp != NULL) {
2591       used += msp->used_words_slow(mdtype);
2592     }
2593   }
2594   return used * BytesPerWord;
2595 }
2596 
2597 size_t MetaspaceAux::free_bytes_slow(Metaspace::MetadataType mdtype) {
2598   size_t free = 0;
2599   ClassLoaderDataGraphMetaspaceIterator iter;
2600   while (iter.repeat()) {
2601     Metaspace* msp = iter.get_next();
2602     if (msp != NULL) {
2603       free += msp->free_words_slow(mdtype);
2604     }
2605   }
2606   return free * BytesPerWord;
2607 }
2608 
2609 size_t MetaspaceAux::capacity_bytes_slow(Metaspace::MetadataType mdtype) {
2610   if ((mdtype == Metaspace::ClassType) && !Metaspace::using_class_space()) {
2611     return 0;
2612   }
2613   // Don't count the space in the freelists.  That space will be
2614   // added to the capacity calculation as needed.
2615   size_t capacity = 0;
2616   ClassLoaderDataGraphMetaspaceIterator iter;
2617   while (iter.repeat()) {
2618     Metaspace* msp = iter.get_next();
2619     if (msp != NULL) {
2620       capacity += msp->capacity_words_slow(mdtype);
2621     }
2622   }
2623   return capacity * BytesPerWord;
2624 }
2625 
2626 size_t MetaspaceAux::capacity_bytes_slow() {
2627 #ifdef PRODUCT
2628   // Use capacity_bytes() in PRODUCT instead of this function.
2629   guarantee(false, "Should not call capacity_bytes_slow() in the PRODUCT");
2630 #endif
2631   size_t class_capacity = capacity_bytes_slow(Metaspace::ClassType);
2632   size_t non_class_capacity = capacity_bytes_slow(Metaspace::NonClassType);
2633   assert(capacity_bytes() == class_capacity + non_class_capacity,
2634          "bad accounting: capacity_bytes() " SIZE_FORMAT
2635          " class_capacity + non_class_capacity " SIZE_FORMAT
2636          " class_capacity " SIZE_FORMAT " non_class_capacity " SIZE_FORMAT,
2637          capacity_bytes(), class_capacity + non_class_capacity,
2638          class_capacity, non_class_capacity);
2639 
2640   return class_capacity + non_class_capacity;
2641 }
2642 
2643 size_t MetaspaceAux::reserved_bytes(Metaspace::MetadataType mdtype) {
2644   VirtualSpaceList* list = Metaspace::get_space_list(mdtype);
2645   return list == NULL ? 0 : list->reserved_bytes();
2646 }
2647 
2648 size_t MetaspaceAux::committed_bytes(Metaspace::MetadataType mdtype) {
2649   VirtualSpaceList* list = Metaspace::get_space_list(mdtype);
2650   return list == NULL ? 0 : list->committed_bytes();
2651 }
2652 
2653 size_t MetaspaceAux::min_chunk_size_words() { return Metaspace::first_chunk_word_size(); }
2654 
2655 size_t MetaspaceAux::free_chunks_total_words(Metaspace::MetadataType mdtype) {
2656   ChunkManager* chunk_manager = Metaspace::get_chunk_manager(mdtype);
2657   if (chunk_manager == NULL) {
2658     return 0;
2659   }
2660   chunk_manager->slow_verify();
2661   return chunk_manager->free_chunks_total_words();
2662 }
2663 
2664 size_t MetaspaceAux::free_chunks_total_bytes(Metaspace::MetadataType mdtype) {
2665   return free_chunks_total_words(mdtype) * BytesPerWord;
2666 }
2667 
2668 size_t MetaspaceAux::free_chunks_total_words() {
2669   return free_chunks_total_words(Metaspace::ClassType) +
2670          free_chunks_total_words(Metaspace::NonClassType);
2671 }
2672 
2673 size_t MetaspaceAux::free_chunks_total_bytes() {
2674   return free_chunks_total_words() * BytesPerWord;
2675 }
2676 
2677 bool MetaspaceAux::has_chunk_free_list(Metaspace::MetadataType mdtype) {
2678   return Metaspace::get_chunk_manager(mdtype) != NULL;
2679 }
2680 
2681 MetaspaceChunkFreeListSummary MetaspaceAux::chunk_free_list_summary(Metaspace::MetadataType mdtype) {
2682   if (!has_chunk_free_list(mdtype)) {
2683     return MetaspaceChunkFreeListSummary();
2684   }
2685 
2686   const ChunkManager* cm = Metaspace::get_chunk_manager(mdtype);
2687   return cm->chunk_free_list_summary();
2688 }
2689 
2690 void MetaspaceAux::print_metaspace_change(size_t prev_metadata_used) {
2691   log_info(gc, metaspace)("Metaspace: "  SIZE_FORMAT "K->" SIZE_FORMAT "K("  SIZE_FORMAT "K)",
2692                           prev_metadata_used/K, used_bytes()/K, reserved_bytes()/K);
2693 }
2694 
2695 void MetaspaceAux::print_on(outputStream* out) {
2696   Metaspace::MetadataType nct = Metaspace::NonClassType;
2697 
2698   out->print_cr(" Metaspace       "
2699                 "used "      SIZE_FORMAT "K, "
2700                 "capacity "  SIZE_FORMAT "K, "
2701                 "committed " SIZE_FORMAT "K, "
2702                 "reserved "  SIZE_FORMAT "K",
2703                 used_bytes()/K,
2704                 capacity_bytes()/K,
2705                 committed_bytes()/K,
2706                 reserved_bytes()/K);
2707 
2708   if (Metaspace::using_class_space()) {
2709     Metaspace::MetadataType ct = Metaspace::ClassType;
2710     out->print_cr("  class space    "
2711                   "used "      SIZE_FORMAT "K, "
2712                   "capacity "  SIZE_FORMAT "K, "
2713                   "committed " SIZE_FORMAT "K, "
2714                   "reserved "  SIZE_FORMAT "K",
2715                   used_bytes(ct)/K,
2716                   capacity_bytes(ct)/K,
2717                   committed_bytes(ct)/K,
2718                   reserved_bytes(ct)/K);
2719   }
2720 }
2721 
2722 // Print information for class space and data space separately.
2723 // This is almost the same as above.
2724 void MetaspaceAux::print_on(outputStream* out, Metaspace::MetadataType mdtype) {
2725   size_t free_chunks_capacity_bytes = free_chunks_total_bytes(mdtype);
2726   size_t capacity_bytes = capacity_bytes_slow(mdtype);
2727   size_t used_bytes = used_bytes_slow(mdtype);
2728   size_t free_bytes = free_bytes_slow(mdtype);
2729   size_t used_and_free = used_bytes + free_bytes +
2730                            free_chunks_capacity_bytes;
2731   out->print_cr("  Chunk accounting: used in chunks " SIZE_FORMAT
2732              "K + unused in chunks " SIZE_FORMAT "K  + "
2733              " capacity in free chunks " SIZE_FORMAT "K = " SIZE_FORMAT
2734              "K  capacity in allocated chunks " SIZE_FORMAT "K",
2735              used_bytes / K,
2736              free_bytes / K,
2737              free_chunks_capacity_bytes / K,
2738              used_and_free / K,
2739              capacity_bytes / K);
2740   // Accounting can only be correct if we got the values during a safepoint
2741   assert(!SafepointSynchronize::is_at_safepoint() || used_and_free == capacity_bytes, "Accounting is wrong");
2742 }
2743 
2744 // Print total fragmentation for class metaspaces
2745 void MetaspaceAux::print_class_waste(outputStream* out) {
2746   assert(Metaspace::using_class_space(), "class metaspace not used");
2747   size_t cls_specialized_waste = 0, cls_small_waste = 0, cls_medium_waste = 0;
2748   size_t cls_specialized_count = 0, cls_small_count = 0, cls_medium_count = 0, cls_humongous_count = 0;
2749   ClassLoaderDataGraphMetaspaceIterator iter;
2750   while (iter.repeat()) {
2751     Metaspace* msp = iter.get_next();
2752     if (msp != NULL) {
2753       cls_specialized_waste += msp->class_vsm()->sum_waste_in_chunks_in_use(SpecializedIndex);
2754       cls_specialized_count += msp->class_vsm()->sum_count_in_chunks_in_use(SpecializedIndex);
2755       cls_small_waste += msp->class_vsm()->sum_waste_in_chunks_in_use(SmallIndex);
2756       cls_small_count += msp->class_vsm()->sum_count_in_chunks_in_use(SmallIndex);
2757       cls_medium_waste += msp->class_vsm()->sum_waste_in_chunks_in_use(MediumIndex);
2758       cls_medium_count += msp->class_vsm()->sum_count_in_chunks_in_use(MediumIndex);
2759       cls_humongous_count += msp->class_vsm()->sum_count_in_chunks_in_use(HumongousIndex);
2760     }
2761   }
2762   out->print_cr(" class: " SIZE_FORMAT " specialized(s) " SIZE_FORMAT ", "
2763                 SIZE_FORMAT " small(s) " SIZE_FORMAT ", "
2764                 SIZE_FORMAT " medium(s) " SIZE_FORMAT ", "
2765                 "large count " SIZE_FORMAT,
2766                 cls_specialized_count, cls_specialized_waste,
2767                 cls_small_count, cls_small_waste,
2768                 cls_medium_count, cls_medium_waste, cls_humongous_count);
2769 }
2770 
2771 // Print total fragmentation for data and class metaspaces separately
2772 void MetaspaceAux::print_waste(outputStream* out) {
2773   size_t specialized_waste = 0, small_waste = 0, medium_waste = 0;
2774   size_t specialized_count = 0, small_count = 0, medium_count = 0, humongous_count = 0;
2775 
2776   ClassLoaderDataGraphMetaspaceIterator iter;
2777   while (iter.repeat()) {
2778     Metaspace* msp = iter.get_next();
2779     if (msp != NULL) {
2780       specialized_waste += msp->vsm()->sum_waste_in_chunks_in_use(SpecializedIndex);
2781       specialized_count += msp->vsm()->sum_count_in_chunks_in_use(SpecializedIndex);
2782       small_waste += msp->vsm()->sum_waste_in_chunks_in_use(SmallIndex);
2783       small_count += msp->vsm()->sum_count_in_chunks_in_use(SmallIndex);
2784       medium_waste += msp->vsm()->sum_waste_in_chunks_in_use(MediumIndex);
2785       medium_count += msp->vsm()->sum_count_in_chunks_in_use(MediumIndex);
2786       humongous_count += msp->vsm()->sum_count_in_chunks_in_use(HumongousIndex);
2787     }
2788   }
2789   out->print_cr("Total fragmentation waste (words) doesn't count free space");
2790   out->print_cr("  data: " SIZE_FORMAT " specialized(s) " SIZE_FORMAT ", "
2791                         SIZE_FORMAT " small(s) " SIZE_FORMAT ", "
2792                         SIZE_FORMAT " medium(s) " SIZE_FORMAT ", "
2793                         "large count " SIZE_FORMAT,
2794              specialized_count, specialized_waste, small_count,
2795              small_waste, medium_count, medium_waste, humongous_count);
2796   if (Metaspace::using_class_space()) {
2797     print_class_waste(out);
2798   }
2799 }
2800 
2801 // Dump global metaspace things from the end of ClassLoaderDataGraph
2802 void MetaspaceAux::dump(outputStream* out) {
2803   out->print_cr("All Metaspace:");
2804   out->print("data space: "); print_on(out, Metaspace::NonClassType);
2805   out->print("class space: "); print_on(out, Metaspace::ClassType);
2806   print_waste(out);
2807 }
2808 
2809 void MetaspaceAux::verify_free_chunks() {
2810   Metaspace::chunk_manager_metadata()->verify();
2811   if (Metaspace::using_class_space()) {
2812     Metaspace::chunk_manager_class()->verify();
2813   }
2814 }
2815 
2816 void MetaspaceAux::verify_capacity() {
2817 #ifdef ASSERT
2818   size_t running_sum_capacity_bytes = capacity_bytes();
2819   // For purposes of the running sum of capacity, verify against capacity
2820   size_t capacity_in_use_bytes = capacity_bytes_slow();
2821   assert(running_sum_capacity_bytes == capacity_in_use_bytes,
2822          "capacity_words() * BytesPerWord " SIZE_FORMAT
2823          " capacity_bytes_slow()" SIZE_FORMAT,
2824          running_sum_capacity_bytes, capacity_in_use_bytes);
2825   for (Metaspace::MetadataType i = Metaspace::ClassType;
2826        i < Metaspace:: MetadataTypeCount;
2827        i = (Metaspace::MetadataType)(i + 1)) {
2828     size_t capacity_in_use_bytes = capacity_bytes_slow(i);
2829     assert(capacity_bytes(i) == capacity_in_use_bytes,
2830            "capacity_bytes(%u) " SIZE_FORMAT
2831            " capacity_bytes_slow(%u)" SIZE_FORMAT,
2832            i, capacity_bytes(i), i, capacity_in_use_bytes);
2833   }
2834 #endif
2835 }
2836 
2837 void MetaspaceAux::verify_used() {
2838 #ifdef ASSERT
2839   size_t running_sum_used_bytes = used_bytes();
2840   // For purposes of the running sum of used, verify against used
2841   size_t used_in_use_bytes = used_bytes_slow();
2842   assert(used_bytes() == used_in_use_bytes,
2843          "used_bytes() " SIZE_FORMAT
2844          " used_bytes_slow()" SIZE_FORMAT,
2845          used_bytes(), used_in_use_bytes);
2846   for (Metaspace::MetadataType i = Metaspace::ClassType;
2847        i < Metaspace:: MetadataTypeCount;
2848        i = (Metaspace::MetadataType)(i + 1)) {
2849     size_t used_in_use_bytes = used_bytes_slow(i);
2850     assert(used_bytes(i) == used_in_use_bytes,
2851            "used_bytes(%u) " SIZE_FORMAT
2852            " used_bytes_slow(%u)" SIZE_FORMAT,
2853            i, used_bytes(i), i, used_in_use_bytes);
2854   }
2855 #endif
2856 }
2857 
2858 void MetaspaceAux::verify_metrics() {
2859   verify_capacity();
2860   verify_used();
2861 }
2862 
2863 
2864 // Metaspace methods
2865 
2866 size_t Metaspace::_first_chunk_word_size = 0;
2867 size_t Metaspace::_first_class_chunk_word_size = 0;
2868 
2869 size_t Metaspace::_commit_alignment = 0;
2870 size_t Metaspace::_reserve_alignment = 0;
2871 
2872 Metaspace::Metaspace(Mutex* lock, MetaspaceType type) {
2873   initialize(lock, type);
2874 }
2875 
2876 Metaspace::~Metaspace() {
2877   delete _vsm;
2878   if (using_class_space()) {
2879     delete _class_vsm;
2880   }
2881 }
2882 
2883 VirtualSpaceList* Metaspace::_space_list = NULL;
2884 VirtualSpaceList* Metaspace::_class_space_list = NULL;
2885 
2886 ChunkManager* Metaspace::_chunk_manager_metadata = NULL;
2887 ChunkManager* Metaspace::_chunk_manager_class = NULL;
2888 
2889 #define VIRTUALSPACEMULTIPLIER 2
2890 
2891 #ifdef _LP64
2892 static const uint64_t UnscaledClassSpaceMax = (uint64_t(max_juint) + 1);
2893 
2894 void Metaspace::set_narrow_klass_base_and_shift(address metaspace_base, address cds_base) {
2895   // Figure out the narrow_klass_base and the narrow_klass_shift.  The
2896   // narrow_klass_base is the lower of the metaspace base and the cds base
2897   // (if cds is enabled).  The narrow_klass_shift depends on the distance
2898   // between the lower base and higher address.
2899   address lower_base;
2900   address higher_address;
2901 #if INCLUDE_CDS
2902   if (UseSharedSpaces) {
2903     higher_address = MAX2((address)(cds_base + FileMapInfo::shared_spaces_size()),
2904                           (address)(metaspace_base + compressed_class_space_size()));
2905     lower_base = MIN2(metaspace_base, cds_base);
2906   } else
2907 #endif
2908   {
2909     higher_address = metaspace_base + compressed_class_space_size();
2910     lower_base = metaspace_base;
2911 
2912     uint64_t klass_encoding_max = UnscaledClassSpaceMax << LogKlassAlignmentInBytes;
2913     // If compressed class space fits in lower 32G, we don't need a base.
2914     if (higher_address <= (address)klass_encoding_max) {
2915       lower_base = 0; // Effectively lower base is zero.
2916     }
2917   }
2918 
2919   Universe::set_narrow_klass_base(lower_base);
2920 
2921   if ((uint64_t)(higher_address - lower_base) <= UnscaledClassSpaceMax) {
2922     Universe::set_narrow_klass_shift(0);
2923   } else {
2924     assert(!UseSharedSpaces, "Cannot shift with UseSharedSpaces");
2925     Universe::set_narrow_klass_shift(LogKlassAlignmentInBytes);
2926   }
2927 }
2928 
2929 #if INCLUDE_CDS
2930 // Return TRUE if the specified metaspace_base and cds_base are close enough
2931 // to work with compressed klass pointers.
2932 bool Metaspace::can_use_cds_with_metaspace_addr(char* metaspace_base, address cds_base) {
2933   assert(cds_base != 0 && UseSharedSpaces, "Only use with CDS");
2934   assert(UseCompressedClassPointers, "Only use with CompressedKlassPtrs");
2935   address lower_base = MIN2((address)metaspace_base, cds_base);
2936   address higher_address = MAX2((address)(cds_base + FileMapInfo::shared_spaces_size()),
2937                                 (address)(metaspace_base + compressed_class_space_size()));
2938   return ((uint64_t)(higher_address - lower_base) <= UnscaledClassSpaceMax);
2939 }
2940 #endif
2941 
2942 // Try to allocate the metaspace at the requested addr.
2943 void Metaspace::allocate_metaspace_compressed_klass_ptrs(char* requested_addr, address cds_base) {
2944   assert(using_class_space(), "called improperly");
2945   assert(UseCompressedClassPointers, "Only use with CompressedKlassPtrs");
2946   assert(compressed_class_space_size() < KlassEncodingMetaspaceMax,
2947          "Metaspace size is too big");
2948   assert_is_ptr_aligned(requested_addr, _reserve_alignment);
2949   assert_is_ptr_aligned(cds_base, _reserve_alignment);
2950   assert_is_size_aligned(compressed_class_space_size(), _reserve_alignment);
2951 
2952   // Don't use large pages for the class space.
2953   bool large_pages = false;
2954 
2955 #ifndef AARCH64
2956   ReservedSpace metaspace_rs = ReservedSpace(compressed_class_space_size(),
2957                                              _reserve_alignment,
2958                                              large_pages,
2959                                              requested_addr);
2960 #else // AARCH64
2961   ReservedSpace metaspace_rs;
2962 
2963   // Our compressed klass pointers may fit nicely into the lower 32
2964   // bits.
2965   if ((uint64_t)requested_addr + compressed_class_space_size() < 4*G) {
2966     metaspace_rs = ReservedSpace(compressed_class_space_size(),
2967                                              _reserve_alignment,
2968                                              large_pages,
2969                                              requested_addr);
2970   }
2971 
2972   if (! metaspace_rs.is_reserved()) {
2973     // Try to align metaspace so that we can decode a compressed klass
2974     // with a single MOVK instruction.  We can do this iff the
2975     // compressed class base is a multiple of 4G.
2976     for (char *a = (char*)align_ptr_up(requested_addr, 4*G);
2977          a < (char*)(1024*G);
2978          a += 4*G) {
2979 
2980 #if INCLUDE_CDS
2981       if (UseSharedSpaces
2982           && ! can_use_cds_with_metaspace_addr(a, cds_base)) {
2983         // We failed to find an aligned base that will reach.  Fall
2984         // back to using our requested addr.
2985         metaspace_rs = ReservedSpace(compressed_class_space_size(),
2986                                      _reserve_alignment,
2987                                      large_pages,
2988                                      requested_addr);
2989         break;
2990       }
2991 #endif
2992 
2993       metaspace_rs = ReservedSpace(compressed_class_space_size(),
2994                                    _reserve_alignment,
2995                                    large_pages,
2996                                    a);
2997       if (metaspace_rs.is_reserved())
2998         break;
2999     }
3000   }
3001 
3002 #endif // AARCH64
3003 
3004   if (!metaspace_rs.is_reserved()) {
3005 #if INCLUDE_CDS
3006     if (UseSharedSpaces) {
3007       size_t increment = align_size_up(1*G, _reserve_alignment);
3008 
3009       // Keep trying to allocate the metaspace, increasing the requested_addr
3010       // by 1GB each time, until we reach an address that will no longer allow
3011       // use of CDS with compressed klass pointers.
3012       char *addr = requested_addr;
3013       while (!metaspace_rs.is_reserved() && (addr + increment > addr) &&
3014              can_use_cds_with_metaspace_addr(addr + increment, cds_base)) {
3015         addr = addr + increment;
3016         metaspace_rs = ReservedSpace(compressed_class_space_size(),
3017                                      _reserve_alignment, large_pages, addr);
3018       }
3019     }
3020 #endif
3021     // If no successful allocation then try to allocate the space anywhere.  If
3022     // that fails then OOM doom.  At this point we cannot try allocating the
3023     // metaspace as if UseCompressedClassPointers is off because too much
3024     // initialization has happened that depends on UseCompressedClassPointers.
3025     // So, UseCompressedClassPointers cannot be turned off at this point.
3026     if (!metaspace_rs.is_reserved()) {
3027       metaspace_rs = ReservedSpace(compressed_class_space_size(),
3028                                    _reserve_alignment, large_pages);
3029       if (!metaspace_rs.is_reserved()) {
3030         vm_exit_during_initialization(err_msg("Could not allocate metaspace: " SIZE_FORMAT " bytes",
3031                                               compressed_class_space_size()));
3032       }
3033     }
3034   }
3035 
3036   // If we got here then the metaspace got allocated.
3037   MemTracker::record_virtual_memory_type((address)metaspace_rs.base(), mtClass);
3038 
3039 #if INCLUDE_CDS
3040   // Verify that we can use shared spaces.  Otherwise, turn off CDS.
3041   if (UseSharedSpaces && !can_use_cds_with_metaspace_addr(metaspace_rs.base(), cds_base)) {
3042     FileMapInfo::stop_sharing_and_unmap(
3043         "Could not allocate metaspace at a compatible address");
3044   }
3045 #endif
3046   set_narrow_klass_base_and_shift((address)metaspace_rs.base(),
3047                                   UseSharedSpaces ? (address)cds_base : 0);
3048 
3049   initialize_class_space(metaspace_rs);
3050 
3051   if (log_is_enabled(Trace, gc, metaspace)) {
3052     LogHandle(gc, metaspace) log;
3053     ResourceMark rm;
3054     print_compressed_class_space(log.trace_stream(), requested_addr);
3055   }
3056 }
3057 
3058 void Metaspace::print_compressed_class_space(outputStream* st, const char* requested_addr) {
3059   st->print_cr("Narrow klass base: " PTR_FORMAT ", Narrow klass shift: %d",
3060                p2i(Universe::narrow_klass_base()), Universe::narrow_klass_shift());
3061   if (_class_space_list != NULL) {
3062     address base = (address)_class_space_list->current_virtual_space()->bottom();
3063     st->print("Compressed class space size: " SIZE_FORMAT " Address: " PTR_FORMAT,
3064                  compressed_class_space_size(), p2i(base));
3065     if (requested_addr != 0) {
3066       st->print(" Req Addr: " PTR_FORMAT, p2i(requested_addr));
3067     }
3068     st->cr();
3069   }
3070 }
3071 
3072 // For UseCompressedClassPointers the class space is reserved above the top of
3073 // the Java heap.  The argument passed in is at the base of the compressed space.
3074 void Metaspace::initialize_class_space(ReservedSpace rs) {
3075   // The reserved space size may be bigger because of alignment, esp with UseLargePages
3076   assert(rs.size() >= CompressedClassSpaceSize,
3077          SIZE_FORMAT " != " SIZE_FORMAT, rs.size(), CompressedClassSpaceSize);
3078   assert(using_class_space(), "Must be using class space");
3079   _class_space_list = new VirtualSpaceList(rs);
3080   _chunk_manager_class = new ChunkManager(SpecializedChunk, ClassSmallChunk, ClassMediumChunk);
3081 
3082   if (!_class_space_list->initialization_succeeded()) {
3083     vm_exit_during_initialization("Failed to setup compressed class space virtual space list.");
3084   }
3085 }
3086 
3087 #endif
3088 
3089 void Metaspace::ergo_initialize() {
3090   if (DumpSharedSpaces) {
3091     // Using large pages when dumping the shared archive is currently not implemented.
3092     FLAG_SET_ERGO(bool, UseLargePagesInMetaspace, false);
3093   }
3094 
3095   size_t page_size = os::vm_page_size();
3096   if (UseLargePages && UseLargePagesInMetaspace) {
3097     page_size = os::large_page_size();
3098   }
3099 
3100   _commit_alignment  = page_size;
3101   _reserve_alignment = MAX2(page_size, (size_t)os::vm_allocation_granularity());
3102 
3103   // Do not use FLAG_SET_ERGO to update MaxMetaspaceSize, since this will
3104   // override if MaxMetaspaceSize was set on the command line or not.
3105   // This information is needed later to conform to the specification of the
3106   // java.lang.management.MemoryUsage API.
3107   //
3108   // Ideally, we would be able to set the default value of MaxMetaspaceSize in
3109   // globals.hpp to the aligned value, but this is not possible, since the
3110   // alignment depends on other flags being parsed.
3111   MaxMetaspaceSize = align_size_down_bounded(MaxMetaspaceSize, _reserve_alignment);
3112 
3113   if (MetaspaceSize > MaxMetaspaceSize) {
3114     MetaspaceSize = MaxMetaspaceSize;
3115   }
3116 
3117   MetaspaceSize = align_size_down_bounded(MetaspaceSize, _commit_alignment);
3118 
3119   assert(MetaspaceSize <= MaxMetaspaceSize, "MetaspaceSize should be limited by MaxMetaspaceSize");
3120 
3121   if (MetaspaceSize < 256*K) {
3122     vm_exit_during_initialization("Too small initial Metaspace size");
3123   }
3124 
3125   MinMetaspaceExpansion = align_size_down_bounded(MinMetaspaceExpansion, _commit_alignment);
3126   MaxMetaspaceExpansion = align_size_down_bounded(MaxMetaspaceExpansion, _commit_alignment);
3127 
3128   CompressedClassSpaceSize = align_size_down_bounded(CompressedClassSpaceSize, _reserve_alignment);
3129   set_compressed_class_space_size(CompressedClassSpaceSize);
3130 }
3131 
3132 void Metaspace::global_initialize() {
3133   MetaspaceGC::initialize();
3134 
3135   // Initialize the alignment for shared spaces.
3136   int max_alignment = os::vm_allocation_granularity();
3137   size_t cds_total = 0;
3138 
3139   MetaspaceShared::set_max_alignment(max_alignment);
3140 
3141   if (DumpSharedSpaces) {
3142 #if INCLUDE_CDS
3143     MetaspaceShared::estimate_regions_size();
3144 
3145     SharedReadOnlySize  = align_size_up(SharedReadOnlySize,  max_alignment);
3146     SharedReadWriteSize = align_size_up(SharedReadWriteSize, max_alignment);
3147     SharedMiscDataSize  = align_size_up(SharedMiscDataSize,  max_alignment);
3148     SharedMiscCodeSize  = align_size_up(SharedMiscCodeSize,  max_alignment);
3149 
3150     // Initialize with the sum of the shared space sizes.  The read-only
3151     // and read write metaspace chunks will be allocated out of this and the
3152     // remainder is the misc code and data chunks.
3153     cds_total = FileMapInfo::shared_spaces_size();
3154     cds_total = align_size_up(cds_total, _reserve_alignment);
3155     _space_list = new VirtualSpaceList(cds_total/wordSize);
3156     _chunk_manager_metadata = new ChunkManager(SpecializedChunk, SmallChunk, MediumChunk);
3157 
3158     if (!_space_list->initialization_succeeded()) {
3159       vm_exit_during_initialization("Unable to dump shared archive.", NULL);
3160     }
3161 
3162 #ifdef _LP64
3163     if (cds_total + compressed_class_space_size() > UnscaledClassSpaceMax) {
3164       vm_exit_during_initialization("Unable to dump shared archive.",
3165           err_msg("Size of archive (" SIZE_FORMAT ") + compressed class space ("
3166                   SIZE_FORMAT ") == total (" SIZE_FORMAT ") is larger than compressed "
3167                   "klass limit: " UINT64_FORMAT, cds_total, compressed_class_space_size(),
3168                   cds_total + compressed_class_space_size(), UnscaledClassSpaceMax));
3169     }
3170 
3171     // Set the compressed klass pointer base so that decoding of these pointers works
3172     // properly when creating the shared archive.
3173     assert(UseCompressedOops && UseCompressedClassPointers,
3174       "UseCompressedOops and UseCompressedClassPointers must be set");
3175     Universe::set_narrow_klass_base((address)_space_list->current_virtual_space()->bottom());
3176     log_develop_trace(gc, metaspace)("Setting_narrow_klass_base to Address: " PTR_FORMAT,
3177                                      p2i(_space_list->current_virtual_space()->bottom()));
3178 
3179     Universe::set_narrow_klass_shift(0);
3180 #endif // _LP64
3181 #endif // INCLUDE_CDS
3182   } else {
3183     // If using shared space, open the file that contains the shared space
3184     // and map in the memory before initializing the rest of metaspace (so
3185     // the addresses don't conflict)
3186     address cds_address = NULL;
3187     if (UseSharedSpaces) {
3188 #if INCLUDE_CDS
3189       FileMapInfo* mapinfo = new FileMapInfo();
3190 
3191       // Open the shared archive file, read and validate the header. If
3192       // initialization fails, shared spaces [UseSharedSpaces] are
3193       // disabled and the file is closed.
3194       // Map in spaces now also
3195       if (mapinfo->initialize() && MetaspaceShared::map_shared_spaces(mapinfo)) {
3196         cds_total = FileMapInfo::shared_spaces_size();
3197         cds_address = (address)mapinfo->header()->region_addr(0);
3198 #ifdef _LP64
3199         if (using_class_space()) {
3200           char* cds_end = (char*)(cds_address + cds_total);
3201           cds_end = (char *)align_ptr_up(cds_end, _reserve_alignment);
3202           // If UseCompressedClassPointers is set then allocate the metaspace area
3203           // above the heap and above the CDS area (if it exists).
3204           allocate_metaspace_compressed_klass_ptrs(cds_end, cds_address);
3205           // Map the shared string space after compressed pointers
3206           // because it relies on compressed class pointers setting to work
3207           mapinfo->map_string_regions();
3208         }
3209 #endif // _LP64
3210       } else {
3211         assert(!mapinfo->is_open() && !UseSharedSpaces,
3212                "archive file not closed or shared spaces not disabled.");
3213       }
3214 #endif // INCLUDE_CDS
3215     }
3216 
3217 #ifdef _LP64
3218     if (!UseSharedSpaces && using_class_space()) {
3219       char* base = (char*)align_ptr_up(Universe::heap()->reserved_region().end(), _reserve_alignment);
3220       allocate_metaspace_compressed_klass_ptrs(base, 0);
3221     }
3222 #endif // _LP64
3223 
3224     // Initialize these before initializing the VirtualSpaceList
3225     _first_chunk_word_size = InitialBootClassLoaderMetaspaceSize / BytesPerWord;
3226     _first_chunk_word_size = align_word_size_up(_first_chunk_word_size);
3227     // Make the first class chunk bigger than a medium chunk so it's not put
3228     // on the medium chunk list.   The next chunk will be small and progress
3229     // from there.  This size calculated by -version.
3230     _first_class_chunk_word_size = MIN2((size_t)MediumChunk*6,
3231                                        (CompressedClassSpaceSize/BytesPerWord)*2);
3232     _first_class_chunk_word_size = align_word_size_up(_first_class_chunk_word_size);
3233     // Arbitrarily set the initial virtual space to a multiple
3234     // of the boot class loader size.
3235     size_t word_size = VIRTUALSPACEMULTIPLIER * _first_chunk_word_size;
3236     word_size = align_size_up(word_size, Metaspace::reserve_alignment_words());
3237 
3238     // Initialize the list of virtual spaces.
3239     _space_list = new VirtualSpaceList(word_size);
3240     _chunk_manager_metadata = new ChunkManager(SpecializedChunk, SmallChunk, MediumChunk);
3241 
3242     if (!_space_list->initialization_succeeded()) {
3243       vm_exit_during_initialization("Unable to setup metadata virtual space list.", NULL);
3244     }
3245   }
3246 
3247   _tracer = new MetaspaceTracer();
3248 }
3249 
3250 void Metaspace::post_initialize() {
3251   MetaspaceGC::post_initialize();
3252 }
3253 
3254 Metachunk* Metaspace::get_initialization_chunk(MetadataType mdtype,
3255                                                size_t chunk_word_size,
3256                                                size_t chunk_bunch) {
3257   // Get a chunk from the chunk freelist
3258   Metachunk* chunk = get_chunk_manager(mdtype)->chunk_freelist_allocate(chunk_word_size);
3259   if (chunk != NULL) {
3260     return chunk;
3261   }
3262 
3263   return get_space_list(mdtype)->get_new_chunk(chunk_word_size, chunk_word_size, chunk_bunch);
3264 }
3265 
3266 void Metaspace::initialize(Mutex* lock, MetaspaceType type) {
3267 
3268   assert(space_list() != NULL,
3269     "Metadata VirtualSpaceList has not been initialized");
3270   assert(chunk_manager_metadata() != NULL,
3271     "Metadata ChunkManager has not been initialized");
3272 
3273   _vsm = new SpaceManager(NonClassType, lock);
3274   if (_vsm == NULL) {
3275     return;
3276   }
3277   size_t word_size;
3278   size_t class_word_size;
3279   vsm()->get_initial_chunk_sizes(type, &word_size, &class_word_size);
3280 
3281   if (using_class_space()) {
3282   assert(class_space_list() != NULL,
3283     "Class VirtualSpaceList has not been initialized");
3284   assert(chunk_manager_class() != NULL,
3285     "Class ChunkManager has not been initialized");
3286 
3287     // Allocate SpaceManager for classes.
3288     _class_vsm = new SpaceManager(ClassType, lock);
3289     if (_class_vsm == NULL) {
3290       return;
3291     }
3292   }
3293 
3294   MutexLockerEx cl(SpaceManager::expand_lock(), Mutex::_no_safepoint_check_flag);
3295 
3296   // Allocate chunk for metadata objects
3297   Metachunk* new_chunk = get_initialization_chunk(NonClassType,
3298                                                   word_size,
3299                                                   vsm()->medium_chunk_bunch());
3300   // For dumping shared archive, report error if allocation has failed.
3301   if (DumpSharedSpaces && new_chunk == NULL) {
3302     report_insufficient_metaspace(MetaspaceAux::committed_bytes() + word_size * BytesPerWord);
3303   }
3304   assert(!DumpSharedSpaces || new_chunk != NULL, "should have enough space for both chunks");
3305   if (new_chunk != NULL) {
3306     // Add to this manager's list of chunks in use and current_chunk().
3307     vsm()->add_chunk(new_chunk, true);
3308   }
3309 
3310   // Allocate chunk for class metadata objects
3311   if (using_class_space()) {
3312     Metachunk* class_chunk = get_initialization_chunk(ClassType,
3313                                                       class_word_size,
3314                                                       class_vsm()->medium_chunk_bunch());
3315     if (class_chunk != NULL) {
3316       class_vsm()->add_chunk(class_chunk, true);
3317     } else {
3318       // For dumping shared archive, report error if allocation has failed.
3319       if (DumpSharedSpaces) {
3320         report_insufficient_metaspace(MetaspaceAux::committed_bytes() + class_word_size * BytesPerWord);
3321       }
3322     }
3323   }
3324 
3325   _alloc_record_head = NULL;
3326   _alloc_record_tail = NULL;
3327 }
3328 
3329 size_t Metaspace::align_word_size_up(size_t word_size) {
3330   size_t byte_size = word_size * wordSize;
3331   return ReservedSpace::allocation_align_size_up(byte_size) / wordSize;
3332 }
3333 
3334 MetaWord* Metaspace::allocate(size_t word_size, MetadataType mdtype) {
3335   // DumpSharedSpaces doesn't use class metadata area (yet)
3336   // Also, don't use class_vsm() unless UseCompressedClassPointers is true.
3337   if (is_class_space_allocation(mdtype)) {
3338     return  class_vsm()->allocate(word_size);
3339   } else {
3340     return  vsm()->allocate(word_size);
3341   }
3342 }
3343 
3344 MetaWord* Metaspace::expand_and_allocate(size_t word_size, MetadataType mdtype) {
3345   size_t delta_bytes = MetaspaceGC::delta_capacity_until_GC(word_size * BytesPerWord);
3346   assert(delta_bytes > 0, "Must be");
3347 
3348   size_t before = 0;
3349   size_t after = 0;
3350   MetaWord* res;
3351   bool incremented;
3352 
3353   // Each thread increments the HWM at most once. Even if the thread fails to increment
3354   // the HWM, an allocation is still attempted. This is because another thread must then
3355   // have incremented the HWM and therefore the allocation might still succeed.
3356   do {
3357     incremented = MetaspaceGC::inc_capacity_until_GC(delta_bytes, &after, &before);
3358     res = allocate(word_size, mdtype);
3359   } while (!incremented && res == NULL);
3360 
3361   if (incremented) {
3362     tracer()->report_gc_threshold(before, after,
3363                                   MetaspaceGCThresholdUpdater::ExpandAndAllocate);
3364     log_trace(gc, metaspace)("Increase capacity to GC from " SIZE_FORMAT " to " SIZE_FORMAT, before, after);
3365   }
3366 
3367   return res;
3368 }
3369 
3370 // Space allocated in the Metaspace.  This may
3371 // be across several metadata virtual spaces.
3372 char* Metaspace::bottom() const {
3373   assert(DumpSharedSpaces, "only useful and valid for dumping shared spaces");
3374   return (char*)vsm()->current_chunk()->bottom();
3375 }
3376 
3377 size_t Metaspace::used_words_slow(MetadataType mdtype) const {
3378   if (mdtype == ClassType) {
3379     return using_class_space() ? class_vsm()->sum_used_in_chunks_in_use() : 0;
3380   } else {
3381     return vsm()->sum_used_in_chunks_in_use();  // includes overhead!
3382   }
3383 }
3384 
3385 size_t Metaspace::free_words_slow(MetadataType mdtype) const {
3386   if (mdtype == ClassType) {
3387     return using_class_space() ? class_vsm()->sum_free_in_chunks_in_use() : 0;
3388   } else {
3389     return vsm()->sum_free_in_chunks_in_use();
3390   }
3391 }
3392 
3393 // Space capacity in the Metaspace.  It includes
3394 // space in the list of chunks from which allocations
3395 // have been made. Don't include space in the global freelist and
3396 // in the space available in the dictionary which
3397 // is already counted in some chunk.
3398 size_t Metaspace::capacity_words_slow(MetadataType mdtype) const {
3399   if (mdtype == ClassType) {
3400     return using_class_space() ? class_vsm()->sum_capacity_in_chunks_in_use() : 0;
3401   } else {
3402     return vsm()->sum_capacity_in_chunks_in_use();
3403   }
3404 }
3405 
3406 size_t Metaspace::used_bytes_slow(MetadataType mdtype) const {
3407   return used_words_slow(mdtype) * BytesPerWord;
3408 }
3409 
3410 size_t Metaspace::capacity_bytes_slow(MetadataType mdtype) const {
3411   return capacity_words_slow(mdtype) * BytesPerWord;
3412 }
3413 
3414 size_t Metaspace::allocated_blocks_bytes() const {
3415   return vsm()->allocated_blocks_bytes() +
3416       (using_class_space() ? class_vsm()->allocated_blocks_bytes() : 0);
3417 }
3418 
3419 size_t Metaspace::allocated_chunks_bytes() const {
3420   return vsm()->allocated_chunks_bytes() +
3421       (using_class_space() ? class_vsm()->allocated_chunks_bytes() : 0);
3422 }
3423 
3424 void Metaspace::deallocate(MetaWord* ptr, size_t word_size, bool is_class) {
3425   assert(!SafepointSynchronize::is_at_safepoint()
3426          || Thread::current()->is_VM_thread(), "should be the VM thread");
3427 
3428   if (DumpSharedSpaces && PrintSharedSpaces) {
3429     record_deallocation(ptr, vsm()->get_raw_word_size(word_size));
3430   }
3431 
3432   MutexLockerEx ml(vsm()->lock(), Mutex::_no_safepoint_check_flag);
3433 
3434   if (word_size < TreeChunk<Metablock, FreeList<Metablock> >::min_size()) {
3435     // Dark matter.  Too small for dictionary.
3436 #ifdef ASSERT
3437     Copy::fill_to_words((HeapWord*)ptr, word_size, 0xf5f5f5f5);
3438 #endif
3439     return;
3440   }
3441   if (is_class && using_class_space()) {
3442     class_vsm()->deallocate(ptr, word_size);
3443   } else {
3444     vsm()->deallocate(ptr, word_size);
3445   }
3446 }
3447 
3448 
3449 MetaWord* Metaspace::allocate(ClassLoaderData* loader_data, size_t word_size,
3450                               bool read_only, MetaspaceObj::Type type, TRAPS) {
3451   if (HAS_PENDING_EXCEPTION) {
3452     assert(false, "Should not allocate with exception pending");
3453     return NULL;  // caller does a CHECK_NULL too
3454   }
3455 
3456   assert(loader_data != NULL, "Should never pass around a NULL loader_data. "
3457         "ClassLoaderData::the_null_class_loader_data() should have been used.");
3458 
3459   // Allocate in metaspaces without taking out a lock, because it deadlocks
3460   // with the SymbolTable_lock.  Dumping is single threaded for now.  We'll have
3461   // to revisit this for application class data sharing.
3462   if (DumpSharedSpaces) {
3463     assert(type > MetaspaceObj::UnknownType && type < MetaspaceObj::_number_of_types, "sanity");
3464     Metaspace* space = read_only ? loader_data->ro_metaspace() : loader_data->rw_metaspace();
3465     MetaWord* result = space->allocate(word_size, NonClassType);
3466     if (result == NULL) {
3467       report_out_of_shared_space(read_only ? SharedReadOnly : SharedReadWrite);
3468     }
3469     if (PrintSharedSpaces) {
3470       space->record_allocation(result, type, space->vsm()->get_raw_word_size(word_size));
3471     }
3472 
3473     // Zero initialize.
3474     Copy::fill_to_words((HeapWord*)result, word_size, 0);
3475 
3476     return result;
3477   }
3478 
3479   MetadataType mdtype = (type == MetaspaceObj::ClassType) ? ClassType : NonClassType;
3480 
3481   // Try to allocate metadata.
3482   MetaWord* result = loader_data->metaspace_non_null()->allocate(word_size, mdtype);
3483 
3484   if (result == NULL) {
3485     tracer()->report_metaspace_allocation_failure(loader_data, word_size, type, mdtype);
3486 
3487     // Allocation failed.
3488     if (is_init_completed()) {
3489       // Only start a GC if the bootstrapping has completed.
3490 
3491       // Try to clean out some memory and retry.
3492       result = Universe::heap()->collector_policy()->satisfy_failed_metadata_allocation(
3493           loader_data, word_size, mdtype);
3494     }
3495   }
3496 
3497   if (result == NULL) {
3498     SpaceManager* sm;
3499     if (is_class_space_allocation(mdtype)) {
3500       sm = loader_data->metaspace_non_null()->class_vsm();
3501     } else {
3502       sm = loader_data->metaspace_non_null()->vsm();
3503     }
3504 
3505     result = sm->get_small_chunk_and_allocate(word_size);
3506 
3507     if (result == NULL) {
3508       report_metadata_oome(loader_data, word_size, type, mdtype, CHECK_NULL);
3509     }
3510   }
3511 
3512   // Zero initialize.
3513   Copy::fill_to_words((HeapWord*)result, word_size, 0);
3514 
3515   return result;
3516 }
3517 
3518 size_t Metaspace::class_chunk_size(size_t word_size) {
3519   assert(using_class_space(), "Has to use class space");
3520   return class_vsm()->calc_chunk_size(word_size);
3521 }
3522 
3523 void Metaspace::report_metadata_oome(ClassLoaderData* loader_data, size_t word_size, MetaspaceObj::Type type, MetadataType mdtype, TRAPS) {
3524   tracer()->report_metadata_oom(loader_data, word_size, type, mdtype);
3525 
3526   // If result is still null, we are out of memory.
3527   LogHandle(gc, metaspace, freelist) log;
3528   if (log.is_trace()) {
3529     log.trace("Metaspace allocation failed for size " SIZE_FORMAT, word_size);
3530     ResourceMark rm;
3531     outputStream* out = log.trace_stream();
3532     if (loader_data->metaspace_or_null() != NULL) {
3533       loader_data->dump(out);
3534     }
3535     MetaspaceAux::dump(out);
3536   }
3537 
3538   bool out_of_compressed_class_space = false;
3539   if (is_class_space_allocation(mdtype)) {
3540     Metaspace* metaspace = loader_data->metaspace_non_null();
3541     out_of_compressed_class_space =
3542       MetaspaceAux::committed_bytes(Metaspace::ClassType) +
3543       (metaspace->class_chunk_size(word_size) * BytesPerWord) >
3544       CompressedClassSpaceSize;
3545   }
3546 
3547   // -XX:+HeapDumpOnOutOfMemoryError and -XX:OnOutOfMemoryError support
3548   const char* space_string = out_of_compressed_class_space ?
3549     "Compressed class space" : "Metaspace";
3550 
3551   report_java_out_of_memory(space_string);
3552 
3553   if (JvmtiExport::should_post_resource_exhausted()) {
3554     JvmtiExport::post_resource_exhausted(
3555         JVMTI_RESOURCE_EXHAUSTED_OOM_ERROR,
3556         space_string);
3557   }
3558 
3559   if (!is_init_completed()) {
3560     vm_exit_during_initialization("OutOfMemoryError", space_string);
3561   }
3562 
3563   if (out_of_compressed_class_space) {
3564     THROW_OOP(Universe::out_of_memory_error_class_metaspace());
3565   } else {
3566     THROW_OOP(Universe::out_of_memory_error_metaspace());
3567   }
3568 }
3569 
3570 const char* Metaspace::metadata_type_name(Metaspace::MetadataType mdtype) {
3571   switch (mdtype) {
3572     case Metaspace::ClassType: return "Class";
3573     case Metaspace::NonClassType: return "Metadata";
3574     default:
3575       assert(false, "Got bad mdtype: %d", (int) mdtype);
3576       return NULL;
3577   }
3578 }
3579 
3580 void Metaspace::record_allocation(void* ptr, MetaspaceObj::Type type, size_t word_size) {
3581   assert(DumpSharedSpaces, "sanity");
3582 
3583   int byte_size = (int)word_size * wordSize;
3584   AllocRecord *rec = new AllocRecord((address)ptr, type, byte_size);
3585 
3586   if (_alloc_record_head == NULL) {
3587     _alloc_record_head = _alloc_record_tail = rec;
3588   } else if (_alloc_record_tail->_ptr + _alloc_record_tail->_byte_size == (address)ptr) {
3589     _alloc_record_tail->_next = rec;
3590     _alloc_record_tail = rec;
3591   } else {
3592     // slow linear search, but this doesn't happen that often, and only when dumping
3593     for (AllocRecord *old = _alloc_record_head; old; old = old->_next) {
3594       if (old->_ptr == ptr) {
3595         assert(old->_type == MetaspaceObj::DeallocatedType, "sanity");
3596         int remain_bytes = old->_byte_size - byte_size;
3597         assert(remain_bytes >= 0, "sanity");
3598         old->_type = type;
3599 
3600         if (remain_bytes == 0) {
3601           delete(rec);
3602         } else {
3603           address remain_ptr = address(ptr) + byte_size;
3604           rec->_ptr = remain_ptr;
3605           rec->_byte_size = remain_bytes;
3606           rec->_type = MetaspaceObj::DeallocatedType;
3607           rec->_next = old->_next;
3608           old->_byte_size = byte_size;
3609           old->_next = rec;
3610         }
3611         return;
3612       }
3613     }
3614     assert(0, "reallocating a freed pointer that was not recorded");
3615   }
3616 }
3617 
3618 void Metaspace::record_deallocation(void* ptr, size_t word_size) {
3619   assert(DumpSharedSpaces, "sanity");
3620 
3621   for (AllocRecord *rec = _alloc_record_head; rec; rec = rec->_next) {
3622     if (rec->_ptr == ptr) {
3623       assert(rec->_byte_size == (int)word_size * wordSize, "sanity");
3624       rec->_type = MetaspaceObj::DeallocatedType;
3625       return;
3626     }
3627   }
3628 
3629   assert(0, "deallocating a pointer that was not recorded");
3630 }
3631 
3632 void Metaspace::iterate(Metaspace::AllocRecordClosure *closure) {
3633   assert(DumpSharedSpaces, "unimplemented for !DumpSharedSpaces");
3634 
3635   address last_addr = (address)bottom();
3636 
3637   for (AllocRecord *rec = _alloc_record_head; rec; rec = rec->_next) {
3638     address ptr = rec->_ptr;
3639     if (last_addr < ptr) {
3640       closure->doit(last_addr, MetaspaceObj::UnknownType, ptr - last_addr);
3641     }
3642     closure->doit(ptr, rec->_type, rec->_byte_size);
3643     last_addr = ptr + rec->_byte_size;
3644   }
3645 
3646   address top = ((address)bottom()) + used_bytes_slow(Metaspace::NonClassType);
3647   if (last_addr < top) {
3648     closure->doit(last_addr, MetaspaceObj::UnknownType, top - last_addr);
3649   }
3650 }
3651 
3652 void Metaspace::purge(MetadataType mdtype) {
3653   get_space_list(mdtype)->purge(get_chunk_manager(mdtype));
3654 }
3655 
3656 void Metaspace::purge() {
3657   MutexLockerEx cl(SpaceManager::expand_lock(),
3658                    Mutex::_no_safepoint_check_flag);
3659   purge(NonClassType);
3660   if (using_class_space()) {
3661     purge(ClassType);
3662   }
3663 }
3664 
3665 void Metaspace::print_on(outputStream* out) const {
3666   // Print both class virtual space counts and metaspace.
3667   if (Verbose) {
3668     vsm()->print_on(out);
3669     if (using_class_space()) {
3670       class_vsm()->print_on(out);
3671     }
3672   }
3673 }
3674 
3675 bool Metaspace::contains(const void* ptr) {
3676   if (UseSharedSpaces && MetaspaceShared::is_in_shared_space(ptr)) {
3677     return true;
3678   }
3679 
3680   if (using_class_space() && get_space_list(ClassType)->contains(ptr)) {
3681      return true;
3682   }
3683 
3684   return get_space_list(NonClassType)->contains(ptr);
3685 }
3686 
3687 void Metaspace::verify() {
3688   vsm()->verify();
3689   if (using_class_space()) {
3690     class_vsm()->verify();
3691   }
3692 }
3693 
3694 void Metaspace::dump(outputStream* const out) const {
3695   out->print_cr("\nVirtual space manager: " INTPTR_FORMAT, p2i(vsm()));
3696   vsm()->dump(out);
3697   if (using_class_space()) {
3698     out->print_cr("\nClass space manager: " INTPTR_FORMAT, p2i(class_vsm()));
3699     class_vsm()->dump(out);
3700   }
3701 }
3702 
3703 /////////////// Unit tests ///////////////
3704 
3705 #ifndef PRODUCT
3706 
3707 class TestMetaspaceAuxTest : AllStatic {
3708  public:
3709   static void test_reserved() {
3710     size_t reserved = MetaspaceAux::reserved_bytes();
3711 
3712     assert(reserved > 0, "assert");
3713 
3714     size_t committed  = MetaspaceAux::committed_bytes();
3715     assert(committed <= reserved, "assert");
3716 
3717     size_t reserved_metadata = MetaspaceAux::reserved_bytes(Metaspace::NonClassType);
3718     assert(reserved_metadata > 0, "assert");
3719     assert(reserved_metadata <= reserved, "assert");
3720 
3721     if (UseCompressedClassPointers) {
3722       size_t reserved_class    = MetaspaceAux::reserved_bytes(Metaspace::ClassType);
3723       assert(reserved_class > 0, "assert");
3724       assert(reserved_class < reserved, "assert");
3725     }
3726   }
3727 
3728   static void test_committed() {
3729     size_t committed = MetaspaceAux::committed_bytes();
3730 
3731     assert(committed > 0, "assert");
3732 
3733     size_t reserved  = MetaspaceAux::reserved_bytes();
3734     assert(committed <= reserved, "assert");
3735 
3736     size_t committed_metadata = MetaspaceAux::committed_bytes(Metaspace::NonClassType);
3737     assert(committed_metadata > 0, "assert");
3738     assert(committed_metadata <= committed, "assert");
3739 
3740     if (UseCompressedClassPointers) {
3741       size_t committed_class    = MetaspaceAux::committed_bytes(Metaspace::ClassType);
3742       assert(committed_class > 0, "assert");
3743       assert(committed_class < committed, "assert");
3744     }
3745   }
3746 
3747   static void test_virtual_space_list_large_chunk() {
3748     VirtualSpaceList* vs_list = new VirtualSpaceList(os::vm_allocation_granularity());
3749     MutexLockerEx cl(SpaceManager::expand_lock(), Mutex::_no_safepoint_check_flag);
3750     // A size larger than VirtualSpaceSize (256k) and add one page to make it _not_ be
3751     // vm_allocation_granularity aligned on Windows.
3752     size_t large_size = (size_t)(2*256*K + (os::vm_page_size()/BytesPerWord));
3753     large_size += (os::vm_page_size()/BytesPerWord);
3754     vs_list->get_new_chunk(large_size, large_size, 0);
3755   }
3756 
3757   static void test() {
3758     test_reserved();
3759     test_committed();
3760     test_virtual_space_list_large_chunk();
3761   }
3762 };
3763 
3764 void TestMetaspaceAux_test() {
3765   TestMetaspaceAuxTest::test();
3766 }
3767 
3768 class TestVirtualSpaceNodeTest {
3769   static void chunk_up(size_t words_left, size_t& num_medium_chunks,
3770                                           size_t& num_small_chunks,
3771                                           size_t& num_specialized_chunks) {
3772     num_medium_chunks = words_left / MediumChunk;
3773     words_left = words_left % MediumChunk;
3774 
3775     num_small_chunks = words_left / SmallChunk;
3776     words_left = words_left % SmallChunk;
3777     // how many specialized chunks can we get?
3778     num_specialized_chunks = words_left / SpecializedChunk;
3779     assert(words_left % SpecializedChunk == 0, "should be nothing left");
3780   }
3781 
3782  public:
3783   static void test() {
3784     MutexLockerEx ml(SpaceManager::expand_lock(), Mutex::_no_safepoint_check_flag);
3785     const size_t vsn_test_size_words = MediumChunk  * 4;
3786     const size_t vsn_test_size_bytes = vsn_test_size_words * BytesPerWord;
3787 
3788     // The chunk sizes must be multiples of eachother, or this will fail
3789     STATIC_ASSERT(MediumChunk % SmallChunk == 0);
3790     STATIC_ASSERT(SmallChunk % SpecializedChunk == 0);
3791 
3792     { // No committed memory in VSN
3793       ChunkManager cm(SpecializedChunk, SmallChunk, MediumChunk);
3794       VirtualSpaceNode vsn(vsn_test_size_bytes);
3795       vsn.initialize();
3796       vsn.retire(&cm);
3797       assert(cm.sum_free_chunks_count() == 0, "did not commit any memory in the VSN");
3798     }
3799 
3800     { // All of VSN is committed, half is used by chunks
3801       ChunkManager cm(SpecializedChunk, SmallChunk, MediumChunk);
3802       VirtualSpaceNode vsn(vsn_test_size_bytes);
3803       vsn.initialize();
3804       vsn.expand_by(vsn_test_size_words, vsn_test_size_words);
3805       vsn.get_chunk_vs(MediumChunk);
3806       vsn.get_chunk_vs(MediumChunk);
3807       vsn.retire(&cm);
3808       assert(cm.sum_free_chunks_count() == 2, "should have been memory left for 2 medium chunks");
3809       assert(cm.sum_free_chunks() == 2*MediumChunk, "sizes should add up");
3810     }
3811 
3812     const size_t page_chunks = 4 * (size_t)os::vm_page_size() / BytesPerWord;
3813     // This doesn't work for systems with vm_page_size >= 16K.
3814     if (page_chunks < MediumChunk) {
3815       // 4 pages of VSN is committed, some is used by chunks
3816       ChunkManager cm(SpecializedChunk, SmallChunk, MediumChunk);
3817       VirtualSpaceNode vsn(vsn_test_size_bytes);
3818 
3819       vsn.initialize();
3820       vsn.expand_by(page_chunks, page_chunks);
3821       vsn.get_chunk_vs(SmallChunk);
3822       vsn.get_chunk_vs(SpecializedChunk);
3823       vsn.retire(&cm);
3824 
3825       // committed - used = words left to retire
3826       const size_t words_left = page_chunks - SmallChunk - SpecializedChunk;
3827 
3828       size_t num_medium_chunks, num_small_chunks, num_spec_chunks;
3829       chunk_up(words_left, num_medium_chunks, num_small_chunks, num_spec_chunks);
3830 
3831       assert(num_medium_chunks == 0, "should not get any medium chunks");
3832       assert(cm.sum_free_chunks_count() == (num_small_chunks + num_spec_chunks), "should be space for 3 chunks");
3833       assert(cm.sum_free_chunks() == words_left, "sizes should add up");
3834     }
3835 
3836     { // Half of VSN is committed, a humongous chunk is used
3837       ChunkManager cm(SpecializedChunk, SmallChunk, MediumChunk);
3838       VirtualSpaceNode vsn(vsn_test_size_bytes);
3839       vsn.initialize();
3840       vsn.expand_by(MediumChunk * 2, MediumChunk * 2);
3841       vsn.get_chunk_vs(MediumChunk + SpecializedChunk); // Humongous chunks will be aligned up to MediumChunk + SpecializedChunk
3842       vsn.retire(&cm);
3843 
3844       const size_t words_left = MediumChunk * 2 - (MediumChunk + SpecializedChunk);
3845       size_t num_medium_chunks, num_small_chunks, num_spec_chunks;
3846       chunk_up(words_left, num_medium_chunks, num_small_chunks, num_spec_chunks);
3847 
3848       assert(num_medium_chunks == 0, "should not get any medium chunks");
3849       assert(cm.sum_free_chunks_count() == (num_small_chunks + num_spec_chunks), "should be space for 3 chunks");
3850       assert(cm.sum_free_chunks() == words_left, "sizes should add up");
3851     }
3852 
3853   }
3854 
3855 #define assert_is_available_positive(word_size) \
3856   assert(vsn.is_available(word_size), \
3857          #word_size ": " PTR_FORMAT " bytes were not available in " \
3858          "VirtualSpaceNode [" PTR_FORMAT ", " PTR_FORMAT ")", \
3859          (uintptr_t)(word_size * BytesPerWord), p2i(vsn.bottom()), p2i(vsn.end()));
3860 
3861 #define assert_is_available_negative(word_size) \
3862   assert(!vsn.is_available(word_size), \
3863          #word_size ": " PTR_FORMAT " bytes should not be available in " \
3864          "VirtualSpaceNode [" PTR_FORMAT ", " PTR_FORMAT ")", \
3865          (uintptr_t)(word_size * BytesPerWord), p2i(vsn.bottom()), p2i(vsn.end()));
3866 
3867   static void test_is_available_positive() {
3868     // Reserve some memory.
3869     VirtualSpaceNode vsn(os::vm_allocation_granularity());
3870     assert(vsn.initialize(), "Failed to setup VirtualSpaceNode");
3871 
3872     // Commit some memory.
3873     size_t commit_word_size = os::vm_allocation_granularity() / BytesPerWord;
3874     bool expanded = vsn.expand_by(commit_word_size, commit_word_size);
3875     assert(expanded, "Failed to commit");
3876 
3877     // Check that is_available accepts the committed size.
3878     assert_is_available_positive(commit_word_size);
3879 
3880     // Check that is_available accepts half the committed size.
3881     size_t expand_word_size = commit_word_size / 2;
3882     assert_is_available_positive(expand_word_size);
3883   }
3884 
3885   static void test_is_available_negative() {
3886     // Reserve some memory.
3887     VirtualSpaceNode vsn(os::vm_allocation_granularity());
3888     assert(vsn.initialize(), "Failed to setup VirtualSpaceNode");
3889 
3890     // Commit some memory.
3891     size_t commit_word_size = os::vm_allocation_granularity() / BytesPerWord;
3892     bool expanded = vsn.expand_by(commit_word_size, commit_word_size);
3893     assert(expanded, "Failed to commit");
3894 
3895     // Check that is_available doesn't accept a too large size.
3896     size_t two_times_commit_word_size = commit_word_size * 2;
3897     assert_is_available_negative(two_times_commit_word_size);
3898   }
3899 
3900   static void test_is_available_overflow() {
3901     // Reserve some memory.
3902     VirtualSpaceNode vsn(os::vm_allocation_granularity());
3903     assert(vsn.initialize(), "Failed to setup VirtualSpaceNode");
3904 
3905     // Commit some memory.
3906     size_t commit_word_size = os::vm_allocation_granularity() / BytesPerWord;
3907     bool expanded = vsn.expand_by(commit_word_size, commit_word_size);
3908     assert(expanded, "Failed to commit");
3909 
3910     // Calculate a size that will overflow the virtual space size.
3911     void* virtual_space_max = (void*)(uintptr_t)-1;
3912     size_t bottom_to_max = pointer_delta(virtual_space_max, vsn.bottom(), 1);
3913     size_t overflow_size = bottom_to_max + BytesPerWord;
3914     size_t overflow_word_size = overflow_size / BytesPerWord;
3915 
3916     // Check that is_available can handle the overflow.
3917     assert_is_available_negative(overflow_word_size);
3918   }
3919 
3920   static void test_is_available() {
3921     TestVirtualSpaceNodeTest::test_is_available_positive();
3922     TestVirtualSpaceNodeTest::test_is_available_negative();
3923     TestVirtualSpaceNodeTest::test_is_available_overflow();
3924   }
3925 };
3926 
3927 void TestVirtualSpaceNode_test() {
3928   TestVirtualSpaceNodeTest::test();
3929   TestVirtualSpaceNodeTest::test_is_available();
3930 }
3931 #endif
--- EOF ---