1 /*
   2  * Copyright (c) 2011, 2014, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 #include "precompiled.hpp"
  25 #include "gc_interface/collectedHeap.hpp"
  26 #include "memory/allocation.hpp"
  27 #include "memory/binaryTreeDictionary.hpp"
  28 #include "memory/freeList.hpp"
  29 #include "memory/collectorPolicy.hpp"
  30 #include "memory/filemap.hpp"
  31 #include "memory/freeList.hpp"
  32 #include "memory/gcLocker.hpp"
  33 #include "memory/metachunk.hpp"
  34 #include "memory/metaspace.hpp"
  35 #include "memory/metaspaceGCThresholdUpdater.hpp"
  36 #include "memory/metaspaceShared.hpp"
  37 #include "memory/metaspaceTracer.hpp"
  38 #include "memory/resourceArea.hpp"
  39 #include "memory/universe.hpp"
  40 #include "runtime/atomic.inline.hpp"
  41 #include "runtime/globals.hpp"
  42 #include "runtime/init.hpp"
  43 #include "runtime/java.hpp"
  44 #include "runtime/mutex.hpp"
  45 #include "runtime/orderAccess.inline.hpp"
  46 #include "services/memTracker.hpp"
  47 #include "services/memoryService.hpp"
  48 #include "utilities/copy.hpp"
  49 #include "utilities/debug.hpp"
  50 
  51 PRAGMA_FORMAT_MUTE_WARNINGS_FOR_GCC
  52 
  53 typedef BinaryTreeDictionary<Metablock, FreeList<Metablock> > BlockTreeDictionary;
  54 typedef BinaryTreeDictionary<Metachunk, FreeList<Metachunk> > ChunkTreeDictionary;
  55 
  56 // Set this constant to enable slow integrity checking of the free chunk lists
  57 const bool metaspace_slow_verify = false;
  58 
  59 size_t const allocation_from_dictionary_limit = 4 * K;
  60 
  61 MetaWord* last_allocated = 0;
  62 
  63 size_t Metaspace::_compressed_class_space_size;
  64 const MetaspaceTracer* Metaspace::_tracer = NULL;
  65 
  66 // Used in declarations in SpaceManager and ChunkManager
  67 enum ChunkIndex {
  68   ZeroIndex = 0,
  69   SpecializedIndex = ZeroIndex,
  70   SmallIndex = SpecializedIndex + 1,
  71   MediumIndex = SmallIndex + 1,
  72   HumongousIndex = MediumIndex + 1,
  73   NumberOfFreeLists = 3,
  74   NumberOfInUseLists = 4
  75 };
  76 
  77 enum ChunkSizes {    // in words.
  78   ClassSpecializedChunk = 128,
  79   SpecializedChunk = 128,
  80   ClassSmallChunk = 256,
  81   SmallChunk = 512,
  82   ClassMediumChunk = 4 * K,
  83   MediumChunk = 8 * K
  84 };
  85 
  86 static ChunkIndex next_chunk_index(ChunkIndex i) {
  87   assert(i < NumberOfInUseLists, "Out of bound");
  88   return (ChunkIndex) (i+1);
  89 }
  90 
  91 volatile intptr_t MetaspaceGC::_capacity_until_GC = 0;
  92 uint MetaspaceGC::_shrink_factor = 0;
  93 bool MetaspaceGC::_should_concurrent_collect = false;
  94 
  95 typedef class FreeList<Metachunk> ChunkList;
  96 
  97 // Manages the global free lists of chunks.
  98 class ChunkManager : public CHeapObj<mtInternal> {
  99   friend class TestVirtualSpaceNodeTest;
 100 
 101   // Free list of chunks of different sizes.
 102   //   SpecializedChunk
 103   //   SmallChunk
 104   //   MediumChunk
 105   //   HumongousChunk
 106   ChunkList _free_chunks[NumberOfFreeLists];
 107 
 108   //   HumongousChunk
 109   ChunkTreeDictionary _humongous_dictionary;
 110 
 111   // ChunkManager in all lists of this type
 112   size_t _free_chunks_total;
 113   size_t _free_chunks_count;
 114 
 115   void dec_free_chunks_total(size_t v) {
 116     assert(_free_chunks_count > 0 &&
 117              _free_chunks_total > 0,
 118              "About to go negative");
 119     Atomic::add_ptr(-1, &_free_chunks_count);
 120     jlong minus_v = (jlong) - (jlong) v;
 121     Atomic::add_ptr(minus_v, &_free_chunks_total);
 122   }
 123 
 124   // Debug support
 125 
 126   size_t sum_free_chunks();
 127   size_t sum_free_chunks_count();
 128 
 129   void locked_verify_free_chunks_total();
 130   void slow_locked_verify_free_chunks_total() {
 131     if (metaspace_slow_verify) {
 132       locked_verify_free_chunks_total();
 133     }
 134   }
 135   void locked_verify_free_chunks_count();
 136   void slow_locked_verify_free_chunks_count() {
 137     if (metaspace_slow_verify) {
 138       locked_verify_free_chunks_count();
 139     }
 140   }
 141   void verify_free_chunks_count();
 142 
 143  public:
 144 
 145   ChunkManager(size_t specialized_size, size_t small_size, size_t medium_size)
 146       : _free_chunks_total(0), _free_chunks_count(0) {
 147     _free_chunks[SpecializedIndex].set_size(specialized_size);
 148     _free_chunks[SmallIndex].set_size(small_size);
 149     _free_chunks[MediumIndex].set_size(medium_size);
 150   }
 151 
 152   // add or delete (return) a chunk to the global freelist.
 153   Metachunk* chunk_freelist_allocate(size_t word_size);
 154 
 155   // Map a size to a list index assuming that there are lists
 156   // for special, small, medium, and humongous chunks.
 157   static ChunkIndex list_index(size_t size);
 158 
 159   // Remove the chunk from its freelist.  It is
 160   // expected to be on one of the _free_chunks[] lists.
 161   void remove_chunk(Metachunk* chunk);
 162 
 163   // Add the simple linked list of chunks to the freelist of chunks
 164   // of type index.
 165   void return_chunks(ChunkIndex index, Metachunk* chunks);
 166 
 167   // Total of the space in the free chunks list
 168   size_t free_chunks_total_words();
 169   size_t free_chunks_total_bytes();
 170 
 171   // Number of chunks in the free chunks list
 172   size_t free_chunks_count();
 173 
 174   void inc_free_chunks_total(size_t v, size_t count = 1) {
 175     Atomic::add_ptr(count, &_free_chunks_count);
 176     Atomic::add_ptr(v, &_free_chunks_total);
 177   }
 178   ChunkTreeDictionary* humongous_dictionary() {
 179     return &_humongous_dictionary;
 180   }
 181 
 182   ChunkList* free_chunks(ChunkIndex index);
 183 
 184   // Returns the list for the given chunk word size.
 185   ChunkList* find_free_chunks_list(size_t word_size);
 186 
 187   // Remove from a list by size.  Selects list based on size of chunk.
 188   Metachunk* free_chunks_get(size_t chunk_word_size);
 189 
 190 #define index_bounds_check(index)                                         \
 191   assert(index == SpecializedIndex ||                                     \
 192          index == SmallIndex ||                                           \
 193          index == MediumIndex ||                                          \
 194          index == HumongousIndex, err_msg("Bad index: %d", (int) index))
 195 
 196   size_t num_free_chunks(ChunkIndex index) const {
 197     index_bounds_check(index);
 198 
 199     if (index == HumongousIndex) {
 200       return _humongous_dictionary.total_free_blocks();
 201     }
 202 
 203     ssize_t count = _free_chunks[index].count();
 204     return count == -1 ? 0 : (size_t) count;
 205   }
 206 
 207   size_t size_free_chunks_in_bytes(ChunkIndex index) const {
 208     index_bounds_check(index);
 209 
 210     size_t word_size = 0;
 211     if (index == HumongousIndex) {
 212       word_size = _humongous_dictionary.total_size();
 213     } else {
 214       const size_t size_per_chunk_in_words = _free_chunks[index].size();
 215       word_size = size_per_chunk_in_words * num_free_chunks(index);
 216     }
 217 
 218     return word_size * BytesPerWord;
 219   }
 220 
 221   MetaspaceChunkFreeListSummary chunk_free_list_summary() const {
 222     return MetaspaceChunkFreeListSummary(num_free_chunks(SpecializedIndex),
 223                                          num_free_chunks(SmallIndex),
 224                                          num_free_chunks(MediumIndex),
 225                                          num_free_chunks(HumongousIndex),
 226                                          size_free_chunks_in_bytes(SpecializedIndex),
 227                                          size_free_chunks_in_bytes(SmallIndex),
 228                                          size_free_chunks_in_bytes(MediumIndex),
 229                                          size_free_chunks_in_bytes(HumongousIndex));
 230   }
 231 
 232   // Debug support
 233   void verify();
 234   void slow_verify() {
 235     if (metaspace_slow_verify) {
 236       verify();
 237     }
 238   }
 239   void locked_verify();
 240   void slow_locked_verify() {
 241     if (metaspace_slow_verify) {
 242       locked_verify();
 243     }
 244   }
 245   void verify_free_chunks_total();
 246 
 247   void locked_print_free_chunks(outputStream* st);
 248   void locked_print_sum_free_chunks(outputStream* st);
 249 
 250   void print_on(outputStream* st) const;
 251 };
 252 
 253 // Used to manage the free list of Metablocks (a block corresponds
 254 // to the allocation of a quantum of metadata).
 255 class BlockFreelist VALUE_OBJ_CLASS_SPEC {
 256   BlockTreeDictionary* _dictionary;
 257 
 258   // Only allocate and split from freelist if the size of the allocation
 259   // is at least 1/4th the size of the available block.
 260   const static int WasteMultiplier = 4;
 261 
 262   // Accessors
 263   BlockTreeDictionary* dictionary() const { return _dictionary; }
 264 
 265  public:
 266   BlockFreelist();
 267   ~BlockFreelist();
 268 
 269   // Get and return a block to the free list
 270   MetaWord* get_block(size_t word_size);
 271   void return_block(MetaWord* p, size_t word_size);
 272 
 273   size_t total_size() {
 274   if (dictionary() == NULL) {
 275     return 0;
 276   } else {
 277     return dictionary()->total_size();
 278   }
 279 }
 280 
 281   void print_on(outputStream* st) const;
 282 };
 283 
 284 // A VirtualSpaceList node.
 285 class VirtualSpaceNode : public CHeapObj<mtClass> {
 286   friend class VirtualSpaceList;
 287 
 288   // Link to next VirtualSpaceNode
 289   VirtualSpaceNode* _next;
 290 
 291   // total in the VirtualSpace
 292   MemRegion _reserved;
 293   ReservedSpace _rs;
 294   VirtualSpace _virtual_space;
 295   MetaWord* _top;
 296   // count of chunks contained in this VirtualSpace
 297   uintx _container_count;
 298 
 299   // Convenience functions to access the _virtual_space
 300   char* low()  const { return virtual_space()->low(); }
 301   char* high() const { return virtual_space()->high(); }
 302 
 303   // The first Metachunk will be allocated at the bottom of the
 304   // VirtualSpace
 305   Metachunk* first_chunk() { return (Metachunk*) bottom(); }
 306 
 307   // Committed but unused space in the virtual space
 308   size_t free_words_in_vs() const;
 309  public:
 310 
 311   VirtualSpaceNode(size_t byte_size);
 312   VirtualSpaceNode(ReservedSpace rs) : _top(NULL), _next(NULL), _rs(rs), _container_count(0) {}
 313   ~VirtualSpaceNode();
 314 
 315   // Convenience functions for logical bottom and end
 316   MetaWord* bottom() const { return (MetaWord*) _virtual_space.low(); }
 317   MetaWord* end() const { return (MetaWord*) _virtual_space.high(); }
 318 
 319   bool contains(const void* ptr) { return ptr >= low() && ptr < high(); }
 320 
 321   size_t reserved_words() const  { return _virtual_space.reserved_size() / BytesPerWord; }
 322   size_t committed_words() const { return _virtual_space.actual_committed_size() / BytesPerWord; }
 323 
 324   bool is_pre_committed() const { return _virtual_space.special(); }
 325 
 326   // address of next available space in _virtual_space;
 327   // Accessors
 328   VirtualSpaceNode* next() { return _next; }
 329   void set_next(VirtualSpaceNode* v) { _next = v; }
 330 
 331   void set_reserved(MemRegion const v) { _reserved = v; }
 332   void set_top(MetaWord* v) { _top = v; }
 333 
 334   // Accessors
 335   MemRegion* reserved() { return &_reserved; }
 336   VirtualSpace* virtual_space() const { return (VirtualSpace*) &_virtual_space; }
 337 
 338   // Returns true if "word_size" is available in the VirtualSpace
 339   bool is_available(size_t word_size) { return word_size <= pointer_delta(end(), _top, sizeof(MetaWord)); }
 340 
 341   MetaWord* top() const { return _top; }
 342   void inc_top(size_t word_size) { _top += word_size; }
 343 
 344   uintx container_count() { return _container_count; }
 345   void inc_container_count();
 346   void dec_container_count();
 347 #ifdef ASSERT
 348   uint container_count_slow();
 349   void verify_container_count();
 350 #endif
 351 
 352   // used and capacity in this single entry in the list
 353   size_t used_words_in_vs() const;
 354   size_t capacity_words_in_vs() const;
 355 
 356   bool initialize();
 357 
 358   // get space from the virtual space
 359   Metachunk* take_from_committed(size_t chunk_word_size);
 360 
 361   // Allocate a chunk from the virtual space and return it.
 362   Metachunk* get_chunk_vs(size_t chunk_word_size);
 363 
 364   // Expands/shrinks the committed space in a virtual space.  Delegates
 365   // to Virtualspace
 366   bool expand_by(size_t min_words, size_t preferred_words);
 367 
 368   // In preparation for deleting this node, remove all the chunks
 369   // in the node from any freelist.
 370   void purge(ChunkManager* chunk_manager);
 371 
 372   // If an allocation doesn't fit in the current node a new node is created.
 373   // Allocate chunks out of the remaining committed space in this node
 374   // to avoid wasting that memory.
 375   // This always adds up because all the chunk sizes are multiples of
 376   // the smallest chunk size.
 377   void retire(ChunkManager* chunk_manager);
 378 
 379 #ifdef ASSERT
 380   // Debug support
 381   void mangle();
 382 #endif
 383 
 384   void print_on(outputStream* st) const;
 385 };
 386 
 387 #define assert_is_ptr_aligned(ptr, alignment) \
 388   assert(is_ptr_aligned(ptr, alignment),      \
 389     err_msg(PTR_FORMAT " is not aligned to "  \
 390       SIZE_FORMAT, ptr, alignment))
 391 
 392 #define assert_is_size_aligned(size, alignment) \
 393   assert(is_size_aligned(size, alignment),      \
 394     err_msg(SIZE_FORMAT " is not aligned to "   \
 395        SIZE_FORMAT, size, alignment))
 396 
 397 
 398 // Decide if large pages should be committed when the memory is reserved.
 399 static bool should_commit_large_pages_when_reserving(size_t bytes) {
 400   if (UseLargePages && UseLargePagesInMetaspace && !os::can_commit_large_page_memory()) {
 401     size_t words = bytes / BytesPerWord;
 402     bool is_class = false; // We never reserve large pages for the class space.
 403     if (MetaspaceGC::can_expand(words, is_class) &&
 404         MetaspaceGC::allowed_expansion() >= words) {
 405       return true;
 406     }
 407   }
 408 
 409   return false;
 410 }
 411 
 412   // byte_size is the size of the associated virtualspace.
 413 VirtualSpaceNode::VirtualSpaceNode(size_t bytes) : _top(NULL), _next(NULL), _rs(), _container_count(0) {
 414   assert_is_size_aligned(bytes, Metaspace::reserve_alignment());
 415 
 416   // This allocates memory with mmap.  For DumpSharedspaces, try to reserve
 417   // configurable address, generally at the top of the Java heap so other
 418   // memory addresses don't conflict.
 419   if (DumpSharedSpaces) {
 420     bool large_pages = false; // No large pages when dumping the CDS archive.
 421     char* shared_base = (char*)align_ptr_up((char*)SharedBaseAddress, Metaspace::reserve_alignment());
 422 
 423     _rs = ReservedSpace(bytes, Metaspace::reserve_alignment(), large_pages, shared_base, 0);
 424     if (_rs.is_reserved()) {
 425       assert(shared_base == 0 || _rs.base() == shared_base, "should match");
 426     } else {
 427       // Get a mmap region anywhere if the SharedBaseAddress fails.
 428       _rs = ReservedSpace(bytes, Metaspace::reserve_alignment(), large_pages);
 429     }
 430     MetaspaceShared::set_shared_rs(&_rs);
 431   } else {
 432     bool large_pages = should_commit_large_pages_when_reserving(bytes);
 433 
 434     _rs = ReservedSpace(bytes, Metaspace::reserve_alignment(), large_pages);
 435   }
 436 
 437   if (_rs.is_reserved()) {
 438     assert(_rs.base() != NULL, "Catch if we get a NULL address");
 439     assert(_rs.size() != 0, "Catch if we get a 0 size");
 440     assert_is_ptr_aligned(_rs.base(), Metaspace::reserve_alignment());
 441     assert_is_size_aligned(_rs.size(), Metaspace::reserve_alignment());
 442 
 443     MemTracker::record_virtual_memory_type((address)_rs.base(), mtClass);
 444   }
 445 }
 446 
 447 void VirtualSpaceNode::purge(ChunkManager* chunk_manager) {
 448   Metachunk* chunk = first_chunk();
 449   Metachunk* invalid_chunk = (Metachunk*) top();
 450   while (chunk < invalid_chunk ) {
 451     assert(chunk->is_tagged_free(), "Should be tagged free");
 452     MetaWord* next = ((MetaWord*)chunk) + chunk->word_size();
 453     chunk_manager->remove_chunk(chunk);
 454     assert(chunk->next() == NULL &&
 455            chunk->prev() == NULL,
 456            "Was not removed from its list");
 457     chunk = (Metachunk*) next;
 458   }
 459 }
 460 
 461 #ifdef ASSERT
 462 uint VirtualSpaceNode::container_count_slow() {
 463   uint count = 0;
 464   Metachunk* chunk = first_chunk();
 465   Metachunk* invalid_chunk = (Metachunk*) top();
 466   while (chunk < invalid_chunk ) {
 467     MetaWord* next = ((MetaWord*)chunk) + chunk->word_size();
 468     // Don't count the chunks on the free lists.  Those are
 469     // still part of the VirtualSpaceNode but not currently
 470     // counted.
 471     if (!chunk->is_tagged_free()) {
 472       count++;
 473     }
 474     chunk = (Metachunk*) next;
 475   }
 476   return count;
 477 }
 478 #endif
 479 
 480 // List of VirtualSpaces for metadata allocation.
 481 class VirtualSpaceList : public CHeapObj<mtClass> {
 482   friend class VirtualSpaceNode;
 483 
 484   enum VirtualSpaceSizes {
 485     VirtualSpaceSize = 256 * K
 486   };
 487 
 488   // Head of the list
 489   VirtualSpaceNode* _virtual_space_list;
 490   // virtual space currently being used for allocations
 491   VirtualSpaceNode* _current_virtual_space;
 492 
 493   // Is this VirtualSpaceList used for the compressed class space
 494   bool _is_class;
 495 
 496   // Sum of reserved and committed memory in the virtual spaces
 497   size_t _reserved_words;
 498   size_t _committed_words;
 499 
 500   // Number of virtual spaces
 501   size_t _virtual_space_count;
 502 
 503   ~VirtualSpaceList();
 504 
 505   VirtualSpaceNode* virtual_space_list() const { return _virtual_space_list; }
 506 
 507   void set_virtual_space_list(VirtualSpaceNode* v) {
 508     _virtual_space_list = v;
 509   }
 510   void set_current_virtual_space(VirtualSpaceNode* v) {
 511     _current_virtual_space = v;
 512   }
 513 
 514   void link_vs(VirtualSpaceNode* new_entry);
 515 
 516   // Get another virtual space and add it to the list.  This
 517   // is typically prompted by a failed attempt to allocate a chunk
 518   // and is typically followed by the allocation of a chunk.
 519   bool create_new_virtual_space(size_t vs_word_size);
 520 
 521   // Chunk up the unused committed space in the current
 522   // virtual space and add the chunks to the free list.
 523   void retire_current_virtual_space();
 524 
 525  public:
 526   VirtualSpaceList(size_t word_size);
 527   VirtualSpaceList(ReservedSpace rs);
 528 
 529   size_t free_bytes();
 530 
 531   Metachunk* get_new_chunk(size_t word_size,
 532                            size_t grow_chunks_by_words,
 533                            size_t medium_chunk_bunch);
 534 
 535   bool expand_node_by(VirtualSpaceNode* node,
 536                       size_t min_words,
 537                       size_t preferred_words);
 538 
 539   bool expand_by(size_t min_words,
 540                  size_t preferred_words);
 541 
 542   VirtualSpaceNode* current_virtual_space() {
 543     return _current_virtual_space;
 544   }
 545 
 546   bool is_class() const { return _is_class; }
 547 
 548   bool initialization_succeeded() { return _virtual_space_list != NULL; }
 549 
 550   size_t reserved_words()  { return _reserved_words; }
 551   size_t reserved_bytes()  { return reserved_words() * BytesPerWord; }
 552   size_t committed_words() { return _committed_words; }
 553   size_t committed_bytes() { return committed_words() * BytesPerWord; }
 554 
 555   void inc_reserved_words(size_t v);
 556   void dec_reserved_words(size_t v);
 557   void inc_committed_words(size_t v);
 558   void dec_committed_words(size_t v);
 559   void inc_virtual_space_count();
 560   void dec_virtual_space_count();
 561 
 562   bool contains(const void* ptr);
 563 
 564   // Unlink empty VirtualSpaceNodes and free it.
 565   void purge(ChunkManager* chunk_manager);
 566 
 567   void print_on(outputStream* st) const;
 568 
 569   class VirtualSpaceListIterator : public StackObj {
 570     VirtualSpaceNode* _virtual_spaces;
 571    public:
 572     VirtualSpaceListIterator(VirtualSpaceNode* virtual_spaces) :
 573       _virtual_spaces(virtual_spaces) {}
 574 
 575     bool repeat() {
 576       return _virtual_spaces != NULL;
 577     }
 578 
 579     VirtualSpaceNode* get_next() {
 580       VirtualSpaceNode* result = _virtual_spaces;
 581       if (_virtual_spaces != NULL) {
 582         _virtual_spaces = _virtual_spaces->next();
 583       }
 584       return result;
 585     }
 586   };
 587 };
 588 
 589 class Metadebug : AllStatic {
 590   // Debugging support for Metaspaces
 591   static int _allocation_fail_alot_count;
 592 
 593  public:
 594 
 595   static void init_allocation_fail_alot_count();
 596 #ifdef ASSERT
 597   static bool test_metadata_failure();
 598 #endif
 599 };
 600 
 601 int Metadebug::_allocation_fail_alot_count = 0;
 602 
 603 //  SpaceManager - used by Metaspace to handle allocations
 604 class SpaceManager : public CHeapObj<mtClass> {
 605   friend class Metaspace;
 606   friend class Metadebug;
 607 
 608  private:
 609 
 610   // protects allocations
 611   Mutex* const _lock;
 612 
 613   // Type of metadata allocated.
 614   Metaspace::MetadataType _mdtype;
 615 
 616   // List of chunks in use by this SpaceManager.  Allocations
 617   // are done from the current chunk.  The list is used for deallocating
 618   // chunks when the SpaceManager is freed.
 619   Metachunk* _chunks_in_use[NumberOfInUseLists];
 620   Metachunk* _current_chunk;
 621 
 622   // Number of small chunks to allocate to a manager
 623   // If class space manager, small chunks are unlimited
 624   static uint const _small_chunk_limit;
 625 
 626   // Sum of all space in allocated chunks
 627   size_t _allocated_blocks_words;
 628 
 629   // Sum of all allocated chunks
 630   size_t _allocated_chunks_words;
 631   size_t _allocated_chunks_count;
 632 
 633   // Free lists of blocks are per SpaceManager since they
 634   // are assumed to be in chunks in use by the SpaceManager
 635   // and all chunks in use by a SpaceManager are freed when
 636   // the class loader using the SpaceManager is collected.
 637   BlockFreelist _block_freelists;
 638 
 639   // protects virtualspace and chunk expansions
 640   static const char*  _expand_lock_name;
 641   static const int    _expand_lock_rank;
 642   static Mutex* const _expand_lock;
 643 
 644  private:
 645   // Accessors
 646   Metachunk* chunks_in_use(ChunkIndex index) const { return _chunks_in_use[index]; }
 647   void set_chunks_in_use(ChunkIndex index, Metachunk* v) {
 648     _chunks_in_use[index] = v;
 649   }
 650 
 651   BlockFreelist* block_freelists() const {
 652     return (BlockFreelist*) &_block_freelists;
 653   }
 654 
 655   Metaspace::MetadataType mdtype() { return _mdtype; }
 656 
 657   VirtualSpaceList* vs_list()   const { return Metaspace::get_space_list(_mdtype); }
 658   ChunkManager* chunk_manager() const { return Metaspace::get_chunk_manager(_mdtype); }
 659 
 660   Metachunk* current_chunk() const { return _current_chunk; }
 661   void set_current_chunk(Metachunk* v) {
 662     _current_chunk = v;
 663   }
 664 
 665   Metachunk* find_current_chunk(size_t word_size);
 666 
 667   // Add chunk to the list of chunks in use
 668   void add_chunk(Metachunk* v, bool make_current);
 669   void retire_current_chunk();
 670 
 671   Mutex* lock() const { return _lock; }
 672 
 673   const char* chunk_size_name(ChunkIndex index) const;
 674 
 675  protected:
 676   void initialize();
 677 
 678  public:
 679   SpaceManager(Metaspace::MetadataType mdtype,
 680                Mutex* lock);
 681   ~SpaceManager();
 682 
 683   enum ChunkMultiples {
 684     MediumChunkMultiple = 4
 685   };
 686 
 687   bool is_class() { return _mdtype == Metaspace::ClassType; }
 688 
 689   // Accessors
 690   size_t specialized_chunk_size() { return (size_t) is_class() ? ClassSpecializedChunk : SpecializedChunk; }
 691   size_t small_chunk_size()       { return (size_t) is_class() ? ClassSmallChunk : SmallChunk; }
 692   size_t medium_chunk_size()      { return (size_t) is_class() ? ClassMediumChunk : MediumChunk; }
 693   size_t medium_chunk_bunch()     { return medium_chunk_size() * MediumChunkMultiple; }
 694 
 695   size_t smallest_chunk_size()  { return specialized_chunk_size(); }
 696 
 697   size_t allocated_blocks_words() const { return _allocated_blocks_words; }
 698   size_t allocated_blocks_bytes() const { return _allocated_blocks_words * BytesPerWord; }
 699   size_t allocated_chunks_words() const { return _allocated_chunks_words; }
 700   size_t allocated_chunks_bytes() const { return _allocated_chunks_words * BytesPerWord; }
 701   size_t allocated_chunks_count() const { return _allocated_chunks_count; }
 702 
 703   bool is_humongous(size_t word_size) { return word_size > medium_chunk_size(); }
 704 
 705   static Mutex* expand_lock() { return _expand_lock; }
 706 
 707   // Increment the per Metaspace and global running sums for Metachunks
 708   // by the given size.  This is used when a Metachunk to added to
 709   // the in-use list.
 710   void inc_size_metrics(size_t words);
 711   // Increment the per Metaspace and global running sums Metablocks by the given
 712   // size.  This is used when a Metablock is allocated.
 713   void inc_used_metrics(size_t words);
 714   // Delete the portion of the running sums for this SpaceManager. That is,
 715   // the globals running sums for the Metachunks and Metablocks are
 716   // decremented for all the Metachunks in-use by this SpaceManager.
 717   void dec_total_from_size_metrics();
 718 
 719   // Set the sizes for the initial chunks.
 720   void get_initial_chunk_sizes(Metaspace::MetaspaceType type,
 721                                size_t* chunk_word_size,
 722                                size_t* class_chunk_word_size);
 723 
 724   size_t sum_capacity_in_chunks_in_use() const;
 725   size_t sum_used_in_chunks_in_use() const;
 726   size_t sum_free_in_chunks_in_use() const;
 727   size_t sum_waste_in_chunks_in_use() const;
 728   size_t sum_waste_in_chunks_in_use(ChunkIndex index ) const;
 729 
 730   size_t sum_count_in_chunks_in_use();
 731   size_t sum_count_in_chunks_in_use(ChunkIndex i);
 732 
 733   Metachunk* get_new_chunk(size_t word_size, size_t grow_chunks_by_words);
 734 
 735   // Block allocation and deallocation.
 736   // Allocates a block from the current chunk
 737   MetaWord* allocate(size_t word_size);
 738 
 739   // Helper for allocations
 740   MetaWord* allocate_work(size_t word_size);
 741 
 742   // Returns a block to the per manager freelist
 743   void deallocate(MetaWord* p, size_t word_size);
 744 
 745   // Based on the allocation size and a minimum chunk size,
 746   // returned chunk size (for expanding space for chunk allocation).
 747   size_t calc_chunk_size(size_t allocation_word_size);
 748 
 749   // Called when an allocation from the current chunk fails.
 750   // Gets a new chunk (may require getting a new virtual space),
 751   // and allocates from that chunk.
 752   MetaWord* grow_and_allocate(size_t word_size);
 753 
 754   // Notify memory usage to MemoryService.
 755   void track_metaspace_memory_usage();
 756 
 757   // debugging support.
 758 
 759   void dump(outputStream* const out) const;
 760   void print_on(outputStream* st) const;
 761   void locked_print_chunks_in_use_on(outputStream* st) const;
 762 
 763   void verify();
 764   void verify_chunk_size(Metachunk* chunk);
 765   NOT_PRODUCT(void mangle_freed_chunks();)
 766 #ifdef ASSERT
 767   void verify_allocated_blocks_words();
 768 #endif
 769 
 770   size_t get_raw_word_size(size_t word_size) {
 771     size_t byte_size = word_size * BytesPerWord;
 772 
 773     size_t raw_bytes_size = MAX2(byte_size, sizeof(Metablock));
 774     raw_bytes_size = align_size_up(raw_bytes_size, Metachunk::object_alignment());
 775 
 776     size_t raw_word_size = raw_bytes_size / BytesPerWord;
 777     assert(raw_word_size * BytesPerWord == raw_bytes_size, "Size problem");
 778 
 779     return raw_word_size;
 780   }
 781 };
 782 
 783 uint const SpaceManager::_small_chunk_limit = 4;
 784 
 785 const char* SpaceManager::_expand_lock_name =
 786   "SpaceManager chunk allocation lock";
 787 const int SpaceManager::_expand_lock_rank = Monitor::leaf - 1;
 788 Mutex* const SpaceManager::_expand_lock =
 789   new Mutex(SpaceManager::_expand_lock_rank,
 790             SpaceManager::_expand_lock_name,
 791             Mutex::_allow_vm_block_flag);
 792 
 793 void VirtualSpaceNode::inc_container_count() {
 794   assert_lock_strong(SpaceManager::expand_lock());
 795   _container_count++;
 796   assert(_container_count == container_count_slow(),
 797          err_msg("Inconsistency in container_count _container_count " SIZE_FORMAT
 798                  " container_count_slow() " SIZE_FORMAT,
 799                  _container_count, container_count_slow()));
 800 }
 801 
 802 void VirtualSpaceNode::dec_container_count() {
 803   assert_lock_strong(SpaceManager::expand_lock());
 804   _container_count--;
 805 }
 806 
 807 #ifdef ASSERT
 808 void VirtualSpaceNode::verify_container_count() {
 809   assert(_container_count == container_count_slow(),
 810     err_msg("Inconsistency in container_count _container_count " SIZE_FORMAT
 811             " container_count_slow() " SIZE_FORMAT, _container_count, container_count_slow()));
 812 }
 813 #endif
 814 
 815 // BlockFreelist methods
 816 
 817 BlockFreelist::BlockFreelist() : _dictionary(NULL) {}
 818 
 819 BlockFreelist::~BlockFreelist() {
 820   if (_dictionary != NULL) {
 821     if (Verbose && TraceMetadataChunkAllocation) {
 822       _dictionary->print_free_lists(gclog_or_tty);
 823     }
 824     delete _dictionary;
 825   }
 826 }
 827 
 828 void BlockFreelist::return_block(MetaWord* p, size_t word_size) {
 829   Metablock* free_chunk = ::new (p) Metablock(word_size);
 830   if (dictionary() == NULL) {
 831    _dictionary = new BlockTreeDictionary();
 832   }
 833   dictionary()->return_chunk(free_chunk);
 834 }
 835 
 836 MetaWord* BlockFreelist::get_block(size_t word_size) {
 837   if (dictionary() == NULL) {
 838     return NULL;
 839   }
 840 
 841   if (word_size < TreeChunk<Metablock, FreeList<Metablock> >::min_size()) {
 842     // Dark matter.  Too small for dictionary.
 843     return NULL;
 844   }
 845 
 846   Metablock* free_block =
 847     dictionary()->get_chunk(word_size, FreeBlockDictionary<Metablock>::atLeast);
 848   if (free_block == NULL) {
 849     return NULL;
 850   }
 851 
 852   const size_t block_size = free_block->size();
 853   if (block_size > WasteMultiplier * word_size) {
 854     return_block((MetaWord*)free_block, block_size);
 855     return NULL;
 856   }
 857 
 858   MetaWord* new_block = (MetaWord*)free_block;
 859   assert(block_size >= word_size, "Incorrect size of block from freelist");
 860   const size_t unused = block_size - word_size;
 861   if (unused >= TreeChunk<Metablock, FreeList<Metablock> >::min_size()) {
 862     return_block(new_block + word_size, unused);
 863   }
 864 
 865   return new_block;
 866 }
 867 
 868 void BlockFreelist::print_on(outputStream* st) const {
 869   if (dictionary() == NULL) {
 870     return;
 871   }
 872   dictionary()->print_free_lists(st);
 873 }
 874 
 875 // VirtualSpaceNode methods
 876 
 877 VirtualSpaceNode::~VirtualSpaceNode() {
 878   _rs.release();
 879 #ifdef ASSERT
 880   size_t word_size = sizeof(*this) / BytesPerWord;
 881   Copy::fill_to_words((HeapWord*) this, word_size, 0xf1f1f1f1);
 882 #endif
 883 }
 884 
 885 size_t VirtualSpaceNode::used_words_in_vs() const {
 886   return pointer_delta(top(), bottom(), sizeof(MetaWord));
 887 }
 888 
 889 // Space committed in the VirtualSpace
 890 size_t VirtualSpaceNode::capacity_words_in_vs() const {
 891   return pointer_delta(end(), bottom(), sizeof(MetaWord));
 892 }
 893 
 894 size_t VirtualSpaceNode::free_words_in_vs() const {
 895   return pointer_delta(end(), top(), sizeof(MetaWord));
 896 }
 897 
 898 // Allocates the chunk from the virtual space only.
 899 // This interface is also used internally for debugging.  Not all
 900 // chunks removed here are necessarily used for allocation.
 901 Metachunk* VirtualSpaceNode::take_from_committed(size_t chunk_word_size) {
 902   // Bottom of the new chunk
 903   MetaWord* chunk_limit = top();
 904   assert(chunk_limit != NULL, "Not safe to call this method");
 905 
 906   // The virtual spaces are always expanded by the
 907   // commit granularity to enforce the following condition.
 908   // Without this the is_available check will not work correctly.
 909   assert(_virtual_space.committed_size() == _virtual_space.actual_committed_size(),
 910       "The committed memory doesn't match the expanded memory.");
 911 
 912   if (!is_available(chunk_word_size)) {
 913     if (TraceMetadataChunkAllocation) {
 914       gclog_or_tty->print("VirtualSpaceNode::take_from_committed() not available %d words ", chunk_word_size);
 915       // Dump some information about the virtual space that is nearly full
 916       print_on(gclog_or_tty);
 917     }
 918     return NULL;
 919   }
 920 
 921   // Take the space  (bump top on the current virtual space).
 922   inc_top(chunk_word_size);
 923 
 924   // Initialize the chunk
 925   Metachunk* result = ::new (chunk_limit) Metachunk(chunk_word_size, this);
 926   return result;
 927 }
 928 
 929 
 930 // Expand the virtual space (commit more of the reserved space)
 931 bool VirtualSpaceNode::expand_by(size_t min_words, size_t preferred_words) {
 932   size_t min_bytes = min_words * BytesPerWord;
 933   size_t preferred_bytes = preferred_words * BytesPerWord;
 934 
 935   size_t uncommitted = virtual_space()->reserved_size() - virtual_space()->actual_committed_size();
 936 
 937   if (uncommitted < min_bytes) {
 938     return false;
 939   }
 940 
 941   size_t commit = MIN2(preferred_bytes, uncommitted);
 942   bool result = virtual_space()->expand_by(commit, false);
 943 
 944   assert(result, "Failed to commit memory");
 945 
 946   return result;
 947 }
 948 
 949 Metachunk* VirtualSpaceNode::get_chunk_vs(size_t chunk_word_size) {
 950   assert_lock_strong(SpaceManager::expand_lock());
 951   Metachunk* result = take_from_committed(chunk_word_size);
 952   if (result != NULL) {
 953     inc_container_count();
 954   }
 955   return result;
 956 }
 957 
 958 bool VirtualSpaceNode::initialize() {
 959 
 960   if (!_rs.is_reserved()) {
 961     return false;
 962   }
 963 
 964   // These are necessary restriction to make sure that the virtual space always
 965   // grows in steps of Metaspace::commit_alignment(). If both base and size are
 966   // aligned only the middle alignment of the VirtualSpace is used.
 967   assert_is_ptr_aligned(_rs.base(), Metaspace::commit_alignment());
 968   assert_is_size_aligned(_rs.size(), Metaspace::commit_alignment());
 969 
 970   // ReservedSpaces marked as special will have the entire memory
 971   // pre-committed. Setting a committed size will make sure that
 972   // committed_size and actual_committed_size agrees.
 973   size_t pre_committed_size = _rs.special() ? _rs.size() : 0;
 974 
 975   bool result = virtual_space()->initialize_with_granularity(_rs, pre_committed_size,
 976                                             Metaspace::commit_alignment());
 977   if (result) {
 978     assert(virtual_space()->committed_size() == virtual_space()->actual_committed_size(),
 979         "Checking that the pre-committed memory was registered by the VirtualSpace");
 980 
 981     set_top((MetaWord*)virtual_space()->low());
 982     set_reserved(MemRegion((HeapWord*)_rs.base(),
 983                  (HeapWord*)(_rs.base() + _rs.size())));
 984 
 985     assert(reserved()->start() == (HeapWord*) _rs.base(),
 986       err_msg("Reserved start was not set properly " PTR_FORMAT
 987         " != " PTR_FORMAT, reserved()->start(), _rs.base()));
 988     assert(reserved()->word_size() == _rs.size() / BytesPerWord,
 989       err_msg("Reserved size was not set properly " SIZE_FORMAT
 990         " != " SIZE_FORMAT, reserved()->word_size(),
 991         _rs.size() / BytesPerWord));
 992   }
 993 
 994   return result;
 995 }
 996 
 997 void VirtualSpaceNode::print_on(outputStream* st) const {
 998   size_t used = used_words_in_vs();
 999   size_t capacity = capacity_words_in_vs();
1000   VirtualSpace* vs = virtual_space();
1001   st->print_cr("   space @ " PTR_FORMAT " " SIZE_FORMAT "K, %3d%% used "
1002            "[" PTR_FORMAT ", " PTR_FORMAT ", "
1003            PTR_FORMAT ", " PTR_FORMAT ")",
1004            vs, capacity / K,
1005            capacity == 0 ? 0 : used * 100 / capacity,
1006            bottom(), top(), end(),
1007            vs->high_boundary());
1008 }
1009 
1010 #ifdef ASSERT
1011 void VirtualSpaceNode::mangle() {
1012   size_t word_size = capacity_words_in_vs();
1013   Copy::fill_to_words((HeapWord*) low(), word_size, 0xf1f1f1f1);
1014 }
1015 #endif // ASSERT
1016 
1017 // VirtualSpaceList methods
1018 // Space allocated from the VirtualSpace
1019 
1020 VirtualSpaceList::~VirtualSpaceList() {
1021   VirtualSpaceListIterator iter(virtual_space_list());
1022   while (iter.repeat()) {
1023     VirtualSpaceNode* vsl = iter.get_next();
1024     delete vsl;
1025   }
1026 }
1027 
1028 void VirtualSpaceList::inc_reserved_words(size_t v) {
1029   assert_lock_strong(SpaceManager::expand_lock());
1030   _reserved_words = _reserved_words + v;
1031 }
1032 void VirtualSpaceList::dec_reserved_words(size_t v) {
1033   assert_lock_strong(SpaceManager::expand_lock());
1034   _reserved_words = _reserved_words - v;
1035 }
1036 
1037 #define assert_committed_below_limit()                             \
1038   assert(MetaspaceAux::committed_bytes() <= MaxMetaspaceSize,      \
1039       err_msg("Too much committed memory. Committed: " SIZE_FORMAT \
1040               " limit (MaxMetaspaceSize): " SIZE_FORMAT,           \
1041           MetaspaceAux::committed_bytes(), MaxMetaspaceSize));
1042 
1043 void VirtualSpaceList::inc_committed_words(size_t v) {
1044   assert_lock_strong(SpaceManager::expand_lock());
1045   _committed_words = _committed_words + v;
1046 
1047   assert_committed_below_limit();
1048 }
1049 void VirtualSpaceList::dec_committed_words(size_t v) {
1050   assert_lock_strong(SpaceManager::expand_lock());
1051   _committed_words = _committed_words - v;
1052 
1053   assert_committed_below_limit();
1054 }
1055 
1056 void VirtualSpaceList::inc_virtual_space_count() {
1057   assert_lock_strong(SpaceManager::expand_lock());
1058   _virtual_space_count++;
1059 }
1060 void VirtualSpaceList::dec_virtual_space_count() {
1061   assert_lock_strong(SpaceManager::expand_lock());
1062   _virtual_space_count--;
1063 }
1064 
1065 void ChunkManager::remove_chunk(Metachunk* chunk) {
1066   size_t word_size = chunk->word_size();
1067   ChunkIndex index = list_index(word_size);
1068   if (index != HumongousIndex) {
1069     free_chunks(index)->remove_chunk(chunk);
1070   } else {
1071     humongous_dictionary()->remove_chunk(chunk);
1072   }
1073 
1074   // Chunk is being removed from the chunks free list.
1075   dec_free_chunks_total(chunk->word_size());
1076 }
1077 
1078 // Walk the list of VirtualSpaceNodes and delete
1079 // nodes with a 0 container_count.  Remove Metachunks in
1080 // the node from their respective freelists.
1081 void VirtualSpaceList::purge(ChunkManager* chunk_manager) {
1082   assert(SafepointSynchronize::is_at_safepoint(), "must be called at safepoint for contains to work");
1083   assert_lock_strong(SpaceManager::expand_lock());
1084   // Don't use a VirtualSpaceListIterator because this
1085   // list is being changed and a straightforward use of an iterator is not safe.
1086   VirtualSpaceNode* purged_vsl = NULL;
1087   VirtualSpaceNode* prev_vsl = virtual_space_list();
1088   VirtualSpaceNode* next_vsl = prev_vsl;
1089   while (next_vsl != NULL) {
1090     VirtualSpaceNode* vsl = next_vsl;
1091     next_vsl = vsl->next();
1092     // Don't free the current virtual space since it will likely
1093     // be needed soon.
1094     if (vsl->container_count() == 0 && vsl != current_virtual_space()) {
1095       // Unlink it from the list
1096       if (prev_vsl == vsl) {
1097         // This is the case of the current node being the first node.
1098         assert(vsl == virtual_space_list(), "Expected to be the first node");
1099         set_virtual_space_list(vsl->next());
1100       } else {
1101         prev_vsl->set_next(vsl->next());
1102       }
1103 
1104       vsl->purge(chunk_manager);
1105       dec_reserved_words(vsl->reserved_words());
1106       dec_committed_words(vsl->committed_words());
1107       dec_virtual_space_count();
1108       purged_vsl = vsl;
1109       delete vsl;
1110     } else {
1111       prev_vsl = vsl;
1112     }
1113   }
1114 #ifdef ASSERT
1115   if (purged_vsl != NULL) {
1116     // List should be stable enough to use an iterator here.
1117     VirtualSpaceListIterator iter(virtual_space_list());
1118     while (iter.repeat()) {
1119       VirtualSpaceNode* vsl = iter.get_next();
1120       assert(vsl != purged_vsl, "Purge of vsl failed");
1121     }
1122   }
1123 #endif
1124 }
1125 
1126 
1127 // This function looks at the mmap regions in the metaspace without locking.
1128 // The chunks are added with store ordering and not deleted except for at
1129 // unloading time during a safepoint.
1130 bool VirtualSpaceList::contains(const void* ptr) {
1131   // List should be stable enough to use an iterator here because removing virtual
1132   // space nodes is only allowed at a safepoint.
1133   VirtualSpaceListIterator iter(virtual_space_list());
1134   while (iter.repeat()) {
1135     VirtualSpaceNode* vsn = iter.get_next();
1136     if (vsn->contains(ptr)) {
1137       return true;
1138     }
1139   }
1140   return false;
1141 }
1142 
1143 void VirtualSpaceList::retire_current_virtual_space() {
1144   assert_lock_strong(SpaceManager::expand_lock());
1145 
1146   VirtualSpaceNode* vsn = current_virtual_space();
1147 
1148   ChunkManager* cm = is_class() ? Metaspace::chunk_manager_class() :
1149                                   Metaspace::chunk_manager_metadata();
1150 
1151   vsn->retire(cm);
1152 }
1153 
1154 void VirtualSpaceNode::retire(ChunkManager* chunk_manager) {
1155   for (int i = (int)MediumIndex; i >= (int)ZeroIndex; --i) {
1156     ChunkIndex index = (ChunkIndex)i;
1157     size_t chunk_size = chunk_manager->free_chunks(index)->size();
1158 
1159     while (free_words_in_vs() >= chunk_size) {
1160       DEBUG_ONLY(verify_container_count();)
1161       Metachunk* chunk = get_chunk_vs(chunk_size);
1162       assert(chunk != NULL, "allocation should have been successful");
1163 
1164       chunk_manager->return_chunks(index, chunk);
1165       chunk_manager->inc_free_chunks_total(chunk_size);
1166       DEBUG_ONLY(verify_container_count();)
1167     }
1168   }
1169   assert(free_words_in_vs() == 0, "should be empty now");
1170 }
1171 
1172 VirtualSpaceList::VirtualSpaceList(size_t word_size) :
1173                                    _is_class(false),
1174                                    _virtual_space_list(NULL),
1175                                    _current_virtual_space(NULL),
1176                                    _reserved_words(0),
1177                                    _committed_words(0),
1178                                    _virtual_space_count(0) {
1179   MutexLockerEx cl(SpaceManager::expand_lock(),
1180                    Mutex::_no_safepoint_check_flag);
1181   create_new_virtual_space(word_size);
1182 }
1183 
1184 VirtualSpaceList::VirtualSpaceList(ReservedSpace rs) :
1185                                    _is_class(true),
1186                                    _virtual_space_list(NULL),
1187                                    _current_virtual_space(NULL),
1188                                    _reserved_words(0),
1189                                    _committed_words(0),
1190                                    _virtual_space_count(0) {
1191   MutexLockerEx cl(SpaceManager::expand_lock(),
1192                    Mutex::_no_safepoint_check_flag);
1193   VirtualSpaceNode* class_entry = new VirtualSpaceNode(rs);
1194   bool succeeded = class_entry->initialize();
1195   if (succeeded) {
1196     link_vs(class_entry);
1197   }
1198 }
1199 
1200 size_t VirtualSpaceList::free_bytes() {
1201   return virtual_space_list()->free_words_in_vs() * BytesPerWord;
1202 }
1203 
1204 // Allocate another meta virtual space and add it to the list.
1205 bool VirtualSpaceList::create_new_virtual_space(size_t vs_word_size) {
1206   assert_lock_strong(SpaceManager::expand_lock());
1207 
1208   if (is_class()) {
1209     assert(false, "We currently don't support more than one VirtualSpace for"
1210                   " the compressed class space. The initialization of the"
1211                   " CCS uses another code path and should not hit this path.");
1212     return false;
1213   }
1214 
1215   if (vs_word_size == 0) {
1216     assert(false, "vs_word_size should always be at least _reserve_alignment large.");
1217     return false;
1218   }
1219 
1220   // Reserve the space
1221   size_t vs_byte_size = vs_word_size * BytesPerWord;
1222   assert_is_size_aligned(vs_byte_size, Metaspace::reserve_alignment());
1223 
1224   // Allocate the meta virtual space and initialize it.
1225   VirtualSpaceNode* new_entry = new VirtualSpaceNode(vs_byte_size);
1226   if (!new_entry->initialize()) {
1227     delete new_entry;
1228     return false;
1229   } else {
1230     assert(new_entry->reserved_words() == vs_word_size,
1231         "Reserved memory size differs from requested memory size");
1232     // ensure lock-free iteration sees fully initialized node
1233     OrderAccess::storestore();
1234     link_vs(new_entry);
1235     return true;
1236   }
1237 }
1238 
1239 void VirtualSpaceList::link_vs(VirtualSpaceNode* new_entry) {
1240   if (virtual_space_list() == NULL) {
1241       set_virtual_space_list(new_entry);
1242   } else {
1243     current_virtual_space()->set_next(new_entry);
1244   }
1245   set_current_virtual_space(new_entry);
1246   inc_reserved_words(new_entry->reserved_words());
1247   inc_committed_words(new_entry->committed_words());
1248   inc_virtual_space_count();
1249 #ifdef ASSERT
1250   new_entry->mangle();
1251 #endif
1252   if (TraceMetavirtualspaceAllocation && Verbose) {
1253     VirtualSpaceNode* vsl = current_virtual_space();
1254     vsl->print_on(gclog_or_tty);
1255   }
1256 }
1257 
1258 bool VirtualSpaceList::expand_node_by(VirtualSpaceNode* node,
1259                                       size_t min_words,
1260                                       size_t preferred_words) {
1261   size_t before = node->committed_words();
1262 
1263   bool result = node->expand_by(min_words, preferred_words);
1264 
1265   size_t after = node->committed_words();
1266 
1267   // after and before can be the same if the memory was pre-committed.
1268   assert(after >= before, "Inconsistency");
1269   inc_committed_words(after - before);
1270 
1271   return result;
1272 }
1273 
1274 bool VirtualSpaceList::expand_by(size_t min_words, size_t preferred_words) {
1275   assert_is_size_aligned(min_words,       Metaspace::commit_alignment_words());
1276   assert_is_size_aligned(preferred_words, Metaspace::commit_alignment_words());
1277   assert(min_words <= preferred_words, "Invalid arguments");
1278 
1279   if (!MetaspaceGC::can_expand(min_words, this->is_class())) {
1280     return  false;
1281   }
1282 
1283   size_t allowed_expansion_words = MetaspaceGC::allowed_expansion();
1284   if (allowed_expansion_words < min_words) {
1285     return false;
1286   }
1287 
1288   size_t max_expansion_words = MIN2(preferred_words, allowed_expansion_words);
1289 
1290   // Commit more memory from the the current virtual space.
1291   bool vs_expanded = expand_node_by(current_virtual_space(),
1292                                     min_words,
1293                                     max_expansion_words);
1294   if (vs_expanded) {
1295     return true;
1296   }
1297   retire_current_virtual_space();
1298 
1299   // Get another virtual space.
1300   size_t grow_vs_words = MAX2((size_t)VirtualSpaceSize, preferred_words);
1301   grow_vs_words = align_size_up(grow_vs_words, Metaspace::reserve_alignment_words());
1302 
1303   if (create_new_virtual_space(grow_vs_words)) {
1304     if (current_virtual_space()->is_pre_committed()) {
1305       // The memory was pre-committed, so we are done here.
1306       assert(min_words <= current_virtual_space()->committed_words(),
1307           "The new VirtualSpace was pre-committed, so it"
1308           "should be large enough to fit the alloc request.");
1309       return true;
1310     }
1311 
1312     return expand_node_by(current_virtual_space(),
1313                           min_words,
1314                           max_expansion_words);
1315   }
1316 
1317   return false;
1318 }
1319 
1320 Metachunk* VirtualSpaceList::get_new_chunk(size_t word_size,
1321                                            size_t grow_chunks_by_words,
1322                                            size_t medium_chunk_bunch) {
1323 
1324   // Allocate a chunk out of the current virtual space.
1325   Metachunk* next = current_virtual_space()->get_chunk_vs(grow_chunks_by_words);
1326 
1327   if (next != NULL) {
1328     return next;
1329   }
1330 
1331   // The expand amount is currently only determined by the requested sizes
1332   // and not how much committed memory is left in the current virtual space.
1333 
1334   size_t min_word_size       = align_size_up(grow_chunks_by_words, Metaspace::commit_alignment_words());
1335   size_t preferred_word_size = align_size_up(medium_chunk_bunch,   Metaspace::commit_alignment_words());
1336   if (min_word_size >= preferred_word_size) {
1337     // Can happen when humongous chunks are allocated.
1338     preferred_word_size = min_word_size;
1339   }
1340 
1341   bool expanded = expand_by(min_word_size, preferred_word_size);
1342   if (expanded) {
1343     next = current_virtual_space()->get_chunk_vs(grow_chunks_by_words);
1344     assert(next != NULL, "The allocation was expected to succeed after the expansion");
1345   }
1346 
1347    return next;
1348 }
1349 
1350 void VirtualSpaceList::print_on(outputStream* st) const {
1351   if (TraceMetadataChunkAllocation && Verbose) {
1352     VirtualSpaceListIterator iter(virtual_space_list());
1353     while (iter.repeat()) {
1354       VirtualSpaceNode* node = iter.get_next();
1355       node->print_on(st);
1356     }
1357   }
1358 }
1359 
1360 // MetaspaceGC methods
1361 
1362 // VM_CollectForMetadataAllocation is the vm operation used to GC.
1363 // Within the VM operation after the GC the attempt to allocate the metadata
1364 // should succeed.  If the GC did not free enough space for the metaspace
1365 // allocation, the HWM is increased so that another virtualspace will be
1366 // allocated for the metadata.  With perm gen the increase in the perm
1367 // gen had bounds, MinMetaspaceExpansion and MaxMetaspaceExpansion.  The
1368 // metaspace policy uses those as the small and large steps for the HWM.
1369 //
1370 // After the GC the compute_new_size() for MetaspaceGC is called to
1371 // resize the capacity of the metaspaces.  The current implementation
1372 // is based on the flags MinMetaspaceFreeRatio and MaxMetaspaceFreeRatio used
1373 // to resize the Java heap by some GC's.  New flags can be implemented
1374 // if really needed.  MinMetaspaceFreeRatio is used to calculate how much
1375 // free space is desirable in the metaspace capacity to decide how much
1376 // to increase the HWM.  MaxMetaspaceFreeRatio is used to decide how much
1377 // free space is desirable in the metaspace capacity before decreasing
1378 // the HWM.
1379 
1380 // Calculate the amount to increase the high water mark (HWM).
1381 // Increase by a minimum amount (MinMetaspaceExpansion) so that
1382 // another expansion is not requested too soon.  If that is not
1383 // enough to satisfy the allocation, increase by MaxMetaspaceExpansion.
1384 // If that is still not enough, expand by the size of the allocation
1385 // plus some.
1386 size_t MetaspaceGC::delta_capacity_until_GC(size_t bytes) {
1387   size_t min_delta = MinMetaspaceExpansion;
1388   size_t max_delta = MaxMetaspaceExpansion;
1389   size_t delta = align_size_up(bytes, Metaspace::commit_alignment());
1390 
1391   if (delta <= min_delta) {
1392     delta = min_delta;
1393   } else if (delta <= max_delta) {
1394     // Don't want to hit the high water mark on the next
1395     // allocation so make the delta greater than just enough
1396     // for this allocation.
1397     delta = max_delta;
1398   } else {
1399     // This allocation is large but the next ones are probably not
1400     // so increase by the minimum.
1401     delta = delta + min_delta;
1402   }
1403 
1404   assert_is_size_aligned(delta, Metaspace::commit_alignment());
1405 
1406   return delta;
1407 }
1408 
1409 size_t MetaspaceGC::capacity_until_GC() {
1410   size_t value = (size_t)OrderAccess::load_ptr_acquire(&_capacity_until_GC);
1411   assert(value >= MetaspaceSize, "Not initialied properly?");
1412   return value;
1413 }
1414 
1415 size_t MetaspaceGC::inc_capacity_until_GC(size_t v) {
1416   assert_is_size_aligned(v, Metaspace::commit_alignment());
1417 
1418   size_t capacity_until_GC = (size_t) _capacity_until_GC;
1419   size_t new_value = capacity_until_GC + v;
1420 
1421   if (new_value < capacity_until_GC) {
1422     // the addition wrapped around, set new_value to max value
1423     new_value = max_uintx;
1424   }
1425 
1426   bool cas_succeeded = false;
1427   size_t num_retries = 5; // just to limit the number of CAS attempts
1428   for (size_t i = 0; i < num_retries; i++) {
1429     intptr_t cmp = (intptr_t) capacity_until_GC;
1430     intptr_t swap = (intptr_t) new_value;
1431     intptr_t prev = Atomic::cmpxchg_ptr(swap, &_capacity_until_GC, cmp);
1432 
1433     if (prev == cmp) {
1434       cas_succeeded = true;
1435       break;
1436     }
1437   }
1438 
1439   // if the CAS did not succeed, just return the current value
1440   return cas_succeeded ? new_value : _capacity_until_GC;
1441 }
1442 
1443 size_t MetaspaceGC::dec_capacity_until_GC(size_t v) {
1444   assert_is_size_aligned(v, Metaspace::commit_alignment());
1445 
1446   return (size_t)Atomic::add_ptr(-(intptr_t)v, &_capacity_until_GC);
1447 }
1448 
1449 void MetaspaceGC::initialize() {
1450   // Set the high-water mark to MaxMetapaceSize during VM initializaton since
1451   // we can't do a GC during initialization.
1452   _capacity_until_GC = MaxMetaspaceSize;
1453 }
1454 
1455 void MetaspaceGC::post_initialize() {
1456   // Reset the high-water mark once the VM initialization is done.
1457   _capacity_until_GC = MAX2(MetaspaceAux::committed_bytes(), MetaspaceSize);
1458 }
1459 
1460 bool MetaspaceGC::can_expand(size_t word_size, bool is_class) {
1461   // Check if the compressed class space is full.
1462   if (is_class && Metaspace::using_class_space()) {
1463     size_t class_committed = MetaspaceAux::committed_bytes(Metaspace::ClassType);
1464     if (class_committed + word_size * BytesPerWord > CompressedClassSpaceSize) {
1465       return false;
1466     }
1467   }
1468 
1469   // Check if the user has imposed a limit on the metaspace memory.
1470   size_t committed_bytes = MetaspaceAux::committed_bytes();
1471   if (committed_bytes + word_size * BytesPerWord > MaxMetaspaceSize) {
1472     return false;
1473   }
1474 
1475   return true;
1476 }
1477 
1478 size_t MetaspaceGC::allowed_expansion() {
1479   size_t committed_bytes = MetaspaceAux::committed_bytes();
1480   size_t capacity_until_gc = capacity_until_GC();
1481 
1482   assert(capacity_until_gc >= committed_bytes,
1483         err_msg("capacity_until_gc: " SIZE_FORMAT " < committed_bytes: " SIZE_FORMAT,
1484                 capacity_until_gc, committed_bytes));
1485 
1486   size_t left_until_max  = MaxMetaspaceSize - committed_bytes;
1487   size_t left_until_GC = capacity_until_gc - committed_bytes;
1488   size_t left_to_commit = MIN2(left_until_GC, left_until_max);
1489 
1490   return left_to_commit / BytesPerWord;
1491 }
1492 
1493 void MetaspaceGC::compute_new_size() {
1494   assert(_shrink_factor <= 100, "invalid shrink factor");
1495   uint current_shrink_factor = _shrink_factor;
1496   _shrink_factor = 0;
1497 
1498   // Using committed_bytes() for used_after_gc is an overestimation, since the
1499   // chunk free lists are included in committed_bytes() and the memory in an
1500   // un-fragmented chunk free list is available for future allocations.
1501   // However, if the chunk free lists becomes fragmented, then the memory may
1502   // not be available for future allocations and the memory is therefore "in use".
1503   // Including the chunk free lists in the definition of "in use" is therefore
1504   // necessary. Not including the chunk free lists can cause capacity_until_GC to
1505   // shrink below committed_bytes() and this has caused serious bugs in the past.
1506   const size_t used_after_gc = MetaspaceAux::committed_bytes();
1507   const size_t capacity_until_GC = MetaspaceGC::capacity_until_GC();
1508 
1509   const double minimum_free_percentage = MinMetaspaceFreeRatio / 100.0;
1510   const double maximum_used_percentage = 1.0 - minimum_free_percentage;
1511 
1512   const double min_tmp = used_after_gc / maximum_used_percentage;
1513   size_t minimum_desired_capacity =
1514     (size_t)MIN2(min_tmp, double(max_uintx));
1515   // Don't shrink less than the initial generation size
1516   minimum_desired_capacity = MAX2(minimum_desired_capacity,
1517                                   MetaspaceSize);
1518 
1519   if (PrintGCDetails && Verbose) {
1520     gclog_or_tty->print_cr("\nMetaspaceGC::compute_new_size: ");
1521     gclog_or_tty->print_cr("  "
1522                   "  minimum_free_percentage: %6.2f"
1523                   "  maximum_used_percentage: %6.2f",
1524                   minimum_free_percentage,
1525                   maximum_used_percentage);
1526     gclog_or_tty->print_cr("  "
1527                   "   used_after_gc       : %6.1fKB",
1528                   used_after_gc / (double) K);
1529   }
1530 
1531 
1532   size_t shrink_bytes = 0;
1533   if (capacity_until_GC < minimum_desired_capacity) {
1534     // If we have less capacity below the metaspace HWM, then
1535     // increment the HWM.
1536     size_t expand_bytes = minimum_desired_capacity - capacity_until_GC;
1537     expand_bytes = align_size_up(expand_bytes, Metaspace::commit_alignment());
1538     // Don't expand unless it's significant
1539     if (expand_bytes >= MinMetaspaceExpansion) {
1540       size_t new_capacity_until_GC = MetaspaceGC::inc_capacity_until_GC(expand_bytes);
1541       Metaspace::tracer()->report_gc_threshold(capacity_until_GC,
1542                                                new_capacity_until_GC,
1543                                                MetaspaceGCThresholdUpdater::ComputeNewSize);
1544       if (PrintGCDetails && Verbose) {
1545         gclog_or_tty->print_cr("    expanding:"
1546                       "  minimum_desired_capacity: %6.1fKB"
1547                       "  expand_bytes: %6.1fKB"
1548                       "  MinMetaspaceExpansion: %6.1fKB"
1549                       "  new metaspace HWM:  %6.1fKB",
1550                       minimum_desired_capacity / (double) K,
1551                       expand_bytes / (double) K,
1552                       MinMetaspaceExpansion / (double) K,
1553                       new_capacity_until_GC / (double) K);
1554       }
1555     }
1556     return;
1557   }
1558 
1559   // No expansion, now see if we want to shrink
1560   // We would never want to shrink more than this
1561   assert(capacity_until_GC >= minimum_desired_capacity,
1562          err_msg(SIZE_FORMAT " >= " SIZE_FORMAT,
1563                  capacity_until_GC, minimum_desired_capacity));
1564   size_t max_shrink_bytes = capacity_until_GC - minimum_desired_capacity;
1565 
1566   // Should shrinking be considered?
1567   if (MaxMetaspaceFreeRatio < 100) {
1568     const double maximum_free_percentage = MaxMetaspaceFreeRatio / 100.0;
1569     const double minimum_used_percentage = 1.0 - maximum_free_percentage;
1570     const double max_tmp = used_after_gc / minimum_used_percentage;
1571     size_t maximum_desired_capacity = (size_t)MIN2(max_tmp, double(max_uintx));
1572     maximum_desired_capacity = MAX2(maximum_desired_capacity,
1573                                     MetaspaceSize);
1574     if (PrintGCDetails && Verbose) {
1575       gclog_or_tty->print_cr("  "
1576                              "  maximum_free_percentage: %6.2f"
1577                              "  minimum_used_percentage: %6.2f",
1578                              maximum_free_percentage,
1579                              minimum_used_percentage);
1580       gclog_or_tty->print_cr("  "
1581                              "  minimum_desired_capacity: %6.1fKB"
1582                              "  maximum_desired_capacity: %6.1fKB",
1583                              minimum_desired_capacity / (double) K,
1584                              maximum_desired_capacity / (double) K);
1585     }
1586 
1587     assert(minimum_desired_capacity <= maximum_desired_capacity,
1588            "sanity check");
1589 
1590     if (capacity_until_GC > maximum_desired_capacity) {
1591       // Capacity too large, compute shrinking size
1592       shrink_bytes = capacity_until_GC - maximum_desired_capacity;
1593       // We don't want shrink all the way back to initSize if people call
1594       // System.gc(), because some programs do that between "phases" and then
1595       // we'd just have to grow the heap up again for the next phase.  So we
1596       // damp the shrinking: 0% on the first call, 10% on the second call, 40%
1597       // on the third call, and 100% by the fourth call.  But if we recompute
1598       // size without shrinking, it goes back to 0%.
1599       shrink_bytes = shrink_bytes / 100 * current_shrink_factor;
1600 
1601       shrink_bytes = align_size_down(shrink_bytes, Metaspace::commit_alignment());
1602 
1603       assert(shrink_bytes <= max_shrink_bytes,
1604         err_msg("invalid shrink size " SIZE_FORMAT " not <= " SIZE_FORMAT,
1605           shrink_bytes, max_shrink_bytes));
1606       if (current_shrink_factor == 0) {
1607         _shrink_factor = 10;
1608       } else {
1609         _shrink_factor = MIN2(current_shrink_factor * 4, (uint) 100);
1610       }
1611       if (PrintGCDetails && Verbose) {
1612         gclog_or_tty->print_cr("  "
1613                       "  shrinking:"
1614                       "  initSize: %.1fK"
1615                       "  maximum_desired_capacity: %.1fK",
1616                       MetaspaceSize / (double) K,
1617                       maximum_desired_capacity / (double) K);
1618         gclog_or_tty->print_cr("  "
1619                       "  shrink_bytes: %.1fK"
1620                       "  current_shrink_factor: %d"
1621                       "  new shrink factor: %d"
1622                       "  MinMetaspaceExpansion: %.1fK",
1623                       shrink_bytes / (double) K,
1624                       current_shrink_factor,
1625                       _shrink_factor,
1626                       MinMetaspaceExpansion / (double) K);
1627       }
1628     }
1629   }
1630 
1631   // Don't shrink unless it's significant
1632   if (shrink_bytes >= MinMetaspaceExpansion &&
1633       ((capacity_until_GC - shrink_bytes) >= MetaspaceSize)) {
1634     size_t new_capacity_until_GC = MetaspaceGC::dec_capacity_until_GC(shrink_bytes);
1635     Metaspace::tracer()->report_gc_threshold(capacity_until_GC,
1636                                              new_capacity_until_GC,
1637                                              MetaspaceGCThresholdUpdater::ComputeNewSize);
1638   }
1639 }
1640 
1641 // Metadebug methods
1642 
1643 void Metadebug::init_allocation_fail_alot_count() {
1644   if (MetadataAllocationFailALot) {
1645     _allocation_fail_alot_count =
1646       1+(long)((double)MetadataAllocationFailALotInterval*os::random()/(max_jint+1.0));
1647   }
1648 }
1649 
1650 #ifdef ASSERT
1651 bool Metadebug::test_metadata_failure() {
1652   if (MetadataAllocationFailALot &&
1653       Threads::is_vm_complete()) {
1654     if (_allocation_fail_alot_count > 0) {
1655       _allocation_fail_alot_count--;
1656     } else {
1657       if (TraceMetadataChunkAllocation && Verbose) {
1658         gclog_or_tty->print_cr("Metadata allocation failing for "
1659                                "MetadataAllocationFailALot");
1660       }
1661       init_allocation_fail_alot_count();
1662       return true;
1663     }
1664   }
1665   return false;
1666 }
1667 #endif
1668 
1669 // ChunkManager methods
1670 
1671 size_t ChunkManager::free_chunks_total_words() {
1672   return _free_chunks_total;
1673 }
1674 
1675 size_t ChunkManager::free_chunks_total_bytes() {
1676   return free_chunks_total_words() * BytesPerWord;
1677 }
1678 
1679 size_t ChunkManager::free_chunks_count() {
1680 #ifdef ASSERT
1681   if (!UseConcMarkSweepGC && !SpaceManager::expand_lock()->is_locked()) {
1682     MutexLockerEx cl(SpaceManager::expand_lock(),
1683                      Mutex::_no_safepoint_check_flag);
1684     // This lock is only needed in debug because the verification
1685     // of the _free_chunks_totals walks the list of free chunks
1686     slow_locked_verify_free_chunks_count();
1687   }
1688 #endif
1689   return _free_chunks_count;
1690 }
1691 
1692 void ChunkManager::locked_verify_free_chunks_total() {
1693   assert_lock_strong(SpaceManager::expand_lock());
1694   assert(sum_free_chunks() == _free_chunks_total,
1695     err_msg("_free_chunks_total " SIZE_FORMAT " is not the"
1696            " same as sum " SIZE_FORMAT, _free_chunks_total,
1697            sum_free_chunks()));
1698 }
1699 
1700 void ChunkManager::verify_free_chunks_total() {
1701   MutexLockerEx cl(SpaceManager::expand_lock(),
1702                      Mutex::_no_safepoint_check_flag);
1703   locked_verify_free_chunks_total();
1704 }
1705 
1706 void ChunkManager::locked_verify_free_chunks_count() {
1707   assert_lock_strong(SpaceManager::expand_lock());
1708   assert(sum_free_chunks_count() == _free_chunks_count,
1709     err_msg("_free_chunks_count " SIZE_FORMAT " is not the"
1710            " same as sum " SIZE_FORMAT, _free_chunks_count,
1711            sum_free_chunks_count()));
1712 }
1713 
1714 void ChunkManager::verify_free_chunks_count() {
1715 #ifdef ASSERT
1716   MutexLockerEx cl(SpaceManager::expand_lock(),
1717                      Mutex::_no_safepoint_check_flag);
1718   locked_verify_free_chunks_count();
1719 #endif
1720 }
1721 
1722 void ChunkManager::verify() {
1723   MutexLockerEx cl(SpaceManager::expand_lock(),
1724                      Mutex::_no_safepoint_check_flag);
1725   locked_verify();
1726 }
1727 
1728 void ChunkManager::locked_verify() {
1729   locked_verify_free_chunks_count();
1730   locked_verify_free_chunks_total();
1731 }
1732 
1733 void ChunkManager::locked_print_free_chunks(outputStream* st) {
1734   assert_lock_strong(SpaceManager::expand_lock());
1735   st->print_cr("Free chunk total " SIZE_FORMAT "  count " SIZE_FORMAT,
1736                 _free_chunks_total, _free_chunks_count);
1737 }
1738 
1739 void ChunkManager::locked_print_sum_free_chunks(outputStream* st) {
1740   assert_lock_strong(SpaceManager::expand_lock());
1741   st->print_cr("Sum free chunk total " SIZE_FORMAT "  count " SIZE_FORMAT,
1742                 sum_free_chunks(), sum_free_chunks_count());
1743 }
1744 ChunkList* ChunkManager::free_chunks(ChunkIndex index) {
1745   return &_free_chunks[index];
1746 }
1747 
1748 // These methods that sum the free chunk lists are used in printing
1749 // methods that are used in product builds.
1750 size_t ChunkManager::sum_free_chunks() {
1751   assert_lock_strong(SpaceManager::expand_lock());
1752   size_t result = 0;
1753   for (ChunkIndex i = ZeroIndex; i < NumberOfFreeLists; i = next_chunk_index(i)) {
1754     ChunkList* list = free_chunks(i);
1755 
1756     if (list == NULL) {
1757       continue;
1758     }
1759 
1760     result = result + list->count() * list->size();
1761   }
1762   result = result + humongous_dictionary()->total_size();
1763   return result;
1764 }
1765 
1766 size_t ChunkManager::sum_free_chunks_count() {
1767   assert_lock_strong(SpaceManager::expand_lock());
1768   size_t count = 0;
1769   for (ChunkIndex i = ZeroIndex; i < NumberOfFreeLists; i = next_chunk_index(i)) {
1770     ChunkList* list = free_chunks(i);
1771     if (list == NULL) {
1772       continue;
1773     }
1774     count = count + list->count();
1775   }
1776   count = count + humongous_dictionary()->total_free_blocks();
1777   return count;
1778 }
1779 
1780 ChunkList* ChunkManager::find_free_chunks_list(size_t word_size) {
1781   ChunkIndex index = list_index(word_size);
1782   assert(index < HumongousIndex, "No humongous list");
1783   return free_chunks(index);
1784 }
1785 
1786 Metachunk* ChunkManager::free_chunks_get(size_t word_size) {
1787   assert_lock_strong(SpaceManager::expand_lock());
1788 
1789   slow_locked_verify();
1790 
1791   Metachunk* chunk = NULL;
1792   if (list_index(word_size) != HumongousIndex) {
1793     ChunkList* free_list = find_free_chunks_list(word_size);
1794     assert(free_list != NULL, "Sanity check");
1795 
1796     chunk = free_list->head();
1797 
1798     if (chunk == NULL) {
1799       return NULL;
1800     }
1801 
1802     // Remove the chunk as the head of the list.
1803     free_list->remove_chunk(chunk);
1804 
1805     if (TraceMetadataChunkAllocation && Verbose) {
1806       gclog_or_tty->print_cr("ChunkManager::free_chunks_get: free_list "
1807                              PTR_FORMAT " head " PTR_FORMAT " size " SIZE_FORMAT,
1808                              free_list, chunk, chunk->word_size());
1809     }
1810   } else {
1811     chunk = humongous_dictionary()->get_chunk(
1812       word_size,
1813       FreeBlockDictionary<Metachunk>::atLeast);
1814 
1815     if (chunk == NULL) {
1816       return NULL;
1817     }
1818 
1819     if (TraceMetadataHumongousAllocation) {
1820       size_t waste = chunk->word_size() - word_size;
1821       gclog_or_tty->print_cr("Free list allocate humongous chunk size "
1822                              SIZE_FORMAT " for requested size " SIZE_FORMAT
1823                              " waste " SIZE_FORMAT,
1824                              chunk->word_size(), word_size, waste);
1825     }
1826   }
1827 
1828   // Chunk is being removed from the chunks free list.
1829   dec_free_chunks_total(chunk->word_size());
1830 
1831   // Remove it from the links to this freelist
1832   chunk->set_next(NULL);
1833   chunk->set_prev(NULL);
1834 #ifdef ASSERT
1835   // Chunk is no longer on any freelist. Setting to false make container_count_slow()
1836   // work.
1837   chunk->set_is_tagged_free(false);
1838 #endif
1839   chunk->container()->inc_container_count();
1840 
1841   slow_locked_verify();
1842   return chunk;
1843 }
1844 
1845 Metachunk* ChunkManager::chunk_freelist_allocate(size_t word_size) {
1846   assert_lock_strong(SpaceManager::expand_lock());
1847   slow_locked_verify();
1848 
1849   // Take from the beginning of the list
1850   Metachunk* chunk = free_chunks_get(word_size);
1851   if (chunk == NULL) {
1852     return NULL;
1853   }
1854 
1855   assert((word_size <= chunk->word_size()) ||
1856          list_index(chunk->word_size() == HumongousIndex),
1857          "Non-humongous variable sized chunk");
1858   if (TraceMetadataChunkAllocation) {
1859     size_t list_count;
1860     if (list_index(word_size) < HumongousIndex) {
1861       ChunkList* list = find_free_chunks_list(word_size);
1862       list_count = list->count();
1863     } else {
1864       list_count = humongous_dictionary()->total_count();
1865     }
1866     gclog_or_tty->print("ChunkManager::chunk_freelist_allocate: " PTR_FORMAT " chunk "
1867                         PTR_FORMAT "  size " SIZE_FORMAT " count " SIZE_FORMAT " ",
1868                         this, chunk, chunk->word_size(), list_count);
1869     locked_print_free_chunks(gclog_or_tty);
1870   }
1871 
1872   return chunk;
1873 }
1874 
1875 void ChunkManager::print_on(outputStream* out) const {
1876   if (PrintFLSStatistics != 0) {
1877     const_cast<ChunkManager *>(this)->humongous_dictionary()->report_statistics();
1878   }
1879 }
1880 
1881 // SpaceManager methods
1882 
1883 void SpaceManager::get_initial_chunk_sizes(Metaspace::MetaspaceType type,
1884                                            size_t* chunk_word_size,
1885                                            size_t* class_chunk_word_size) {
1886   switch (type) {
1887   case Metaspace::BootMetaspaceType:
1888     *chunk_word_size = Metaspace::first_chunk_word_size();
1889     *class_chunk_word_size = Metaspace::first_class_chunk_word_size();
1890     break;
1891   case Metaspace::ROMetaspaceType:
1892     *chunk_word_size = SharedReadOnlySize / wordSize;
1893     *class_chunk_word_size = ClassSpecializedChunk;
1894     break;
1895   case Metaspace::ReadWriteMetaspaceType:
1896     *chunk_word_size = SharedReadWriteSize / wordSize;
1897     *class_chunk_word_size = ClassSpecializedChunk;
1898     break;
1899   case Metaspace::AnonymousMetaspaceType:
1900   case Metaspace::ReflectionMetaspaceType:
1901     *chunk_word_size = SpecializedChunk;
1902     *class_chunk_word_size = ClassSpecializedChunk;
1903     break;
1904   default:
1905     *chunk_word_size = SmallChunk;
1906     *class_chunk_word_size = ClassSmallChunk;
1907     break;
1908   }
1909   assert(*chunk_word_size != 0 && *class_chunk_word_size != 0,
1910     err_msg("Initial chunks sizes bad: data  " SIZE_FORMAT
1911             " class " SIZE_FORMAT,
1912             *chunk_word_size, *class_chunk_word_size));
1913 }
1914 
1915 size_t SpaceManager::sum_free_in_chunks_in_use() const {
1916   MutexLockerEx cl(lock(), Mutex::_no_safepoint_check_flag);
1917   size_t free = 0;
1918   for (ChunkIndex i = ZeroIndex; i < NumberOfInUseLists; i = next_chunk_index(i)) {
1919     Metachunk* chunk = chunks_in_use(i);
1920     while (chunk != NULL) {
1921       free += chunk->free_word_size();
1922       chunk = chunk->next();
1923     }
1924   }
1925   return free;
1926 }
1927 
1928 size_t SpaceManager::sum_waste_in_chunks_in_use() const {
1929   MutexLockerEx cl(lock(), Mutex::_no_safepoint_check_flag);
1930   size_t result = 0;
1931   for (ChunkIndex i = ZeroIndex; i < NumberOfInUseLists; i = next_chunk_index(i)) {
1932    result += sum_waste_in_chunks_in_use(i);
1933   }
1934 
1935   return result;
1936 }
1937 
1938 size_t SpaceManager::sum_waste_in_chunks_in_use(ChunkIndex index) const {
1939   size_t result = 0;
1940   Metachunk* chunk = chunks_in_use(index);
1941   // Count the free space in all the chunk but not the
1942   // current chunk from which allocations are still being done.
1943   while (chunk != NULL) {
1944     if (chunk != current_chunk()) {
1945       result += chunk->free_word_size();
1946     }
1947     chunk = chunk->next();
1948   }
1949   return result;
1950 }
1951 
1952 size_t SpaceManager::sum_capacity_in_chunks_in_use() const {
1953   // For CMS use "allocated_chunks_words()" which does not need the
1954   // Metaspace lock.  For the other collectors sum over the
1955   // lists.  Use both methods as a check that "allocated_chunks_words()"
1956   // is correct.  That is, sum_capacity_in_chunks() is too expensive
1957   // to use in the product and allocated_chunks_words() should be used
1958   // but allow for  checking that allocated_chunks_words() returns the same
1959   // value as sum_capacity_in_chunks_in_use() which is the definitive
1960   // answer.
1961   if (UseConcMarkSweepGC) {
1962     return allocated_chunks_words();
1963   } else {
1964     MutexLockerEx cl(lock(), Mutex::_no_safepoint_check_flag);
1965     size_t sum = 0;
1966     for (ChunkIndex i = ZeroIndex; i < NumberOfInUseLists; i = next_chunk_index(i)) {
1967       Metachunk* chunk = chunks_in_use(i);
1968       while (chunk != NULL) {
1969         sum += chunk->word_size();
1970         chunk = chunk->next();
1971       }
1972     }
1973   return sum;
1974   }
1975 }
1976 
1977 size_t SpaceManager::sum_count_in_chunks_in_use() {
1978   size_t count = 0;
1979   for (ChunkIndex i = ZeroIndex; i < NumberOfInUseLists; i = next_chunk_index(i)) {
1980     count = count + sum_count_in_chunks_in_use(i);
1981   }
1982 
1983   return count;
1984 }
1985 
1986 size_t SpaceManager::sum_count_in_chunks_in_use(ChunkIndex i) {
1987   size_t count = 0;
1988   Metachunk* chunk = chunks_in_use(i);
1989   while (chunk != NULL) {
1990     count++;
1991     chunk = chunk->next();
1992   }
1993   return count;
1994 }
1995 
1996 
1997 size_t SpaceManager::sum_used_in_chunks_in_use() const {
1998   MutexLockerEx cl(lock(), Mutex::_no_safepoint_check_flag);
1999   size_t used = 0;
2000   for (ChunkIndex i = ZeroIndex; i < NumberOfInUseLists; i = next_chunk_index(i)) {
2001     Metachunk* chunk = chunks_in_use(i);
2002     while (chunk != NULL) {
2003       used += chunk->used_word_size();
2004       chunk = chunk->next();
2005     }
2006   }
2007   return used;
2008 }
2009 
2010 void SpaceManager::locked_print_chunks_in_use_on(outputStream* st) const {
2011 
2012   for (ChunkIndex i = ZeroIndex; i < NumberOfInUseLists; i = next_chunk_index(i)) {
2013     Metachunk* chunk = chunks_in_use(i);
2014     st->print("SpaceManager: %s " PTR_FORMAT,
2015                  chunk_size_name(i), chunk);
2016     if (chunk != NULL) {
2017       st->print_cr(" free " SIZE_FORMAT,
2018                    chunk->free_word_size());
2019     } else {
2020       st->cr();
2021     }
2022   }
2023 
2024   chunk_manager()->locked_print_free_chunks(st);
2025   chunk_manager()->locked_print_sum_free_chunks(st);
2026 }
2027 
2028 size_t SpaceManager::calc_chunk_size(size_t word_size) {
2029 
2030   // Decide between a small chunk and a medium chunk.  Up to
2031   // _small_chunk_limit small chunks can be allocated but
2032   // once a medium chunk has been allocated, no more small
2033   // chunks will be allocated.
2034   size_t chunk_word_size;
2035   if (chunks_in_use(MediumIndex) == NULL &&
2036       sum_count_in_chunks_in_use(SmallIndex) < _small_chunk_limit) {
2037     chunk_word_size = (size_t) small_chunk_size();
2038     if (word_size + Metachunk::overhead() > small_chunk_size()) {
2039       chunk_word_size = medium_chunk_size();
2040     }
2041   } else {
2042     chunk_word_size = medium_chunk_size();
2043   }
2044 
2045   // Might still need a humongous chunk.  Enforce
2046   // humongous allocations sizes to be aligned up to
2047   // the smallest chunk size.
2048   size_t if_humongous_sized_chunk =
2049     align_size_up(word_size + Metachunk::overhead(),
2050                   smallest_chunk_size());
2051   chunk_word_size =
2052     MAX2((size_t) chunk_word_size, if_humongous_sized_chunk);
2053 
2054   assert(!SpaceManager::is_humongous(word_size) ||
2055          chunk_word_size == if_humongous_sized_chunk,
2056          err_msg("Size calculation is wrong, word_size " SIZE_FORMAT
2057                  " chunk_word_size " SIZE_FORMAT,
2058                  word_size, chunk_word_size));
2059   if (TraceMetadataHumongousAllocation &&
2060       SpaceManager::is_humongous(word_size)) {
2061     gclog_or_tty->print_cr("Metadata humongous allocation:");
2062     gclog_or_tty->print_cr("  word_size " PTR_FORMAT, word_size);
2063     gclog_or_tty->print_cr("  chunk_word_size " PTR_FORMAT,
2064                            chunk_word_size);
2065     gclog_or_tty->print_cr("    chunk overhead " PTR_FORMAT,
2066                            Metachunk::overhead());
2067   }
2068   return chunk_word_size;
2069 }
2070 
2071 void SpaceManager::track_metaspace_memory_usage() {
2072   if (is_init_completed()) {
2073     if (is_class()) {
2074       MemoryService::track_compressed_class_memory_usage();
2075     }
2076     MemoryService::track_metaspace_memory_usage();
2077   }
2078 }
2079 
2080 MetaWord* SpaceManager::grow_and_allocate(size_t word_size) {
2081   assert(vs_list()->current_virtual_space() != NULL,
2082          "Should have been set");
2083   assert(current_chunk() == NULL ||
2084          current_chunk()->allocate(word_size) == NULL,
2085          "Don't need to expand");
2086   MutexLockerEx cl(SpaceManager::expand_lock(), Mutex::_no_safepoint_check_flag);
2087 
2088   if (TraceMetadataChunkAllocation && Verbose) {
2089     size_t words_left = 0;
2090     size_t words_used = 0;
2091     if (current_chunk() != NULL) {
2092       words_left = current_chunk()->free_word_size();
2093       words_used = current_chunk()->used_word_size();
2094     }
2095     gclog_or_tty->print_cr("SpaceManager::grow_and_allocate for " SIZE_FORMAT
2096                            " words " SIZE_FORMAT " words used " SIZE_FORMAT
2097                            " words left",
2098                             word_size, words_used, words_left);
2099   }
2100 
2101   // Get another chunk out of the virtual space
2102   size_t grow_chunks_by_words = calc_chunk_size(word_size);
2103   Metachunk* next = get_new_chunk(word_size, grow_chunks_by_words);
2104 
2105   MetaWord* mem = NULL;
2106 
2107   // If a chunk was available, add it to the in-use chunk list
2108   // and do an allocation from it.
2109   if (next != NULL) {
2110     // Add to this manager's list of chunks in use.
2111     add_chunk(next, false);
2112     mem = next->allocate(word_size);
2113   }
2114 
2115   // Track metaspace memory usage statistic.
2116   track_metaspace_memory_usage();
2117 
2118   return mem;
2119 }
2120 
2121 void SpaceManager::print_on(outputStream* st) const {
2122 
2123   for (ChunkIndex i = ZeroIndex;
2124        i < NumberOfInUseLists ;
2125        i = next_chunk_index(i) ) {
2126     st->print_cr("  chunks_in_use " PTR_FORMAT " chunk size " PTR_FORMAT,
2127                  chunks_in_use(i),
2128                  chunks_in_use(i) == NULL ? 0 : chunks_in_use(i)->word_size());
2129   }
2130   st->print_cr("    waste:  Small " SIZE_FORMAT " Medium " SIZE_FORMAT
2131                " Humongous " SIZE_FORMAT,
2132                sum_waste_in_chunks_in_use(SmallIndex),
2133                sum_waste_in_chunks_in_use(MediumIndex),
2134                sum_waste_in_chunks_in_use(HumongousIndex));
2135   // block free lists
2136   if (block_freelists() != NULL) {
2137     st->print_cr("total in block free lists " SIZE_FORMAT,
2138       block_freelists()->total_size());
2139   }
2140 }
2141 
2142 SpaceManager::SpaceManager(Metaspace::MetadataType mdtype,
2143                            Mutex* lock) :
2144   _mdtype(mdtype),
2145   _allocated_blocks_words(0),
2146   _allocated_chunks_words(0),
2147   _allocated_chunks_count(0),
2148   _lock(lock)
2149 {
2150   initialize();
2151 }
2152 
2153 void SpaceManager::inc_size_metrics(size_t words) {
2154   assert_lock_strong(SpaceManager::expand_lock());
2155   // Total of allocated Metachunks and allocated Metachunks count
2156   // for each SpaceManager
2157   _allocated_chunks_words = _allocated_chunks_words + words;
2158   _allocated_chunks_count++;
2159   // Global total of capacity in allocated Metachunks
2160   MetaspaceAux::inc_capacity(mdtype(), words);
2161   // Global total of allocated Metablocks.
2162   // used_words_slow() includes the overhead in each
2163   // Metachunk so include it in the used when the
2164   // Metachunk is first added (so only added once per
2165   // Metachunk).
2166   MetaspaceAux::inc_used(mdtype(), Metachunk::overhead());
2167 }
2168 
2169 void SpaceManager::inc_used_metrics(size_t words) {
2170   // Add to the per SpaceManager total
2171   Atomic::add_ptr(words, &_allocated_blocks_words);
2172   // Add to the global total
2173   MetaspaceAux::inc_used(mdtype(), words);
2174 }
2175 
2176 void SpaceManager::dec_total_from_size_metrics() {
2177   MetaspaceAux::dec_capacity(mdtype(), allocated_chunks_words());
2178   MetaspaceAux::dec_used(mdtype(), allocated_blocks_words());
2179   // Also deduct the overhead per Metachunk
2180   MetaspaceAux::dec_used(mdtype(), allocated_chunks_count() * Metachunk::overhead());
2181 }
2182 
2183 void SpaceManager::initialize() {
2184   Metadebug::init_allocation_fail_alot_count();
2185   for (ChunkIndex i = ZeroIndex; i < NumberOfInUseLists; i = next_chunk_index(i)) {
2186     _chunks_in_use[i] = NULL;
2187   }
2188   _current_chunk = NULL;
2189   if (TraceMetadataChunkAllocation && Verbose) {
2190     gclog_or_tty->print_cr("SpaceManager(): " PTR_FORMAT, this);
2191   }
2192 }
2193 
2194 void ChunkManager::return_chunks(ChunkIndex index, Metachunk* chunks) {
2195   if (chunks == NULL) {
2196     return;
2197   }
2198   ChunkList* list = free_chunks(index);
2199   assert(list->size() == chunks->word_size(), "Mismatch in chunk sizes");
2200   assert_lock_strong(SpaceManager::expand_lock());
2201   Metachunk* cur = chunks;
2202 
2203   // This returns chunks one at a time.  If a new
2204   // class List can be created that is a base class
2205   // of FreeList then something like FreeList::prepend()
2206   // can be used in place of this loop
2207   while (cur != NULL) {
2208     assert(cur->container() != NULL, "Container should have been set");
2209     cur->container()->dec_container_count();
2210     // Capture the next link before it is changed
2211     // by the call to return_chunk_at_head();
2212     Metachunk* next = cur->next();
2213     DEBUG_ONLY(cur->set_is_tagged_free(true);)
2214     list->return_chunk_at_head(cur);
2215     cur = next;
2216   }
2217 }
2218 
2219 SpaceManager::~SpaceManager() {
2220   // This call this->_lock which can't be done while holding expand_lock()
2221   assert(sum_capacity_in_chunks_in_use() == allocated_chunks_words(),
2222     err_msg("sum_capacity_in_chunks_in_use() " SIZE_FORMAT
2223             " allocated_chunks_words() " SIZE_FORMAT,
2224             sum_capacity_in_chunks_in_use(), allocated_chunks_words()));
2225 
2226   MutexLockerEx fcl(SpaceManager::expand_lock(),
2227                     Mutex::_no_safepoint_check_flag);
2228 
2229   chunk_manager()->slow_locked_verify();
2230 
2231   dec_total_from_size_metrics();
2232 
2233   if (TraceMetadataChunkAllocation && Verbose) {
2234     gclog_or_tty->print_cr("~SpaceManager(): " PTR_FORMAT, this);
2235     locked_print_chunks_in_use_on(gclog_or_tty);
2236   }
2237 
2238   // Do not mangle freed Metachunks.  The chunk size inside Metachunks
2239   // is during the freeing of a VirtualSpaceNodes.
2240 
2241   // Have to update before the chunks_in_use lists are emptied
2242   // below.
2243   chunk_manager()->inc_free_chunks_total(allocated_chunks_words(),
2244                                          sum_count_in_chunks_in_use());
2245 
2246   // Add all the chunks in use by this space manager
2247   // to the global list of free chunks.
2248 
2249   // Follow each list of chunks-in-use and add them to the
2250   // free lists.  Each list is NULL terminated.
2251 
2252   for (ChunkIndex i = ZeroIndex; i < HumongousIndex; i = next_chunk_index(i)) {
2253     if (TraceMetadataChunkAllocation && Verbose) {
2254       gclog_or_tty->print_cr("returned %d %s chunks to freelist",
2255                              sum_count_in_chunks_in_use(i),
2256                              chunk_size_name(i));
2257     }
2258     Metachunk* chunks = chunks_in_use(i);
2259     chunk_manager()->return_chunks(i, chunks);
2260     set_chunks_in_use(i, NULL);
2261     if (TraceMetadataChunkAllocation && Verbose) {
2262       gclog_or_tty->print_cr("updated freelist count %d %s",
2263                              chunk_manager()->free_chunks(i)->count(),
2264                              chunk_size_name(i));
2265     }
2266     assert(i != HumongousIndex, "Humongous chunks are handled explicitly later");
2267   }
2268 
2269   // The medium chunk case may be optimized by passing the head and
2270   // tail of the medium chunk list to add_at_head().  The tail is often
2271   // the current chunk but there are probably exceptions.
2272 
2273   // Humongous chunks
2274   if (TraceMetadataChunkAllocation && Verbose) {
2275     gclog_or_tty->print_cr("returned %d %s humongous chunks to dictionary",
2276                             sum_count_in_chunks_in_use(HumongousIndex),
2277                             chunk_size_name(HumongousIndex));
2278     gclog_or_tty->print("Humongous chunk dictionary: ");
2279   }
2280   // Humongous chunks are never the current chunk.
2281   Metachunk* humongous_chunks = chunks_in_use(HumongousIndex);
2282 
2283   while (humongous_chunks != NULL) {
2284 #ifdef ASSERT
2285     humongous_chunks->set_is_tagged_free(true);
2286 #endif
2287     if (TraceMetadataChunkAllocation && Verbose) {
2288       gclog_or_tty->print(PTR_FORMAT " (" SIZE_FORMAT ") ",
2289                           humongous_chunks,
2290                           humongous_chunks->word_size());
2291     }
2292     assert(humongous_chunks->word_size() == (size_t)
2293            align_size_up(humongous_chunks->word_size(),
2294                              smallest_chunk_size()),
2295            err_msg("Humongous chunk size is wrong: word size " SIZE_FORMAT
2296                    " granularity %d",
2297                    humongous_chunks->word_size(), smallest_chunk_size()));
2298     Metachunk* next_humongous_chunks = humongous_chunks->next();
2299     humongous_chunks->container()->dec_container_count();
2300     chunk_manager()->humongous_dictionary()->return_chunk(humongous_chunks);
2301     humongous_chunks = next_humongous_chunks;
2302   }
2303   if (TraceMetadataChunkAllocation && Verbose) {
2304     gclog_or_tty->cr();
2305     gclog_or_tty->print_cr("updated dictionary count %d %s",
2306                      chunk_manager()->humongous_dictionary()->total_count(),
2307                      chunk_size_name(HumongousIndex));
2308   }
2309   chunk_manager()->slow_locked_verify();
2310 }
2311 
2312 const char* SpaceManager::chunk_size_name(ChunkIndex index) const {
2313   switch (index) {
2314     case SpecializedIndex:
2315       return "Specialized";
2316     case SmallIndex:
2317       return "Small";
2318     case MediumIndex:
2319       return "Medium";
2320     case HumongousIndex:
2321       return "Humongous";
2322     default:
2323       return NULL;
2324   }
2325 }
2326 
2327 ChunkIndex ChunkManager::list_index(size_t size) {
2328   switch (size) {
2329     case SpecializedChunk:
2330       assert(SpecializedChunk == ClassSpecializedChunk,
2331              "Need branch for ClassSpecializedChunk");
2332       return SpecializedIndex;
2333     case SmallChunk:
2334     case ClassSmallChunk:
2335       return SmallIndex;
2336     case MediumChunk:
2337     case ClassMediumChunk:
2338       return MediumIndex;
2339     default:
2340       assert(size > MediumChunk || size > ClassMediumChunk,
2341              "Not a humongous chunk");
2342       return HumongousIndex;
2343   }
2344 }
2345 
2346 void SpaceManager::deallocate(MetaWord* p, size_t word_size) {
2347   assert_lock_strong(_lock);
2348   size_t raw_word_size = get_raw_word_size(word_size);
2349   size_t min_size = TreeChunk<Metablock, FreeList<Metablock> >::min_size();
2350   assert(raw_word_size >= min_size,
2351          err_msg("Should not deallocate dark matter " SIZE_FORMAT "<" SIZE_FORMAT, word_size, min_size));
2352   block_freelists()->return_block(p, raw_word_size);
2353 }
2354 
2355 // Adds a chunk to the list of chunks in use.
2356 void SpaceManager::add_chunk(Metachunk* new_chunk, bool make_current) {
2357 
2358   assert(new_chunk != NULL, "Should not be NULL");
2359   assert(new_chunk->next() == NULL, "Should not be on a list");
2360 
2361   new_chunk->reset_empty();
2362 
2363   // Find the correct list and and set the current
2364   // chunk for that list.
2365   ChunkIndex index = ChunkManager::list_index(new_chunk->word_size());
2366 
2367   if (index != HumongousIndex) {
2368     retire_current_chunk();
2369     set_current_chunk(new_chunk);
2370     new_chunk->set_next(chunks_in_use(index));
2371     set_chunks_in_use(index, new_chunk);
2372   } else {
2373     // For null class loader data and DumpSharedSpaces, the first chunk isn't
2374     // small, so small will be null.  Link this first chunk as the current
2375     // chunk.
2376     if (make_current) {
2377       // Set as the current chunk but otherwise treat as a humongous chunk.
2378       set_current_chunk(new_chunk);
2379     }
2380     // Link at head.  The _current_chunk only points to a humongous chunk for
2381     // the null class loader metaspace (class and data virtual space managers)
2382     // any humongous chunks so will not point to the tail
2383     // of the humongous chunks list.
2384     new_chunk->set_next(chunks_in_use(HumongousIndex));
2385     set_chunks_in_use(HumongousIndex, new_chunk);
2386 
2387     assert(new_chunk->word_size() > medium_chunk_size(), "List inconsistency");
2388   }
2389 
2390   // Add to the running sum of capacity
2391   inc_size_metrics(new_chunk->word_size());
2392 
2393   assert(new_chunk->is_empty(), "Not ready for reuse");
2394   if (TraceMetadataChunkAllocation && Verbose) {
2395     gclog_or_tty->print("SpaceManager::add_chunk: %d) ",
2396                         sum_count_in_chunks_in_use());
2397     new_chunk->print_on(gclog_or_tty);
2398     chunk_manager()->locked_print_free_chunks(gclog_or_tty);
2399   }
2400 }
2401 
2402 void SpaceManager::retire_current_chunk() {
2403   if (current_chunk() != NULL) {
2404     size_t remaining_words = current_chunk()->free_word_size();
2405     if (remaining_words >= TreeChunk<Metablock, FreeList<Metablock> >::min_size()) {
2406       block_freelists()->return_block(current_chunk()->allocate(remaining_words), remaining_words);
2407       inc_used_metrics(remaining_words);
2408     }
2409   }
2410 }
2411 
2412 Metachunk* SpaceManager::get_new_chunk(size_t word_size,
2413                                        size_t grow_chunks_by_words) {
2414   // Get a chunk from the chunk freelist
2415   Metachunk* next = chunk_manager()->chunk_freelist_allocate(grow_chunks_by_words);
2416 
2417   if (next == NULL) {
2418     next = vs_list()->get_new_chunk(word_size,
2419                                     grow_chunks_by_words,
2420                                     medium_chunk_bunch());
2421   }
2422 
2423   if (TraceMetadataHumongousAllocation && next != NULL &&
2424       SpaceManager::is_humongous(next->word_size())) {
2425     gclog_or_tty->print_cr("  new humongous chunk word size "
2426                            PTR_FORMAT, next->word_size());
2427   }
2428 
2429   return next;
2430 }
2431 
2432 MetaWord* SpaceManager::allocate(size_t word_size) {
2433   MutexLockerEx cl(lock(), Mutex::_no_safepoint_check_flag);
2434 
2435   size_t raw_word_size = get_raw_word_size(word_size);
2436   BlockFreelist* fl =  block_freelists();
2437   MetaWord* p = NULL;
2438   // Allocation from the dictionary is expensive in the sense that
2439   // the dictionary has to be searched for a size.  Don't allocate
2440   // from the dictionary until it starts to get fat.  Is this
2441   // a reasonable policy?  Maybe an skinny dictionary is fast enough
2442   // for allocations.  Do some profiling.  JJJ
2443   if (fl->total_size() > allocation_from_dictionary_limit) {
2444     p = fl->get_block(raw_word_size);
2445   }
2446   if (p == NULL) {
2447     p = allocate_work(raw_word_size);
2448   }
2449 
2450   return p;
2451 }
2452 
2453 // Returns the address of spaced allocated for "word_size".
2454 // This methods does not know about blocks (Metablocks)
2455 MetaWord* SpaceManager::allocate_work(size_t word_size) {
2456   assert_lock_strong(_lock);
2457 #ifdef ASSERT
2458   if (Metadebug::test_metadata_failure()) {
2459     return NULL;
2460   }
2461 #endif
2462   // Is there space in the current chunk?
2463   MetaWord* result = NULL;
2464 
2465   // For DumpSharedSpaces, only allocate out of the current chunk which is
2466   // never null because we gave it the size we wanted.   Caller reports out
2467   // of memory if this returns null.
2468   if (DumpSharedSpaces) {
2469     assert(current_chunk() != NULL, "should never happen");
2470     inc_used_metrics(word_size);
2471     return current_chunk()->allocate(word_size); // caller handles null result
2472   }
2473 
2474   if (current_chunk() != NULL) {
2475     result = current_chunk()->allocate(word_size);
2476   }
2477 
2478   if (result == NULL) {
2479     result = grow_and_allocate(word_size);
2480   }
2481 
2482   if (result != NULL) {
2483     inc_used_metrics(word_size);
2484     assert(result != (MetaWord*) chunks_in_use(MediumIndex),
2485            "Head of the list is being allocated");
2486   }
2487 
2488   return result;
2489 }
2490 
2491 void SpaceManager::verify() {
2492   // If there are blocks in the dictionary, then
2493   // verification of chunks does not work since
2494   // being in the dictionary alters a chunk.
2495   if (block_freelists()->total_size() == 0) {
2496     for (ChunkIndex i = ZeroIndex; i < NumberOfInUseLists; i = next_chunk_index(i)) {
2497       Metachunk* curr = chunks_in_use(i);
2498       while (curr != NULL) {
2499         curr->verify();
2500         verify_chunk_size(curr);
2501         curr = curr->next();
2502       }
2503     }
2504   }
2505 }
2506 
2507 void SpaceManager::verify_chunk_size(Metachunk* chunk) {
2508   assert(is_humongous(chunk->word_size()) ||
2509          chunk->word_size() == medium_chunk_size() ||
2510          chunk->word_size() == small_chunk_size() ||
2511          chunk->word_size() == specialized_chunk_size(),
2512          "Chunk size is wrong");
2513   return;
2514 }
2515 
2516 #ifdef ASSERT
2517 void SpaceManager::verify_allocated_blocks_words() {
2518   // Verification is only guaranteed at a safepoint.
2519   assert(SafepointSynchronize::is_at_safepoint() || !Universe::is_fully_initialized(),
2520     "Verification can fail if the applications is running");
2521   assert(allocated_blocks_words() == sum_used_in_chunks_in_use(),
2522     err_msg("allocation total is not consistent " SIZE_FORMAT
2523             " vs " SIZE_FORMAT,
2524             allocated_blocks_words(), sum_used_in_chunks_in_use()));
2525 }
2526 
2527 #endif
2528 
2529 void SpaceManager::dump(outputStream* const out) const {
2530   size_t curr_total = 0;
2531   size_t waste = 0;
2532   uint i = 0;
2533   size_t used = 0;
2534   size_t capacity = 0;
2535 
2536   // Add up statistics for all chunks in this SpaceManager.
2537   for (ChunkIndex index = ZeroIndex;
2538        index < NumberOfInUseLists;
2539        index = next_chunk_index(index)) {
2540     for (Metachunk* curr = chunks_in_use(index);
2541          curr != NULL;
2542          curr = curr->next()) {
2543       out->print("%d) ", i++);
2544       curr->print_on(out);
2545       curr_total += curr->word_size();
2546       used += curr->used_word_size();
2547       capacity += curr->word_size();
2548       waste += curr->free_word_size() + curr->overhead();;
2549     }
2550   }
2551 
2552   if (TraceMetadataChunkAllocation && Verbose) {
2553     block_freelists()->print_on(out);
2554   }
2555 
2556   size_t free = current_chunk() == NULL ? 0 : current_chunk()->free_word_size();
2557   // Free space isn't wasted.
2558   waste -= free;
2559 
2560   out->print_cr("total of all chunks "  SIZE_FORMAT " used " SIZE_FORMAT
2561                 " free " SIZE_FORMAT " capacity " SIZE_FORMAT
2562                 " waste " SIZE_FORMAT, curr_total, used, free, capacity, waste);
2563 }
2564 
2565 #ifndef PRODUCT
2566 void SpaceManager::mangle_freed_chunks() {
2567   for (ChunkIndex index = ZeroIndex;
2568        index < NumberOfInUseLists;
2569        index = next_chunk_index(index)) {
2570     for (Metachunk* curr = chunks_in_use(index);
2571          curr != NULL;
2572          curr = curr->next()) {
2573       curr->mangle();
2574     }
2575   }
2576 }
2577 #endif // PRODUCT
2578 
2579 // MetaspaceAux
2580 
2581 
2582 size_t MetaspaceAux::_capacity_words[] = {0, 0};
2583 size_t MetaspaceAux::_used_words[] = {0, 0};
2584 
2585 size_t MetaspaceAux::free_bytes(Metaspace::MetadataType mdtype) {
2586   VirtualSpaceList* list = Metaspace::get_space_list(mdtype);
2587   return list == NULL ? 0 : list->free_bytes();
2588 }
2589 
2590 size_t MetaspaceAux::free_bytes() {
2591   return free_bytes(Metaspace::ClassType) + free_bytes(Metaspace::NonClassType);
2592 }
2593 
2594 void MetaspaceAux::dec_capacity(Metaspace::MetadataType mdtype, size_t words) {
2595   assert_lock_strong(SpaceManager::expand_lock());
2596   assert(words <= capacity_words(mdtype),
2597     err_msg("About to decrement below 0: words " SIZE_FORMAT
2598             " is greater than _capacity_words[%u] " SIZE_FORMAT,
2599             words, mdtype, capacity_words(mdtype)));
2600   _capacity_words[mdtype] -= words;
2601 }
2602 
2603 void MetaspaceAux::inc_capacity(Metaspace::MetadataType mdtype, size_t words) {
2604   assert_lock_strong(SpaceManager::expand_lock());
2605   // Needs to be atomic
2606   _capacity_words[mdtype] += words;
2607 }
2608 
2609 void MetaspaceAux::dec_used(Metaspace::MetadataType mdtype, size_t words) {
2610   assert(words <= used_words(mdtype),
2611     err_msg("About to decrement below 0: words " SIZE_FORMAT
2612             " is greater than _used_words[%u] " SIZE_FORMAT,
2613             words, mdtype, used_words(mdtype)));
2614   // For CMS deallocation of the Metaspaces occurs during the
2615   // sweep which is a concurrent phase.  Protection by the expand_lock()
2616   // is not enough since allocation is on a per Metaspace basis
2617   // and protected by the Metaspace lock.
2618   jlong minus_words = (jlong) - (jlong) words;
2619   Atomic::add_ptr(minus_words, &_used_words[mdtype]);
2620 }
2621 
2622 void MetaspaceAux::inc_used(Metaspace::MetadataType mdtype, size_t words) {
2623   // _used_words tracks allocations for
2624   // each piece of metadata.  Those allocations are
2625   // generally done concurrently by different application
2626   // threads so must be done atomically.
2627   Atomic::add_ptr(words, &_used_words[mdtype]);
2628 }
2629 
2630 size_t MetaspaceAux::used_bytes_slow(Metaspace::MetadataType mdtype) {
2631   size_t used = 0;
2632   ClassLoaderDataGraphMetaspaceIterator iter;
2633   while (iter.repeat()) {
2634     Metaspace* msp = iter.get_next();
2635     // Sum allocated_blocks_words for each metaspace
2636     if (msp != NULL) {
2637       used += msp->used_words_slow(mdtype);
2638     }
2639   }
2640   return used * BytesPerWord;
2641 }
2642 
2643 size_t MetaspaceAux::free_bytes_slow(Metaspace::MetadataType mdtype) {
2644   size_t free = 0;
2645   ClassLoaderDataGraphMetaspaceIterator iter;
2646   while (iter.repeat()) {
2647     Metaspace* msp = iter.get_next();
2648     if (msp != NULL) {
2649       free += msp->free_words_slow(mdtype);
2650     }
2651   }
2652   return free * BytesPerWord;
2653 }
2654 
2655 size_t MetaspaceAux::capacity_bytes_slow(Metaspace::MetadataType mdtype) {
2656   if ((mdtype == Metaspace::ClassType) && !Metaspace::using_class_space()) {
2657     return 0;
2658   }
2659   // Don't count the space in the freelists.  That space will be
2660   // added to the capacity calculation as needed.
2661   size_t capacity = 0;
2662   ClassLoaderDataGraphMetaspaceIterator iter;
2663   while (iter.repeat()) {
2664     Metaspace* msp = iter.get_next();
2665     if (msp != NULL) {
2666       capacity += msp->capacity_words_slow(mdtype);
2667     }
2668   }
2669   return capacity * BytesPerWord;
2670 }
2671 
2672 size_t MetaspaceAux::capacity_bytes_slow() {
2673 #ifdef PRODUCT
2674   // Use capacity_bytes() in PRODUCT instead of this function.
2675   guarantee(false, "Should not call capacity_bytes_slow() in the PRODUCT");
2676 #endif
2677   size_t class_capacity = capacity_bytes_slow(Metaspace::ClassType);
2678   size_t non_class_capacity = capacity_bytes_slow(Metaspace::NonClassType);
2679   assert(capacity_bytes() == class_capacity + non_class_capacity,
2680       err_msg("bad accounting: capacity_bytes() " SIZE_FORMAT
2681         " class_capacity + non_class_capacity " SIZE_FORMAT
2682         " class_capacity " SIZE_FORMAT " non_class_capacity " SIZE_FORMAT,
2683         capacity_bytes(), class_capacity + non_class_capacity,
2684         class_capacity, non_class_capacity));
2685 
2686   return class_capacity + non_class_capacity;
2687 }
2688 
2689 size_t MetaspaceAux::reserved_bytes(Metaspace::MetadataType mdtype) {
2690   VirtualSpaceList* list = Metaspace::get_space_list(mdtype);
2691   return list == NULL ? 0 : list->reserved_bytes();
2692 }
2693 
2694 size_t MetaspaceAux::committed_bytes(Metaspace::MetadataType mdtype) {
2695   VirtualSpaceList* list = Metaspace::get_space_list(mdtype);
2696   return list == NULL ? 0 : list->committed_bytes();
2697 }
2698 
2699 size_t MetaspaceAux::min_chunk_size_words() { return Metaspace::first_chunk_word_size(); }
2700 
2701 size_t MetaspaceAux::free_chunks_total_words(Metaspace::MetadataType mdtype) {
2702   ChunkManager* chunk_manager = Metaspace::get_chunk_manager(mdtype);
2703   if (chunk_manager == NULL) {
2704     return 0;
2705   }
2706   chunk_manager->slow_verify();
2707   return chunk_manager->free_chunks_total_words();
2708 }
2709 
2710 size_t MetaspaceAux::free_chunks_total_bytes(Metaspace::MetadataType mdtype) {
2711   return free_chunks_total_words(mdtype) * BytesPerWord;
2712 }
2713 
2714 size_t MetaspaceAux::free_chunks_total_words() {
2715   return free_chunks_total_words(Metaspace::ClassType) +
2716          free_chunks_total_words(Metaspace::NonClassType);
2717 }
2718 
2719 size_t MetaspaceAux::free_chunks_total_bytes() {
2720   return free_chunks_total_words() * BytesPerWord;
2721 }
2722 
2723 bool MetaspaceAux::has_chunk_free_list(Metaspace::MetadataType mdtype) {
2724   return Metaspace::get_chunk_manager(mdtype) != NULL;
2725 }
2726 
2727 MetaspaceChunkFreeListSummary MetaspaceAux::chunk_free_list_summary(Metaspace::MetadataType mdtype) {
2728   if (!has_chunk_free_list(mdtype)) {
2729     return MetaspaceChunkFreeListSummary();
2730   }
2731 
2732   const ChunkManager* cm = Metaspace::get_chunk_manager(mdtype);
2733   return cm->chunk_free_list_summary();
2734 }
2735 
2736 void MetaspaceAux::print_metaspace_change(size_t prev_metadata_used) {
2737   gclog_or_tty->print(", [Metaspace:");
2738   if (PrintGCDetails && Verbose) {
2739     gclog_or_tty->print(" "  SIZE_FORMAT
2740                         "->" SIZE_FORMAT
2741                         "("  SIZE_FORMAT ")",
2742                         prev_metadata_used,
2743                         used_bytes(),
2744                         reserved_bytes());
2745   } else {
2746     gclog_or_tty->print(" "  SIZE_FORMAT "K"
2747                         "->" SIZE_FORMAT "K"
2748                         "("  SIZE_FORMAT "K)",
2749                         prev_metadata_used/K,
2750                         used_bytes()/K,
2751                         reserved_bytes()/K);
2752   }
2753 
2754   gclog_or_tty->print("]");
2755 }
2756 
2757 // This is printed when PrintGCDetails
2758 void MetaspaceAux::print_on(outputStream* out) {
2759   Metaspace::MetadataType nct = Metaspace::NonClassType;
2760 
2761   out->print_cr(" Metaspace       "
2762                 "used "      SIZE_FORMAT "K, "
2763                 "capacity "  SIZE_FORMAT "K, "
2764                 "committed " SIZE_FORMAT "K, "
2765                 "reserved "  SIZE_FORMAT "K",
2766                 used_bytes()/K,
2767                 capacity_bytes()/K,
2768                 committed_bytes()/K,
2769                 reserved_bytes()/K);
2770 
2771   if (Metaspace::using_class_space()) {
2772     Metaspace::MetadataType ct = Metaspace::ClassType;
2773     out->print_cr("  class space    "
2774                   "used "      SIZE_FORMAT "K, "
2775                   "capacity "  SIZE_FORMAT "K, "
2776                   "committed " SIZE_FORMAT "K, "
2777                   "reserved "  SIZE_FORMAT "K",
2778                   used_bytes(ct)/K,
2779                   capacity_bytes(ct)/K,
2780                   committed_bytes(ct)/K,
2781                   reserved_bytes(ct)/K);
2782   }
2783 }
2784 
2785 // Print information for class space and data space separately.
2786 // This is almost the same as above.
2787 void MetaspaceAux::print_on(outputStream* out, Metaspace::MetadataType mdtype) {
2788   size_t free_chunks_capacity_bytes = free_chunks_total_bytes(mdtype);
2789   size_t capacity_bytes = capacity_bytes_slow(mdtype);
2790   size_t used_bytes = used_bytes_slow(mdtype);
2791   size_t free_bytes = free_bytes_slow(mdtype);
2792   size_t used_and_free = used_bytes + free_bytes +
2793                            free_chunks_capacity_bytes;
2794   out->print_cr("  Chunk accounting: used in chunks " SIZE_FORMAT
2795              "K + unused in chunks " SIZE_FORMAT "K  + "
2796              " capacity in free chunks " SIZE_FORMAT "K = " SIZE_FORMAT
2797              "K  capacity in allocated chunks " SIZE_FORMAT "K",
2798              used_bytes / K,
2799              free_bytes / K,
2800              free_chunks_capacity_bytes / K,
2801              used_and_free / K,
2802              capacity_bytes / K);
2803   // Accounting can only be correct if we got the values during a safepoint
2804   assert(!SafepointSynchronize::is_at_safepoint() || used_and_free == capacity_bytes, "Accounting is wrong");
2805 }
2806 
2807 // Print total fragmentation for class metaspaces
2808 void MetaspaceAux::print_class_waste(outputStream* out) {
2809   assert(Metaspace::using_class_space(), "class metaspace not used");
2810   size_t cls_specialized_waste = 0, cls_small_waste = 0, cls_medium_waste = 0;
2811   size_t cls_specialized_count = 0, cls_small_count = 0, cls_medium_count = 0, cls_humongous_count = 0;
2812   ClassLoaderDataGraphMetaspaceIterator iter;
2813   while (iter.repeat()) {
2814     Metaspace* msp = iter.get_next();
2815     if (msp != NULL) {
2816       cls_specialized_waste += msp->class_vsm()->sum_waste_in_chunks_in_use(SpecializedIndex);
2817       cls_specialized_count += msp->class_vsm()->sum_count_in_chunks_in_use(SpecializedIndex);
2818       cls_small_waste += msp->class_vsm()->sum_waste_in_chunks_in_use(SmallIndex);
2819       cls_small_count += msp->class_vsm()->sum_count_in_chunks_in_use(SmallIndex);
2820       cls_medium_waste += msp->class_vsm()->sum_waste_in_chunks_in_use(MediumIndex);
2821       cls_medium_count += msp->class_vsm()->sum_count_in_chunks_in_use(MediumIndex);
2822       cls_humongous_count += msp->class_vsm()->sum_count_in_chunks_in_use(HumongousIndex);
2823     }
2824   }
2825   out->print_cr(" class: " SIZE_FORMAT " specialized(s) " SIZE_FORMAT ", "
2826                 SIZE_FORMAT " small(s) " SIZE_FORMAT ", "
2827                 SIZE_FORMAT " medium(s) " SIZE_FORMAT ", "
2828                 "large count " SIZE_FORMAT,
2829                 cls_specialized_count, cls_specialized_waste,
2830                 cls_small_count, cls_small_waste,
2831                 cls_medium_count, cls_medium_waste, cls_humongous_count);
2832 }
2833 
2834 // Print total fragmentation for data and class metaspaces separately
2835 void MetaspaceAux::print_waste(outputStream* out) {
2836   size_t specialized_waste = 0, small_waste = 0, medium_waste = 0;
2837   size_t specialized_count = 0, small_count = 0, medium_count = 0, humongous_count = 0;
2838 
2839   ClassLoaderDataGraphMetaspaceIterator iter;
2840   while (iter.repeat()) {
2841     Metaspace* msp = iter.get_next();
2842     if (msp != NULL) {
2843       specialized_waste += msp->vsm()->sum_waste_in_chunks_in_use(SpecializedIndex);
2844       specialized_count += msp->vsm()->sum_count_in_chunks_in_use(SpecializedIndex);
2845       small_waste += msp->vsm()->sum_waste_in_chunks_in_use(SmallIndex);
2846       small_count += msp->vsm()->sum_count_in_chunks_in_use(SmallIndex);
2847       medium_waste += msp->vsm()->sum_waste_in_chunks_in_use(MediumIndex);
2848       medium_count += msp->vsm()->sum_count_in_chunks_in_use(MediumIndex);
2849       humongous_count += msp->vsm()->sum_count_in_chunks_in_use(HumongousIndex);
2850     }
2851   }
2852   out->print_cr("Total fragmentation waste (words) doesn't count free space");
2853   out->print_cr("  data: " SIZE_FORMAT " specialized(s) " SIZE_FORMAT ", "
2854                         SIZE_FORMAT " small(s) " SIZE_FORMAT ", "
2855                         SIZE_FORMAT " medium(s) " SIZE_FORMAT ", "
2856                         "large count " SIZE_FORMAT,
2857              specialized_count, specialized_waste, small_count,
2858              small_waste, medium_count, medium_waste, humongous_count);
2859   if (Metaspace::using_class_space()) {
2860     print_class_waste(out);
2861   }
2862 }
2863 
2864 // Dump global metaspace things from the end of ClassLoaderDataGraph
2865 void MetaspaceAux::dump(outputStream* out) {
2866   out->print_cr("All Metaspace:");
2867   out->print("data space: "); print_on(out, Metaspace::NonClassType);
2868   out->print("class space: "); print_on(out, Metaspace::ClassType);
2869   print_waste(out);
2870 }
2871 
2872 void MetaspaceAux::verify_free_chunks() {
2873   Metaspace::chunk_manager_metadata()->verify();
2874   if (Metaspace::using_class_space()) {
2875     Metaspace::chunk_manager_class()->verify();
2876   }
2877 }
2878 
2879 void MetaspaceAux::verify_capacity() {
2880 #ifdef ASSERT
2881   size_t running_sum_capacity_bytes = capacity_bytes();
2882   // For purposes of the running sum of capacity, verify against capacity
2883   size_t capacity_in_use_bytes = capacity_bytes_slow();
2884   assert(running_sum_capacity_bytes == capacity_in_use_bytes,
2885     err_msg("capacity_words() * BytesPerWord " SIZE_FORMAT
2886             " capacity_bytes_slow()" SIZE_FORMAT,
2887             running_sum_capacity_bytes, capacity_in_use_bytes));
2888   for (Metaspace::MetadataType i = Metaspace::ClassType;
2889        i < Metaspace:: MetadataTypeCount;
2890        i = (Metaspace::MetadataType)(i + 1)) {
2891     size_t capacity_in_use_bytes = capacity_bytes_slow(i);
2892     assert(capacity_bytes(i) == capacity_in_use_bytes,
2893       err_msg("capacity_bytes(%u) " SIZE_FORMAT
2894               " capacity_bytes_slow(%u)" SIZE_FORMAT,
2895               i, capacity_bytes(i), i, capacity_in_use_bytes));
2896   }
2897 #endif
2898 }
2899 
2900 void MetaspaceAux::verify_used() {
2901 #ifdef ASSERT
2902   size_t running_sum_used_bytes = used_bytes();
2903   // For purposes of the running sum of used, verify against used
2904   size_t used_in_use_bytes = used_bytes_slow();
2905   assert(used_bytes() == used_in_use_bytes,
2906     err_msg("used_bytes() " SIZE_FORMAT
2907             " used_bytes_slow()" SIZE_FORMAT,
2908             used_bytes(), used_in_use_bytes));
2909   for (Metaspace::MetadataType i = Metaspace::ClassType;
2910        i < Metaspace:: MetadataTypeCount;
2911        i = (Metaspace::MetadataType)(i + 1)) {
2912     size_t used_in_use_bytes = used_bytes_slow(i);
2913     assert(used_bytes(i) == used_in_use_bytes,
2914       err_msg("used_bytes(%u) " SIZE_FORMAT
2915               " used_bytes_slow(%u)" SIZE_FORMAT,
2916               i, used_bytes(i), i, used_in_use_bytes));
2917   }
2918 #endif
2919 }
2920 
2921 void MetaspaceAux::verify_metrics() {
2922   verify_capacity();
2923   verify_used();
2924 }
2925 
2926 
2927 // Metaspace methods
2928 
2929 size_t Metaspace::_first_chunk_word_size = 0;
2930 size_t Metaspace::_first_class_chunk_word_size = 0;
2931 
2932 size_t Metaspace::_commit_alignment = 0;
2933 size_t Metaspace::_reserve_alignment = 0;
2934 
2935 Metaspace::Metaspace(Mutex* lock, MetaspaceType type) {
2936   initialize(lock, type);
2937 }
2938 
2939 Metaspace::~Metaspace() {
2940   delete _vsm;
2941   if (using_class_space()) {
2942     delete _class_vsm;
2943   }
2944 }
2945 
2946 VirtualSpaceList* Metaspace::_space_list = NULL;
2947 VirtualSpaceList* Metaspace::_class_space_list = NULL;
2948 
2949 ChunkManager* Metaspace::_chunk_manager_metadata = NULL;
2950 ChunkManager* Metaspace::_chunk_manager_class = NULL;
2951 
2952 #define VIRTUALSPACEMULTIPLIER 2
2953 
2954 #ifdef _LP64
2955 static const uint64_t UnscaledClassSpaceMax = (uint64_t(max_juint) + 1);
2956 
2957 void Metaspace::set_narrow_klass_base_and_shift(address metaspace_base, address cds_base) {
2958   // Figure out the narrow_klass_base and the narrow_klass_shift.  The
2959   // narrow_klass_base is the lower of the metaspace base and the cds base
2960   // (if cds is enabled).  The narrow_klass_shift depends on the distance
2961   // between the lower base and higher address.
2962   address lower_base;
2963   address higher_address;
2964   if (UseSharedSpaces) {
2965     higher_address = MAX2((address)(cds_base + FileMapInfo::shared_spaces_size()),
2966                           (address)(metaspace_base + compressed_class_space_size()));
2967     lower_base = MIN2(metaspace_base, cds_base);
2968   } else {
2969     higher_address = metaspace_base + compressed_class_space_size();
2970     lower_base = metaspace_base;
2971 
2972     uint64_t klass_encoding_max = UnscaledClassSpaceMax << LogKlassAlignmentInBytes;
2973     // If compressed class space fits in lower 32G, we don't need a base.
2974     if (higher_address <= (address)klass_encoding_max) {
2975       lower_base = 0; // Effectively lower base is zero.
2976     }
2977   }
2978 
2979   Universe::set_narrow_klass_base(lower_base);
2980 
2981   if ((uint64_t)(higher_address - lower_base) <= UnscaledClassSpaceMax) {
2982     Universe::set_narrow_klass_shift(0);
2983   } else {
2984     assert(!UseSharedSpaces, "Cannot shift with UseSharedSpaces");
2985     Universe::set_narrow_klass_shift(LogKlassAlignmentInBytes);
2986   }
2987 }
2988 
2989 // Return TRUE if the specified metaspace_base and cds_base are close enough
2990 // to work with compressed klass pointers.
2991 bool Metaspace::can_use_cds_with_metaspace_addr(char* metaspace_base, address cds_base) {
2992   assert(cds_base != 0 && UseSharedSpaces, "Only use with CDS");
2993   assert(UseCompressedClassPointers, "Only use with CompressedKlassPtrs");
2994   address lower_base = MIN2((address)metaspace_base, cds_base);
2995   address higher_address = MAX2((address)(cds_base + FileMapInfo::shared_spaces_size()),
2996                                 (address)(metaspace_base + compressed_class_space_size()));
2997   return ((uint64_t)(higher_address - lower_base) <= UnscaledClassSpaceMax);
2998 }
2999 
3000 // Try to allocate the metaspace at the requested addr.
3001 void Metaspace::allocate_metaspace_compressed_klass_ptrs(char* requested_addr, address cds_base) {
3002   assert(using_class_space(), "called improperly");
3003   assert(UseCompressedClassPointers, "Only use with CompressedKlassPtrs");
3004   assert(compressed_class_space_size() < KlassEncodingMetaspaceMax,
3005          "Metaspace size is too big");
3006   assert_is_ptr_aligned(requested_addr, _reserve_alignment);
3007   assert_is_ptr_aligned(cds_base, _reserve_alignment);
3008   assert_is_size_aligned(compressed_class_space_size(), _reserve_alignment);
3009 
3010   // Don't use large pages for the class space.
3011   bool large_pages = false;
3012 
3013   ReservedSpace metaspace_rs = ReservedSpace(compressed_class_space_size(),
3014                                              _reserve_alignment,
3015                                              large_pages,
3016                                              requested_addr, 0);
3017   if (!metaspace_rs.is_reserved()) {
3018     if (UseSharedSpaces) {
3019       size_t increment = align_size_up(1*G, _reserve_alignment);
3020 
3021       // Keep trying to allocate the metaspace, increasing the requested_addr
3022       // by 1GB each time, until we reach an address that will no longer allow
3023       // use of CDS with compressed klass pointers.
3024       char *addr = requested_addr;
3025       while (!metaspace_rs.is_reserved() && (addr + increment > addr) &&
3026              can_use_cds_with_metaspace_addr(addr + increment, cds_base)) {
3027         addr = addr + increment;
3028         metaspace_rs = ReservedSpace(compressed_class_space_size(),
3029                                      _reserve_alignment, large_pages, addr, 0);
3030       }
3031     }
3032 
3033     // If no successful allocation then try to allocate the space anywhere.  If
3034     // that fails then OOM doom.  At this point we cannot try allocating the
3035     // metaspace as if UseCompressedClassPointers is off because too much
3036     // initialization has happened that depends on UseCompressedClassPointers.
3037     // So, UseCompressedClassPointers cannot be turned off at this point.
3038     if (!metaspace_rs.is_reserved()) {
3039       metaspace_rs = ReservedSpace(compressed_class_space_size(),
3040                                    _reserve_alignment, large_pages);
3041       if (!metaspace_rs.is_reserved()) {
3042         vm_exit_during_initialization(err_msg("Could not allocate metaspace: %d bytes",
3043                                               compressed_class_space_size()));
3044       }
3045     }
3046   }
3047 
3048   // If we got here then the metaspace got allocated.
3049   MemTracker::record_virtual_memory_type((address)metaspace_rs.base(), mtClass);
3050 
3051   // Verify that we can use shared spaces.  Otherwise, turn off CDS.
3052   if (UseSharedSpaces && !can_use_cds_with_metaspace_addr(metaspace_rs.base(), cds_base)) {
3053     FileMapInfo::stop_sharing_and_unmap(
3054         "Could not allocate metaspace at a compatible address");
3055   }
3056 
3057   set_narrow_klass_base_and_shift((address)metaspace_rs.base(),
3058                                   UseSharedSpaces ? (address)cds_base : 0);
3059 
3060   initialize_class_space(metaspace_rs);
3061 
3062   if (PrintCompressedOopsMode || (PrintMiscellaneous && Verbose)) {
3063     gclog_or_tty->print_cr("Narrow klass base: " PTR_FORMAT ", Narrow klass shift: " SIZE_FORMAT,
3064                             Universe::narrow_klass_base(), Universe::narrow_klass_shift());
3065     gclog_or_tty->print_cr("Compressed class space size: " SIZE_FORMAT " Address: " PTR_FORMAT " Req Addr: " PTR_FORMAT,
3066                            compressed_class_space_size(), metaspace_rs.base(), requested_addr);
3067   }
3068 }
3069 
3070 // For UseCompressedClassPointers the class space is reserved above the top of
3071 // the Java heap.  The argument passed in is at the base of the compressed space.
3072 void Metaspace::initialize_class_space(ReservedSpace rs) {
3073   // The reserved space size may be bigger because of alignment, esp with UseLargePages
3074   assert(rs.size() >= CompressedClassSpaceSize,
3075          err_msg(SIZE_FORMAT " != " UINTX_FORMAT, rs.size(), CompressedClassSpaceSize));
3076   assert(using_class_space(), "Must be using class space");
3077   _class_space_list = new VirtualSpaceList(rs);
3078   _chunk_manager_class = new ChunkManager(SpecializedChunk, ClassSmallChunk, ClassMediumChunk);
3079 
3080   if (!_class_space_list->initialization_succeeded()) {
3081     vm_exit_during_initialization("Failed to setup compressed class space virtual space list.");
3082   }
3083 }
3084 
3085 #endif
3086 
3087 void Metaspace::ergo_initialize() {
3088   if (DumpSharedSpaces) {
3089     // Using large pages when dumping the shared archive is currently not implemented.
3090     FLAG_SET_ERGO(bool, UseLargePagesInMetaspace, false);
3091   }
3092 
3093   size_t page_size = os::vm_page_size();
3094   if (UseLargePages && UseLargePagesInMetaspace) {
3095     page_size = os::large_page_size();
3096   }
3097 
3098   _commit_alignment  = page_size;
3099   _reserve_alignment = MAX2(page_size, (size_t)os::vm_allocation_granularity());
3100 
3101   // Do not use FLAG_SET_ERGO to update MaxMetaspaceSize, since this will
3102   // override if MaxMetaspaceSize was set on the command line or not.
3103   // This information is needed later to conform to the specification of the
3104   // java.lang.management.MemoryUsage API.
3105   //
3106   // Ideally, we would be able to set the default value of MaxMetaspaceSize in
3107   // globals.hpp to the aligned value, but this is not possible, since the
3108   // alignment depends on other flags being parsed.
3109   MaxMetaspaceSize = align_size_down_bounded(MaxMetaspaceSize, _reserve_alignment);
3110 
3111   if (MetaspaceSize > MaxMetaspaceSize) {
3112     MetaspaceSize = MaxMetaspaceSize;
3113   }
3114 
3115   MetaspaceSize = align_size_down_bounded(MetaspaceSize, _commit_alignment);
3116 
3117   assert(MetaspaceSize <= MaxMetaspaceSize, "MetaspaceSize should be limited by MaxMetaspaceSize");
3118 
3119   if (MetaspaceSize < 256*K) {
3120     vm_exit_during_initialization("Too small initial Metaspace size");
3121   }
3122 
3123   MinMetaspaceExpansion = align_size_down_bounded(MinMetaspaceExpansion, _commit_alignment);
3124   MaxMetaspaceExpansion = align_size_down_bounded(MaxMetaspaceExpansion, _commit_alignment);
3125 
3126   CompressedClassSpaceSize = align_size_down_bounded(CompressedClassSpaceSize, _reserve_alignment);
3127   set_compressed_class_space_size(CompressedClassSpaceSize);
3128 }
3129 
3130 void Metaspace::global_initialize() {
3131   MetaspaceGC::initialize();
3132 
3133   // Initialize the alignment for shared spaces.
3134   int max_alignment = os::vm_allocation_granularity();
3135   size_t cds_total = 0;
3136 
3137   MetaspaceShared::set_max_alignment(max_alignment);
3138 
3139   if (DumpSharedSpaces) {
3140     SharedReadOnlySize  = align_size_up(SharedReadOnlySize,  max_alignment);
3141     SharedReadWriteSize = align_size_up(SharedReadWriteSize, max_alignment);
3142     SharedMiscDataSize  = align_size_up(SharedMiscDataSize,  max_alignment);
3143     SharedMiscCodeSize  = align_size_up(SharedMiscCodeSize,  max_alignment);
3144 
3145     // Initialize with the sum of the shared space sizes.  The read-only
3146     // and read write metaspace chunks will be allocated out of this and the
3147     // remainder is the misc code and data chunks.
3148     cds_total = FileMapInfo::shared_spaces_size();
3149     cds_total = align_size_up(cds_total, _reserve_alignment);
3150     _space_list = new VirtualSpaceList(cds_total/wordSize);
3151     _chunk_manager_metadata = new ChunkManager(SpecializedChunk, SmallChunk, MediumChunk);
3152 
3153     if (!_space_list->initialization_succeeded()) {
3154       vm_exit_during_initialization("Unable to dump shared archive.", NULL);
3155     }
3156 
3157 #ifdef _LP64
3158     if (cds_total + compressed_class_space_size() > UnscaledClassSpaceMax) {
3159       vm_exit_during_initialization("Unable to dump shared archive.",
3160           err_msg("Size of archive (" SIZE_FORMAT ") + compressed class space ("
3161                   SIZE_FORMAT ") == total (" SIZE_FORMAT ") is larger than compressed "
3162                   "klass limit: " SIZE_FORMAT, cds_total, compressed_class_space_size(),
3163                   cds_total + compressed_class_space_size(), UnscaledClassSpaceMax));
3164     }
3165 
3166     // Set the compressed klass pointer base so that decoding of these pointers works
3167     // properly when creating the shared archive.
3168     assert(UseCompressedOops && UseCompressedClassPointers,
3169       "UseCompressedOops and UseCompressedClassPointers must be set");
3170     Universe::set_narrow_klass_base((address)_space_list->current_virtual_space()->bottom());
3171     if (TraceMetavirtualspaceAllocation && Verbose) {
3172       gclog_or_tty->print_cr("Setting_narrow_klass_base to Address: " PTR_FORMAT,
3173                              _space_list->current_virtual_space()->bottom());
3174     }
3175 
3176     Universe::set_narrow_klass_shift(0);
3177 #endif
3178 
3179   } else {
3180     // If using shared space, open the file that contains the shared space
3181     // and map in the memory before initializing the rest of metaspace (so
3182     // the addresses don't conflict)
3183     address cds_address = NULL;
3184     if (UseSharedSpaces) {
3185       FileMapInfo* mapinfo = new FileMapInfo();
3186       memset(mapinfo, 0, sizeof(FileMapInfo));
3187 
3188       // Open the shared archive file, read and validate the header. If
3189       // initialization fails, shared spaces [UseSharedSpaces] are
3190       // disabled and the file is closed.
3191       // Map in spaces now also
3192       if (mapinfo->initialize() && MetaspaceShared::map_shared_spaces(mapinfo)) {
3193         FileMapInfo::set_current_info(mapinfo);
3194         cds_total = FileMapInfo::shared_spaces_size();
3195         cds_address = (address)mapinfo->region_base(0);
3196       } else {
3197         assert(!mapinfo->is_open() && !UseSharedSpaces,
3198                "archive file not closed or shared spaces not disabled.");
3199       }
3200     }
3201 
3202 #ifdef _LP64
3203     // If UseCompressedClassPointers is set then allocate the metaspace area
3204     // above the heap and above the CDS area (if it exists).
3205     if (using_class_space()) {
3206       if (UseSharedSpaces) {
3207         char* cds_end = (char*)(cds_address + cds_total);
3208         cds_end = (char *)align_ptr_up(cds_end, _reserve_alignment);
3209         allocate_metaspace_compressed_klass_ptrs(cds_end, cds_address);
3210       } else {
3211         char* base = (char*)align_ptr_up(Universe::heap()->reserved_region().end(), _reserve_alignment);
3212         allocate_metaspace_compressed_klass_ptrs(base, 0);
3213       }
3214     }
3215 #endif
3216 
3217     // Initialize these before initializing the VirtualSpaceList
3218     _first_chunk_word_size = InitialBootClassLoaderMetaspaceSize / BytesPerWord;
3219     _first_chunk_word_size = align_word_size_up(_first_chunk_word_size);
3220     // Make the first class chunk bigger than a medium chunk so it's not put
3221     // on the medium chunk list.   The next chunk will be small and progress
3222     // from there.  This size calculated by -version.
3223     _first_class_chunk_word_size = MIN2((size_t)MediumChunk*6,
3224                                        (CompressedClassSpaceSize/BytesPerWord)*2);
3225     _first_class_chunk_word_size = align_word_size_up(_first_class_chunk_word_size);
3226     // Arbitrarily set the initial virtual space to a multiple
3227     // of the boot class loader size.
3228     size_t word_size = VIRTUALSPACEMULTIPLIER * _first_chunk_word_size;
3229     word_size = align_size_up(word_size, Metaspace::reserve_alignment_words());
3230 
3231     // Initialize the list of virtual spaces.
3232     _space_list = new VirtualSpaceList(word_size);
3233     _chunk_manager_metadata = new ChunkManager(SpecializedChunk, SmallChunk, MediumChunk);
3234 
3235     if (!_space_list->initialization_succeeded()) {
3236       vm_exit_during_initialization("Unable to setup metadata virtual space list.", NULL);
3237     }
3238   }
3239 
3240   _tracer = new MetaspaceTracer();
3241 }
3242 
3243 void Metaspace::post_initialize() {
3244   MetaspaceGC::post_initialize();
3245 }
3246 
3247 Metachunk* Metaspace::get_initialization_chunk(MetadataType mdtype,
3248                                                size_t chunk_word_size,
3249                                                size_t chunk_bunch) {
3250   // Get a chunk from the chunk freelist
3251   Metachunk* chunk = get_chunk_manager(mdtype)->chunk_freelist_allocate(chunk_word_size);
3252   if (chunk != NULL) {
3253     return chunk;
3254   }
3255 
3256   return get_space_list(mdtype)->get_new_chunk(chunk_word_size, chunk_word_size, chunk_bunch);
3257 }
3258 
3259 void Metaspace::initialize(Mutex* lock, MetaspaceType type) {
3260 
3261   assert(space_list() != NULL,
3262     "Metadata VirtualSpaceList has not been initialized");
3263   assert(chunk_manager_metadata() != NULL,
3264     "Metadata ChunkManager has not been initialized");
3265 
3266   _vsm = new SpaceManager(NonClassType, lock);
3267   if (_vsm == NULL) {
3268     return;
3269   }
3270   size_t word_size;
3271   size_t class_word_size;
3272   vsm()->get_initial_chunk_sizes(type, &word_size, &class_word_size);
3273 
3274   if (using_class_space()) {
3275   assert(class_space_list() != NULL,
3276     "Class VirtualSpaceList has not been initialized");
3277   assert(chunk_manager_class() != NULL,
3278     "Class ChunkManager has not been initialized");
3279 
3280     // Allocate SpaceManager for classes.
3281     _class_vsm = new SpaceManager(ClassType, lock);
3282     if (_class_vsm == NULL) {
3283       return;
3284     }
3285   }
3286 
3287   MutexLockerEx cl(SpaceManager::expand_lock(), Mutex::_no_safepoint_check_flag);
3288 
3289   // Allocate chunk for metadata objects
3290   Metachunk* new_chunk = get_initialization_chunk(NonClassType,
3291                                                   word_size,
3292                                                   vsm()->medium_chunk_bunch());
3293   assert(!DumpSharedSpaces || new_chunk != NULL, "should have enough space for both chunks");
3294   if (new_chunk != NULL) {
3295     // Add to this manager's list of chunks in use and current_chunk().
3296     vsm()->add_chunk(new_chunk, true);
3297   }
3298 
3299   // Allocate chunk for class metadata objects
3300   if (using_class_space()) {
3301     Metachunk* class_chunk = get_initialization_chunk(ClassType,
3302                                                       class_word_size,
3303                                                       class_vsm()->medium_chunk_bunch());
3304     if (class_chunk != NULL) {
3305       class_vsm()->add_chunk(class_chunk, true);
3306     }
3307   }
3308 
3309   _alloc_record_head = NULL;
3310   _alloc_record_tail = NULL;
3311 }
3312 
3313 size_t Metaspace::align_word_size_up(size_t word_size) {
3314   size_t byte_size = word_size * wordSize;
3315   return ReservedSpace::allocation_align_size_up(byte_size) / wordSize;
3316 }
3317 
3318 MetaWord* Metaspace::allocate(size_t word_size, MetadataType mdtype) {
3319   // DumpSharedSpaces doesn't use class metadata area (yet)
3320   // Also, don't use class_vsm() unless UseCompressedClassPointers is true.
3321   if (is_class_space_allocation(mdtype)) {
3322     return  class_vsm()->allocate(word_size);
3323   } else {
3324     return  vsm()->allocate(word_size);
3325   }
3326 }
3327 
3328 MetaWord* Metaspace::expand_and_allocate(size_t word_size, MetadataType mdtype) {
3329   size_t delta_bytes = MetaspaceGC::delta_capacity_until_GC(word_size * BytesPerWord);
3330   assert(delta_bytes > 0, "Must be");
3331 
3332   size_t after_inc = MetaspaceGC::inc_capacity_until_GC(delta_bytes);
3333 
3334   // capacity_until_GC might be updated concurrently, must calculate previous value.
3335   size_t before_inc = after_inc - delta_bytes;
3336 
3337   tracer()->report_gc_threshold(before_inc, after_inc,
3338                                 MetaspaceGCThresholdUpdater::ExpandAndAllocate);
3339   if (PrintGCDetails && Verbose) {
3340     gclog_or_tty->print_cr("Increase capacity to GC from " SIZE_FORMAT
3341         " to " SIZE_FORMAT, before_inc, after_inc);
3342   }
3343 
3344   return allocate(word_size, mdtype);
3345 }
3346 
3347 // Space allocated in the Metaspace.  This may
3348 // be across several metadata virtual spaces.
3349 char* Metaspace::bottom() const {
3350   assert(DumpSharedSpaces, "only useful and valid for dumping shared spaces");
3351   return (char*)vsm()->current_chunk()->bottom();
3352 }
3353 
3354 size_t Metaspace::used_words_slow(MetadataType mdtype) const {
3355   if (mdtype == ClassType) {
3356     return using_class_space() ? class_vsm()->sum_used_in_chunks_in_use() : 0;
3357   } else {
3358     return vsm()->sum_used_in_chunks_in_use();  // includes overhead!
3359   }
3360 }
3361 
3362 size_t Metaspace::free_words_slow(MetadataType mdtype) const {
3363   if (mdtype == ClassType) {
3364     return using_class_space() ? class_vsm()->sum_free_in_chunks_in_use() : 0;
3365   } else {
3366     return vsm()->sum_free_in_chunks_in_use();
3367   }
3368 }
3369 
3370 // Space capacity in the Metaspace.  It includes
3371 // space in the list of chunks from which allocations
3372 // have been made. Don't include space in the global freelist and
3373 // in the space available in the dictionary which
3374 // is already counted in some chunk.
3375 size_t Metaspace::capacity_words_slow(MetadataType mdtype) const {
3376   if (mdtype == ClassType) {
3377     return using_class_space() ? class_vsm()->sum_capacity_in_chunks_in_use() : 0;
3378   } else {
3379     return vsm()->sum_capacity_in_chunks_in_use();
3380   }
3381 }
3382 
3383 size_t Metaspace::used_bytes_slow(MetadataType mdtype) const {
3384   return used_words_slow(mdtype) * BytesPerWord;
3385 }
3386 
3387 size_t Metaspace::capacity_bytes_slow(MetadataType mdtype) const {
3388   return capacity_words_slow(mdtype) * BytesPerWord;
3389 }
3390 
3391 size_t Metaspace::allocated_blocks_bytes() const {
3392   return vsm()->allocated_blocks_bytes() +
3393       (using_class_space() ? class_vsm()->allocated_blocks_bytes() : 0);
3394 }
3395 
3396 size_t Metaspace::allocated_chunks_bytes() const {
3397   return vsm()->allocated_chunks_bytes() +
3398       (using_class_space() ? class_vsm()->allocated_chunks_bytes() : 0);
3399 }
3400 
3401 void Metaspace::deallocate(MetaWord* ptr, size_t word_size, bool is_class) {
3402   assert(!SafepointSynchronize::is_at_safepoint()
3403          || Thread::current()->is_VM_thread(), "should be the VM thread");
3404 
3405   MutexLockerEx ml(vsm()->lock(), Mutex::_no_safepoint_check_flag);
3406 
3407   if (word_size < TreeChunk<Metablock, FreeList<Metablock> >::min_size()) {
3408     // Dark matter.  Too small for dictionary.
3409 #ifdef ASSERT
3410     Copy::fill_to_words((HeapWord*)ptr, word_size, 0xf5f5f5f5);
3411 #endif
3412     return;
3413   }
3414   if (is_class && using_class_space()) {
3415     class_vsm()->deallocate(ptr, word_size);
3416   } else {
3417     vsm()->deallocate(ptr, word_size);
3418   }
3419 }
3420 
3421 
3422 MetaWord* Metaspace::allocate(ClassLoaderData* loader_data, size_t word_size,
3423                               bool read_only, MetaspaceObj::Type type, TRAPS) {
3424   if (HAS_PENDING_EXCEPTION) {
3425     assert(false, "Should not allocate with exception pending");
3426     return NULL;  // caller does a CHECK_NULL too
3427   }
3428 
3429   assert(loader_data != NULL, "Should never pass around a NULL loader_data. "
3430         "ClassLoaderData::the_null_class_loader_data() should have been used.");
3431 
3432   // Allocate in metaspaces without taking out a lock, because it deadlocks
3433   // with the SymbolTable_lock.  Dumping is single threaded for now.  We'll have
3434   // to revisit this for application class data sharing.
3435   if (DumpSharedSpaces) {
3436     assert(type > MetaspaceObj::UnknownType && type < MetaspaceObj::_number_of_types, "sanity");
3437     Metaspace* space = read_only ? loader_data->ro_metaspace() : loader_data->rw_metaspace();
3438     MetaWord* result = space->allocate(word_size, NonClassType);
3439     if (result == NULL) {
3440       report_out_of_shared_space(read_only ? SharedReadOnly : SharedReadWrite);
3441     }
3442 
3443     space->record_allocation(result, type, space->vsm()->get_raw_word_size(word_size));
3444 
3445     // Zero initialize.
3446     Copy::fill_to_aligned_words((HeapWord*)result, word_size, 0);
3447 
3448     return result;
3449   }
3450 
3451   MetadataType mdtype = (type == MetaspaceObj::ClassType) ? ClassType : NonClassType;
3452 
3453   // Try to allocate metadata.
3454   MetaWord* result = loader_data->metaspace_non_null()->allocate(word_size, mdtype);
3455 
3456   if (result == NULL) {
3457     tracer()->report_metaspace_allocation_failure(loader_data, word_size, type, mdtype);
3458 
3459     // Allocation failed.
3460     if (is_init_completed()) {
3461       // Only start a GC if the bootstrapping has completed.
3462 
3463       // Try to clean out some memory and retry.
3464       result = Universe::heap()->collector_policy()->satisfy_failed_metadata_allocation(
3465           loader_data, word_size, mdtype);
3466     }
3467   }
3468 
3469   if (result == NULL) {
3470     report_metadata_oome(loader_data, word_size, type, mdtype, CHECK_NULL);
3471   }
3472 
3473   // Zero initialize.
3474   Copy::fill_to_aligned_words((HeapWord*)result, word_size, 0);
3475 
3476   return result;
3477 }
3478 
3479 size_t Metaspace::class_chunk_size(size_t word_size) {
3480   assert(using_class_space(), "Has to use class space");
3481   return class_vsm()->calc_chunk_size(word_size);
3482 }
3483 
3484 void Metaspace::report_metadata_oome(ClassLoaderData* loader_data, size_t word_size, MetaspaceObj::Type type, MetadataType mdtype, TRAPS) {
3485   tracer()->report_metadata_oom(loader_data, word_size, type, mdtype);
3486 
3487   // If result is still null, we are out of memory.
3488   if (Verbose && TraceMetadataChunkAllocation) {
3489     gclog_or_tty->print_cr("Metaspace allocation failed for size "
3490         SIZE_FORMAT, word_size);
3491     if (loader_data->metaspace_or_null() != NULL) {
3492       loader_data->dump(gclog_or_tty);
3493     }
3494     MetaspaceAux::dump(gclog_or_tty);
3495   }
3496 
3497   bool out_of_compressed_class_space = false;
3498   if (is_class_space_allocation(mdtype)) {
3499     Metaspace* metaspace = loader_data->metaspace_non_null();
3500     out_of_compressed_class_space =
3501       MetaspaceAux::committed_bytes(Metaspace::ClassType) +
3502       (metaspace->class_chunk_size(word_size) * BytesPerWord) >
3503       CompressedClassSpaceSize;
3504   }
3505 
3506   // -XX:+HeapDumpOnOutOfMemoryError and -XX:OnOutOfMemoryError support
3507   const char* space_string = out_of_compressed_class_space ?
3508     "Compressed class space" : "Metaspace";
3509 
3510   report_java_out_of_memory(space_string);
3511 
3512   if (JvmtiExport::should_post_resource_exhausted()) {
3513     JvmtiExport::post_resource_exhausted(
3514         JVMTI_RESOURCE_EXHAUSTED_OOM_ERROR,
3515         space_string);
3516   }
3517 
3518   if (!is_init_completed()) {
3519     vm_exit_during_initialization("OutOfMemoryError", space_string);
3520   }
3521 
3522   if (out_of_compressed_class_space) {
3523     THROW_OOP(Universe::out_of_memory_error_class_metaspace());
3524   } else {
3525     THROW_OOP(Universe::out_of_memory_error_metaspace());
3526   }
3527 }
3528 
3529 const char* Metaspace::metadata_type_name(Metaspace::MetadataType mdtype) {
3530   switch (mdtype) {
3531     case Metaspace::ClassType: return "Class";
3532     case Metaspace::NonClassType: return "Metadata";
3533     default:
3534       assert(false, err_msg("Got bad mdtype: %d", (int) mdtype));
3535       return NULL;
3536   }
3537 }
3538 
3539 void Metaspace::record_allocation(void* ptr, MetaspaceObj::Type type, size_t word_size) {
3540   assert(DumpSharedSpaces, "sanity");
3541 
3542   AllocRecord *rec = new AllocRecord((address)ptr, type, (int)word_size * HeapWordSize);
3543   if (_alloc_record_head == NULL) {
3544     _alloc_record_head = _alloc_record_tail = rec;
3545   } else {
3546     _alloc_record_tail->_next = rec;
3547     _alloc_record_tail = rec;
3548   }
3549 }
3550 
3551 void Metaspace::iterate(Metaspace::AllocRecordClosure *closure) {
3552   assert(DumpSharedSpaces, "unimplemented for !DumpSharedSpaces");
3553 
3554   address last_addr = (address)bottom();
3555 
3556   for (AllocRecord *rec = _alloc_record_head; rec; rec = rec->_next) {
3557     address ptr = rec->_ptr;
3558     if (last_addr < ptr) {
3559       closure->doit(last_addr, MetaspaceObj::UnknownType, ptr - last_addr);
3560     }
3561     closure->doit(ptr, rec->_type, rec->_byte_size);
3562     last_addr = ptr + rec->_byte_size;
3563   }
3564 
3565   address top = ((address)bottom()) + used_bytes_slow(Metaspace::NonClassType);
3566   if (last_addr < top) {
3567     closure->doit(last_addr, MetaspaceObj::UnknownType, top - last_addr);
3568   }
3569 }
3570 
3571 void Metaspace::purge(MetadataType mdtype) {
3572   get_space_list(mdtype)->purge(get_chunk_manager(mdtype));
3573 }
3574 
3575 void Metaspace::purge() {
3576   MutexLockerEx cl(SpaceManager::expand_lock(),
3577                    Mutex::_no_safepoint_check_flag);
3578   purge(NonClassType);
3579   if (using_class_space()) {
3580     purge(ClassType);
3581   }
3582 }
3583 
3584 void Metaspace::print_on(outputStream* out) const {
3585   // Print both class virtual space counts and metaspace.
3586   if (Verbose) {
3587     vsm()->print_on(out);
3588     if (using_class_space()) {
3589       class_vsm()->print_on(out);
3590     }
3591   }
3592 }
3593 
3594 bool Metaspace::contains(const void* ptr) {
3595   if (UseSharedSpaces && MetaspaceShared::is_in_shared_space(ptr)) {
3596     return true;
3597   }
3598 
3599   if (using_class_space() && get_space_list(ClassType)->contains(ptr)) {
3600      return true;
3601   }
3602 
3603   return get_space_list(NonClassType)->contains(ptr);
3604 }
3605 
3606 void Metaspace::verify() {
3607   vsm()->verify();
3608   if (using_class_space()) {
3609     class_vsm()->verify();
3610   }
3611 }
3612 
3613 void Metaspace::dump(outputStream* const out) const {
3614   out->print_cr("\nVirtual space manager: " INTPTR_FORMAT, vsm());
3615   vsm()->dump(out);
3616   if (using_class_space()) {
3617     out->print_cr("\nClass space manager: " INTPTR_FORMAT, class_vsm());
3618     class_vsm()->dump(out);
3619   }
3620 }
3621 
3622 /////////////// Unit tests ///////////////
3623 
3624 #ifndef PRODUCT
3625 
3626 class TestMetaspaceAuxTest : AllStatic {
3627  public:
3628   static void test_reserved() {
3629     size_t reserved = MetaspaceAux::reserved_bytes();
3630 
3631     assert(reserved > 0, "assert");
3632 
3633     size_t committed  = MetaspaceAux::committed_bytes();
3634     assert(committed <= reserved, "assert");
3635 
3636     size_t reserved_metadata = MetaspaceAux::reserved_bytes(Metaspace::NonClassType);
3637     assert(reserved_metadata > 0, "assert");
3638     assert(reserved_metadata <= reserved, "assert");
3639 
3640     if (UseCompressedClassPointers) {
3641       size_t reserved_class    = MetaspaceAux::reserved_bytes(Metaspace::ClassType);
3642       assert(reserved_class > 0, "assert");
3643       assert(reserved_class < reserved, "assert");
3644     }
3645   }
3646 
3647   static void test_committed() {
3648     size_t committed = MetaspaceAux::committed_bytes();
3649 
3650     assert(committed > 0, "assert");
3651 
3652     size_t reserved  = MetaspaceAux::reserved_bytes();
3653     assert(committed <= reserved, "assert");
3654 
3655     size_t committed_metadata = MetaspaceAux::committed_bytes(Metaspace::NonClassType);
3656     assert(committed_metadata > 0, "assert");
3657     assert(committed_metadata <= committed, "assert");
3658 
3659     if (UseCompressedClassPointers) {
3660       size_t committed_class    = MetaspaceAux::committed_bytes(Metaspace::ClassType);
3661       assert(committed_class > 0, "assert");
3662       assert(committed_class < committed, "assert");
3663     }
3664   }
3665 
3666   static void test_virtual_space_list_large_chunk() {
3667     VirtualSpaceList* vs_list = new VirtualSpaceList(os::vm_allocation_granularity());
3668     MutexLockerEx cl(SpaceManager::expand_lock(), Mutex::_no_safepoint_check_flag);
3669     // A size larger than VirtualSpaceSize (256k) and add one page to make it _not_ be
3670     // vm_allocation_granularity aligned on Windows.
3671     size_t large_size = (size_t)(2*256*K + (os::vm_page_size()/BytesPerWord));
3672     large_size += (os::vm_page_size()/BytesPerWord);
3673     vs_list->get_new_chunk(large_size, large_size, 0);
3674   }
3675 
3676   static void test() {
3677     test_reserved();
3678     test_committed();
3679     test_virtual_space_list_large_chunk();
3680   }
3681 };
3682 
3683 void TestMetaspaceAux_test() {
3684   TestMetaspaceAuxTest::test();
3685 }
3686 
3687 class TestVirtualSpaceNodeTest {
3688   static void chunk_up(size_t words_left, size_t& num_medium_chunks,
3689                                           size_t& num_small_chunks,
3690                                           size_t& num_specialized_chunks) {
3691     num_medium_chunks = words_left / MediumChunk;
3692     words_left = words_left % MediumChunk;
3693 
3694     num_small_chunks = words_left / SmallChunk;
3695     words_left = words_left % SmallChunk;
3696     // how many specialized chunks can we get?
3697     num_specialized_chunks = words_left / SpecializedChunk;
3698     assert(words_left % SpecializedChunk == 0, "should be nothing left");
3699   }
3700 
3701  public:
3702   static void test() {
3703     MutexLockerEx ml(SpaceManager::expand_lock(), Mutex::_no_safepoint_check_flag);
3704     const size_t vsn_test_size_words = MediumChunk  * 4;
3705     const size_t vsn_test_size_bytes = vsn_test_size_words * BytesPerWord;
3706 
3707     // The chunk sizes must be multiples of eachother, or this will fail
3708     STATIC_ASSERT(MediumChunk % SmallChunk == 0);
3709     STATIC_ASSERT(SmallChunk % SpecializedChunk == 0);
3710 
3711     { // No committed memory in VSN
3712       ChunkManager cm(SpecializedChunk, SmallChunk, MediumChunk);
3713       VirtualSpaceNode vsn(vsn_test_size_bytes);
3714       vsn.initialize();
3715       vsn.retire(&cm);
3716       assert(cm.sum_free_chunks_count() == 0, "did not commit any memory in the VSN");
3717     }
3718 
3719     { // All of VSN is committed, half is used by chunks
3720       ChunkManager cm(SpecializedChunk, SmallChunk, MediumChunk);
3721       VirtualSpaceNode vsn(vsn_test_size_bytes);
3722       vsn.initialize();
3723       vsn.expand_by(vsn_test_size_words, vsn_test_size_words);
3724       vsn.get_chunk_vs(MediumChunk);
3725       vsn.get_chunk_vs(MediumChunk);
3726       vsn.retire(&cm);
3727       assert(cm.sum_free_chunks_count() == 2, "should have been memory left for 2 medium chunks");
3728       assert(cm.sum_free_chunks() == 2*MediumChunk, "sizes should add up");
3729     }
3730 
3731     { // 4 pages of VSN is committed, some is used by chunks
3732       ChunkManager cm(SpecializedChunk, SmallChunk, MediumChunk);
3733       VirtualSpaceNode vsn(vsn_test_size_bytes);
3734       const size_t page_chunks = 4 * (size_t)os::vm_page_size() / BytesPerWord;
3735       assert(page_chunks < MediumChunk, "Test expects medium chunks to be at least 4*page_size");
3736       vsn.initialize();
3737       vsn.expand_by(page_chunks, page_chunks);
3738       vsn.get_chunk_vs(SmallChunk);
3739       vsn.get_chunk_vs(SpecializedChunk);
3740       vsn.retire(&cm);
3741 
3742       // committed - used = words left to retire
3743       const size_t words_left = page_chunks - SmallChunk - SpecializedChunk;
3744 
3745       size_t num_medium_chunks, num_small_chunks, num_spec_chunks;
3746       chunk_up(words_left, num_medium_chunks, num_small_chunks, num_spec_chunks);
3747 
3748       assert(num_medium_chunks == 0, "should not get any medium chunks");
3749       assert(cm.sum_free_chunks_count() == (num_small_chunks + num_spec_chunks), "should be space for 3 chunks");
3750       assert(cm.sum_free_chunks() == words_left, "sizes should add up");
3751     }
3752 
3753     { // Half of VSN is committed, a humongous chunk is used
3754       ChunkManager cm(SpecializedChunk, SmallChunk, MediumChunk);
3755       VirtualSpaceNode vsn(vsn_test_size_bytes);
3756       vsn.initialize();
3757       vsn.expand_by(MediumChunk * 2, MediumChunk * 2);
3758       vsn.get_chunk_vs(MediumChunk + SpecializedChunk); // Humongous chunks will be aligned up to MediumChunk + SpecializedChunk
3759       vsn.retire(&cm);
3760 
3761       const size_t words_left = MediumChunk * 2 - (MediumChunk + SpecializedChunk);
3762       size_t num_medium_chunks, num_small_chunks, num_spec_chunks;
3763       chunk_up(words_left, num_medium_chunks, num_small_chunks, num_spec_chunks);
3764 
3765       assert(num_medium_chunks == 0, "should not get any medium chunks");
3766       assert(cm.sum_free_chunks_count() == (num_small_chunks + num_spec_chunks), "should be space for 3 chunks");
3767       assert(cm.sum_free_chunks() == words_left, "sizes should add up");
3768     }
3769 
3770   }
3771 
3772 #define assert_is_available_positive(word_size) \
3773   assert(vsn.is_available(word_size), \
3774     err_msg(#word_size ": " PTR_FORMAT " bytes were not available in " \
3775             "VirtualSpaceNode [" PTR_FORMAT ", " PTR_FORMAT ")", \
3776             (uintptr_t)(word_size * BytesPerWord), vsn.bottom(), vsn.end()));
3777 
3778 #define assert_is_available_negative(word_size) \
3779   assert(!vsn.is_available(word_size), \
3780     err_msg(#word_size ": " PTR_FORMAT " bytes should not be available in " \
3781             "VirtualSpaceNode [" PTR_FORMAT ", " PTR_FORMAT ")", \
3782             (uintptr_t)(word_size * BytesPerWord), vsn.bottom(), vsn.end()));
3783 
3784   static void test_is_available_positive() {
3785     // Reserve some memory.
3786     VirtualSpaceNode vsn(os::vm_allocation_granularity());
3787     assert(vsn.initialize(), "Failed to setup VirtualSpaceNode");
3788 
3789     // Commit some memory.
3790     size_t commit_word_size = os::vm_allocation_granularity() / BytesPerWord;
3791     bool expanded = vsn.expand_by(commit_word_size, commit_word_size);
3792     assert(expanded, "Failed to commit");
3793 
3794     // Check that is_available accepts the committed size.
3795     assert_is_available_positive(commit_word_size);
3796 
3797     // Check that is_available accepts half the committed size.
3798     size_t expand_word_size = commit_word_size / 2;
3799     assert_is_available_positive(expand_word_size);
3800   }
3801 
3802   static void test_is_available_negative() {
3803     // Reserve some memory.
3804     VirtualSpaceNode vsn(os::vm_allocation_granularity());
3805     assert(vsn.initialize(), "Failed to setup VirtualSpaceNode");
3806 
3807     // Commit some memory.
3808     size_t commit_word_size = os::vm_allocation_granularity() / BytesPerWord;
3809     bool expanded = vsn.expand_by(commit_word_size, commit_word_size);
3810     assert(expanded, "Failed to commit");
3811 
3812     // Check that is_available doesn't accept a too large size.
3813     size_t two_times_commit_word_size = commit_word_size * 2;
3814     assert_is_available_negative(two_times_commit_word_size);
3815   }
3816 
3817   static void test_is_available_overflow() {
3818     // Reserve some memory.
3819     VirtualSpaceNode vsn(os::vm_allocation_granularity());
3820     assert(vsn.initialize(), "Failed to setup VirtualSpaceNode");
3821 
3822     // Commit some memory.
3823     size_t commit_word_size = os::vm_allocation_granularity() / BytesPerWord;
3824     bool expanded = vsn.expand_by(commit_word_size, commit_word_size);
3825     assert(expanded, "Failed to commit");
3826 
3827     // Calculate a size that will overflow the virtual space size.
3828     void* virtual_space_max = (void*)(uintptr_t)-1;
3829     size_t bottom_to_max = pointer_delta(virtual_space_max, vsn.bottom(), 1);
3830     size_t overflow_size = bottom_to_max + BytesPerWord;
3831     size_t overflow_word_size = overflow_size / BytesPerWord;
3832 
3833     // Check that is_available can handle the overflow.
3834     assert_is_available_negative(overflow_word_size);
3835   }
3836 
3837   static void test_is_available() {
3838     TestVirtualSpaceNodeTest::test_is_available_positive();
3839     TestVirtualSpaceNodeTest::test_is_available_negative();
3840     TestVirtualSpaceNodeTest::test_is_available_overflow();
3841   }
3842 };
3843 
3844 void TestVirtualSpaceNode_test() {
3845   TestVirtualSpaceNodeTest::test();
3846   TestVirtualSpaceNodeTest::test_is_available();
3847 }
3848 #endif