# HG changeset patch # User stuefe # Date 1526019114 -7200 # Fri May 11 08:11:54 2018 +0200 # Node ID 63bfcf877bbc6c57745701a013adc23788e65745 # Parent 5ec7380f671d52f9763cddd1ffee56ae58a981ef imported patch metaspace-split diff --git a/src/hotspot/share/gc/cms/vmStructs_cms.hpp b/src/hotspot/share/gc/cms/vmStructs_cms.hpp --- a/src/hotspot/share/gc/cms/vmStructs_cms.hpp +++ b/src/hotspot/share/gc/cms/vmStructs_cms.hpp @@ -70,7 +70,7 @@ declare_toplevel_type(CMSCollector) \ declare_toplevel_type(CMSBitMap) \ declare_toplevel_type(FreeChunk) \ - declare_toplevel_type(Metablock) \ + declare_toplevel_type(metaspace::Metablock) \ declare_toplevel_type(ConcurrentMarkSweepThread*) \ declare_toplevel_type(ConcurrentMarkSweepGeneration*) \ declare_toplevel_type(CompactibleFreeListSpace*) \ diff --git a/src/hotspot/share/memory/binaryTreeDictionary.hpp b/src/hotspot/share/memory/binaryTreeDictionary.hpp --- a/src/hotspot/share/memory/binaryTreeDictionary.hpp +++ b/src/hotspot/share/memory/binaryTreeDictionary.hpp @@ -26,6 +26,9 @@ #define SHARE_VM_MEMORY_BINARYTREEDICTIONARY_HPP #include "memory/freeList.hpp" +#include "memory/memRegion.hpp" + +class Mutex; /* * A binary tree based search structure for free blocks. @@ -276,15 +279,7 @@ } size_t max_chunk_size() const; - size_t total_chunk_size(debug_only(const Mutex* lock)) const { - debug_only( - if (lock != NULL && lock->owned_by_self()) { - assert(total_size_in_tree(root()) == total_size(), - "_total_size inconsistency"); - } - ) - return total_size(); - } + inline size_t total_chunk_size(debug_only(const Mutex* lock)) const; size_t min_size() const { return TreeChunk::min_size(); diff --git a/src/hotspot/share/memory/binaryTreeDictionary.inline.hpp b/src/hotspot/share/memory/binaryTreeDictionary.inline.hpp --- a/src/hotspot/share/memory/binaryTreeDictionary.inline.hpp +++ b/src/hotspot/share/memory/binaryTreeDictionary.inline.hpp @@ -30,8 +30,8 @@ #include "logging/logStream.hpp" #include "memory/binaryTreeDictionary.hpp" #include "memory/freeList.inline.hpp" -#include "memory/metachunk.hpp" #include "memory/resourceArea.hpp" +#include "runtime/mutex.hpp" #include "runtime/globals.hpp" #include "utilities/macros.hpp" #include "utilities/ostream.hpp" @@ -1022,4 +1022,15 @@ guarantee(total_size() == total_size_in_tree(root()), "Total Size inconsistency"); } +template +size_t BinaryTreeDictionary::total_chunk_size(debug_only(const Mutex* lock)) const { + debug_only( + if (lock != NULL && lock->owned_by_self()) { + assert(total_size_in_tree(root()) == total_size(), + "_total_size inconsistency"); + } + ) + return total_size(); +} + #endif // SHARE_VM_MEMORY_BINARYTREEDICTIONARY_INLINE_HPP diff --git a/src/hotspot/share/memory/freeList.inline.hpp b/src/hotspot/share/memory/freeList.inline.hpp --- a/src/hotspot/share/memory/freeList.inline.hpp +++ b/src/hotspot/share/memory/freeList.inline.hpp @@ -27,7 +27,6 @@ #include "gc/shared/collectedHeap.hpp" #include "memory/freeList.hpp" -#include "memory/metachunk.hpp" #include "runtime/globals.hpp" #include "runtime/mutex.hpp" #include "runtime/vmThread.hpp" diff --git a/src/hotspot/share/memory/metaspace.cpp b/src/hotspot/share/memory/metaspace.cpp --- a/src/hotspot/share/memory/metaspace.cpp +++ b/src/hotspot/share/memory/metaspace.cpp @@ -21,53 +21,31 @@ * questions. * */ + #include "precompiled.hpp" + #include "aot/aotLoader.hpp" -#include "gc/shared/collectedHeap.hpp" -#include "gc/shared/collectorPolicy.hpp" #include "logging/log.hpp" #include "logging/logStream.hpp" -#include "memory/allocation.hpp" -#include "memory/binaryTreeDictionary.inline.hpp" #include "memory/filemap.hpp" -#include "memory/freeList.inline.hpp" -#include "memory/metachunk.hpp" #include "memory/metaspace.hpp" +#include "memory/metaspace/chunkManager.hpp" +#include "memory/metaspace/metachunk.hpp" #include "memory/metaspace/metaspaceCommon.hpp" -#include "memory/metaspace/metaspaceStatistics.hpp" -#include "memory/metaspaceGCThresholdUpdater.hpp" +#include "memory/metaspace/printCLDMetaspaceInfoClosure.hpp" +#include "memory/metaspace/spaceManager.hpp" +#include "memory/metaspace/virtualSpaceList.hpp" #include "memory/metaspaceShared.hpp" #include "memory/metaspaceTracer.hpp" -#include "memory/resourceArea.hpp" -#include "memory/universe.hpp" -#include "runtime/atomic.hpp" -#include "runtime/globals.hpp" #include "runtime/init.hpp" -#include "runtime/java.hpp" -#include "runtime/mutex.hpp" -#include "runtime/mutexLocker.hpp" #include "runtime/orderAccess.inline.hpp" #include "services/memTracker.hpp" -#include "services/memoryService.hpp" -#include "utilities/align.hpp" #include "utilities/copy.hpp" #include "utilities/debug.hpp" #include "utilities/globalDefinitions.hpp" -#include "utilities/macros.hpp" -using namespace metaspace::internals; -typedef BinaryTreeDictionary > BlockTreeDictionary; -typedef BinaryTreeDictionary > ChunkTreeDictionary; - -// Helper function that does a bunch of checks for a chunk. -DEBUG_ONLY(static void do_verify_chunk(Metachunk* chunk);) - -// Given a Metachunk, update its in-use information (both in the -// chunk and the occupancy map). -static void do_update_in_use_info_for_chunk(Metachunk* chunk, bool inuse); - -size_t const allocation_from_dictionary_limit = 4 * K; +using namespace metaspace; MetaWord* last_allocated = 0; @@ -76,103 +54,6 @@ DEBUG_ONLY(bool Metaspace::_frozen = false;) -// Internal statistics. -#ifdef ASSERT -static struct { - // Number of allocations. - uintx num_allocs; - // Number of times a ClassLoaderMetaspace was born... - uintx num_metaspace_births; - // ... and died. - uintx num_metaspace_deaths; - // Number of times VirtualSpaceListNodes were created... - uintx num_vsnodes_created; - // ... and purged. - uintx num_vsnodes_purged; - // Number of times we expanded the committed section of the space. - uintx num_committed_space_expanded; - // Number of deallocations - uintx num_deallocs; - // Number of deallocations triggered from outside ("real" deallocations). - uintx num_external_deallocs; - // Number of times an allocation was satisfied from deallocated blocks. - uintx num_allocs_from_deallocated_blocks; -} g_internal_statistics; -#endif - -enum ChunkSizes { // in words. - ClassSpecializedChunk = 128, - SpecializedChunk = 128, - ClassSmallChunk = 256, - SmallChunk = 512, - ClassMediumChunk = 4 * K, - MediumChunk = 8 * K -}; - -// Returns size of this chunk type. -size_t get_size_for_nonhumongous_chunktype(ChunkIndex chunktype, bool is_class) { - assert(is_valid_nonhumongous_chunktype(chunktype), "invalid chunk type."); - size_t size = 0; - if (is_class) { - switch(chunktype) { - case SpecializedIndex: size = ClassSpecializedChunk; break; - case SmallIndex: size = ClassSmallChunk; break; - case MediumIndex: size = ClassMediumChunk; break; - default: - ShouldNotReachHere(); - } - } else { - switch(chunktype) { - case SpecializedIndex: size = SpecializedChunk; break; - case SmallIndex: size = SmallChunk; break; - case MediumIndex: size = MediumChunk; break; - default: - ShouldNotReachHere(); - } - } - return size; -} - -ChunkIndex get_chunk_type_by_size(size_t size, bool is_class) { - if (is_class) { - if (size == ClassSpecializedChunk) { - return SpecializedIndex; - } else if (size == ClassSmallChunk) { - return SmallIndex; - } else if (size == ClassMediumChunk) { - return MediumIndex; - } else if (size > ClassMediumChunk) { - // A valid humongous chunk size is a multiple of the smallest chunk size. - assert(is_aligned(size, ClassSpecializedChunk), "Invalid chunk size"); - return HumongousIndex; - } - } else { - if (size == SpecializedChunk) { - return SpecializedIndex; - } else if (size == SmallChunk) { - return SmallIndex; - } else if (size == MediumChunk) { - return MediumIndex; - } else if (size > MediumChunk) { - // A valid humongous chunk size is a multiple of the smallest chunk size. - assert(is_aligned(size, SpecializedChunk), "Invalid chunk size"); - return HumongousIndex; - } - } - ShouldNotReachHere(); - return (ChunkIndex)-1; -} - -ChunkIndex next_chunk_index(ChunkIndex i) { - assert(i < NumberOfInUseLists, "Out of bound"); - return (ChunkIndex) (i+1); -} - -ChunkIndex prev_chunk_index(ChunkIndex i) { - assert(i > ZeroIndex, "Out of bound"); - return (ChunkIndex) (i-1); -} - static const char* space_type_name(Metaspace::MetaspaceType t) { const char* s = NULL; switch (t) { @@ -189,2176 +70,10 @@ uint MetaspaceGC::_shrink_factor = 0; bool MetaspaceGC::_should_concurrent_collect = false; - -typedef class FreeList ChunkList; - -// Manages the global free lists of chunks. -class ChunkManager : public CHeapObj { - friend class TestVirtualSpaceNodeTest; - - // Free list of chunks of different sizes. - // SpecializedChunk - // SmallChunk - // MediumChunk - ChunkList _free_chunks[NumberOfFreeLists]; - - // Whether or not this is the class chunkmanager. - const bool _is_class; - - // Return non-humongous chunk list by its index. - ChunkList* free_chunks(ChunkIndex index); - - // Returns non-humongous chunk list for the given chunk word size. - ChunkList* find_free_chunks_list(size_t word_size); - - // HumongousChunk - ChunkTreeDictionary _humongous_dictionary; - - // Returns the humongous chunk dictionary. - ChunkTreeDictionary* humongous_dictionary() { - return &_humongous_dictionary; - } - - // Size, in metaspace words, of all chunks managed by this ChunkManager - size_t _free_chunks_total; - // Number of chunks in this ChunkManager - size_t _free_chunks_count; - - // Update counters after a chunk was added or removed removed. - void account_for_added_chunk(const Metachunk* c); - void account_for_removed_chunk(const Metachunk* c); - - // Debug support - - size_t sum_free_chunks(); - size_t sum_free_chunks_count(); - - void locked_verify_free_chunks_total(); - void slow_locked_verify_free_chunks_total() { - if (VerifyMetaspace) { - locked_verify_free_chunks_total(); - } - } - void locked_verify_free_chunks_count(); - void slow_locked_verify_free_chunks_count() { - if (VerifyMetaspace) { - locked_verify_free_chunks_count(); - } - } - - // Given a pointer to a chunk, attempts to merge it with neighboring - // free chunks to form a bigger chunk. Returns true if successful. - bool attempt_to_coalesce_around_chunk(Metachunk* chunk, ChunkIndex target_chunk_type); - - // Helper for chunk merging: - // Given an address range with 1-n chunks which are all supposed to be - // free and hence currently managed by this ChunkManager, remove them - // from this ChunkManager and mark them as invalid. - // - This does not correct the occupancy map. - // - This does not adjust the counters in ChunkManager. - // - Does not adjust container count counter in containing VirtualSpaceNode. - // Returns number of chunks removed. - int remove_chunks_in_area(MetaWord* p, size_t word_size); - - // Helper for chunk splitting: given a target chunk size and a larger free chunk, - // split up the larger chunk into n smaller chunks, at least one of which should be - // the target chunk of target chunk size. The smaller chunks, including the target - // chunk, are returned to the freelist. The pointer to the target chunk is returned. - // Note that this chunk is supposed to be removed from the freelist right away. - Metachunk* split_chunk(size_t target_chunk_word_size, Metachunk* chunk); - - public: - - ChunkManager(bool is_class) - : _is_class(is_class), _free_chunks_total(0), _free_chunks_count(0) { - _free_chunks[SpecializedIndex].set_size(get_size_for_nonhumongous_chunktype(SpecializedIndex, is_class)); - _free_chunks[SmallIndex].set_size(get_size_for_nonhumongous_chunktype(SmallIndex, is_class)); - _free_chunks[MediumIndex].set_size(get_size_for_nonhumongous_chunktype(MediumIndex, is_class)); - } - - // Add or delete (return) a chunk to the global freelist. - Metachunk* chunk_freelist_allocate(size_t word_size); - - // Map a size to a list index assuming that there are lists - // for special, small, medium, and humongous chunks. - ChunkIndex list_index(size_t size); - - // Map a given index to the chunk size. - size_t size_by_index(ChunkIndex index) const; - - bool is_class() const { return _is_class; } - - // Convenience accessors. - size_t medium_chunk_word_size() const { return size_by_index(MediumIndex); } - size_t small_chunk_word_size() const { return size_by_index(SmallIndex); } - size_t specialized_chunk_word_size() const { return size_by_index(SpecializedIndex); } - - // Take a chunk from the ChunkManager. The chunk is expected to be in - // the chunk manager (the freelist if non-humongous, the dictionary if - // humongous). - void remove_chunk(Metachunk* chunk); - - // Return a single chunk of type index to the ChunkManager. - void return_single_chunk(Metachunk* chunk); - - // Add the simple linked list of chunks to the freelist of chunks - // of type index. - void return_chunk_list(Metachunk* chunks); - - // Total of the space in the free chunks list - size_t free_chunks_total_words(); - size_t free_chunks_total_bytes(); - - // Number of chunks in the free chunks list - size_t free_chunks_count(); - - // Remove from a list by size. Selects list based on size of chunk. - Metachunk* free_chunks_get(size_t chunk_word_size); - -#define index_bounds_check(index) \ - assert(is_valid_chunktype(index), "Bad index: %d", (int) index) - - size_t num_free_chunks(ChunkIndex index) const { - index_bounds_check(index); - - if (index == HumongousIndex) { - return _humongous_dictionary.total_free_blocks(); - } - - ssize_t count = _free_chunks[index].count(); - return count == -1 ? 0 : (size_t) count; - } - - size_t size_free_chunks_in_bytes(ChunkIndex index) const { - index_bounds_check(index); - - size_t word_size = 0; - if (index == HumongousIndex) { - word_size = _humongous_dictionary.total_size(); - } else { - const size_t size_per_chunk_in_words = _free_chunks[index].size(); - word_size = size_per_chunk_in_words * num_free_chunks(index); - } - - return word_size * BytesPerWord; - } - - MetaspaceChunkFreeListSummary chunk_free_list_summary() const { - return MetaspaceChunkFreeListSummary(num_free_chunks(SpecializedIndex), - num_free_chunks(SmallIndex), - num_free_chunks(MediumIndex), - num_free_chunks(HumongousIndex), - size_free_chunks_in_bytes(SpecializedIndex), - size_free_chunks_in_bytes(SmallIndex), - size_free_chunks_in_bytes(MediumIndex), - size_free_chunks_in_bytes(HumongousIndex)); - } - - // Debug support - void verify(); - void slow_verify() { - if (VerifyMetaspace) { - verify(); - } - } - void locked_verify(); - void slow_locked_verify() { - if (VerifyMetaspace) { - locked_verify(); - } - } - - void locked_print_free_chunks(outputStream* st); - void locked_print_sum_free_chunks(outputStream* st); - - // Fill in current statistic values to the given statistics object. - void collect_statistics(ChunkManagerStatistics* out) const; - -}; - -class SmallBlocks : public CHeapObj { - const static uint _small_block_max_size = sizeof(TreeChunk >)/HeapWordSize; - // Note: this corresponds to the imposed miminum allocation size, see SpaceManager::get_allocation_word_size() - const static uint _small_block_min_size = sizeof(Metablock)/HeapWordSize; - - private: - FreeList _small_lists[_small_block_max_size - _small_block_min_size]; - - FreeList& list_at(size_t word_size) { - assert(word_size >= _small_block_min_size, "There are no metaspace objects less than %u words", _small_block_min_size); - return _small_lists[word_size - _small_block_min_size]; - } - - public: - SmallBlocks() { - for (uint i = _small_block_min_size; i < _small_block_max_size; i++) { - uint k = i - _small_block_min_size; - _small_lists[k].set_size(i); - } - } - - // Returns the total size, in words, of all blocks, across all block sizes. - size_t total_size() const { - size_t result = 0; - for (uint i = _small_block_min_size; i < _small_block_max_size; i++) { - uint k = i - _small_block_min_size; - result = result + _small_lists[k].count() * _small_lists[k].size(); - } - return result; - } - - // Returns the total number of all blocks across all block sizes. - uintx total_num_blocks() const { - uintx result = 0; - for (uint i = _small_block_min_size; i < _small_block_max_size; i++) { - uint k = i - _small_block_min_size; - result = result + _small_lists[k].count(); - } - return result; - } - - static uint small_block_max_size() { return _small_block_max_size; } - static uint small_block_min_size() { return _small_block_min_size; } - - MetaWord* get_block(size_t word_size) { - if (list_at(word_size).count() > 0) { - MetaWord* new_block = (MetaWord*) list_at(word_size).get_chunk_at_head(); - return new_block; - } else { - return NULL; - } - } - void return_block(Metablock* free_chunk, size_t word_size) { - list_at(word_size).return_chunk_at_head(free_chunk, false); - assert(list_at(word_size).count() > 0, "Should have a chunk"); - } - - void print_on(outputStream* st) const { - st->print_cr("SmallBlocks:"); - for (uint i = _small_block_min_size; i < _small_block_max_size; i++) { - uint k = i - _small_block_min_size; - st->print_cr("small_lists size " SIZE_FORMAT " count " SIZE_FORMAT, _small_lists[k].size(), _small_lists[k].count()); - } - } -}; - -// Used to manage the free list of Metablocks (a block corresponds -// to the allocation of a quantum of metadata). -class BlockFreelist : public CHeapObj { - BlockTreeDictionary* const _dictionary; - SmallBlocks* _small_blocks; - - // Only allocate and split from freelist if the size of the allocation - // is at least 1/4th the size of the available block. - const static int WasteMultiplier = 4; - - // Accessors - BlockTreeDictionary* dictionary() const { return _dictionary; } - SmallBlocks* small_blocks() { - if (_small_blocks == NULL) { - _small_blocks = new SmallBlocks(); - } - return _small_blocks; - } - - public: - BlockFreelist(); - ~BlockFreelist(); - - // Get and return a block to the free list - MetaWord* get_block(size_t word_size); - void return_block(MetaWord* p, size_t word_size); - - // Returns the total size, in words, of all blocks kept in this structure. - size_t total_size() const { - size_t result = dictionary()->total_size(); - if (_small_blocks != NULL) { - result = result + _small_blocks->total_size(); - } - return result; - } - - // Returns the number of all blocks kept in this structure. - uintx num_blocks() const { - uintx result = dictionary()->total_free_blocks(); - if (_small_blocks != NULL) { - result = result + _small_blocks->total_num_blocks(); - } - return result; - } - - static size_t min_dictionary_size() { return TreeChunk >::min_size(); } - void print_on(outputStream* st) const; -}; - -// Helper for Occupancy Bitmap. A type trait to give an all-bits-are-one-unsigned constant. -template struct all_ones { static const T value; }; -template <> struct all_ones { static const uint64_t value = 0xFFFFFFFFFFFFFFFFULL; }; -template <> struct all_ones { static const uint32_t value = 0xFFFFFFFF; }; - -// The OccupancyMap is a bitmap which, for a given VirtualSpaceNode, -// keeps information about -// - where a chunk starts -// - whether a chunk is in-use or free -// A bit in this bitmap represents one range of memory in the smallest -// chunk size (SpecializedChunk or ClassSpecializedChunk). -class OccupancyMap : public CHeapObj { - - // The address range this map covers. - const MetaWord* const _reference_address; - const size_t _word_size; - - // The word size of a specialized chunk, aka the number of words one - // bit in this map represents. - const size_t _smallest_chunk_word_size; - - // map data - // Data are organized in two bit layers: - // The first layer is the chunk-start-map. Here, a bit is set to mark - // the corresponding region as the head of a chunk. - // The second layer is the in-use-map. Here, a set bit indicates that - // the corresponding belongs to a chunk which is in use. - uint8_t* _map[2]; - - enum { layer_chunk_start_map = 0, layer_in_use_map = 1 }; - - // length, in bytes, of bitmap data - size_t _map_size; - - // Returns true if bit at position pos at bit-layer layer is set. - bool get_bit_at_position(unsigned pos, unsigned layer) const { - assert(layer == 0 || layer == 1, "Invalid layer %d", layer); - const unsigned byteoffset = pos / 8; - assert(byteoffset < _map_size, - "invalid byte offset (%u), map size is " SIZE_FORMAT ".", byteoffset, _map_size); - const unsigned mask = 1 << (pos % 8); - return (_map[layer][byteoffset] & mask) > 0; - } - - // Changes bit at position pos at bit-layer layer to value v. - void set_bit_at_position(unsigned pos, unsigned layer, bool v) { - assert(layer == 0 || layer == 1, "Invalid layer %d", layer); - const unsigned byteoffset = pos / 8; - assert(byteoffset < _map_size, - "invalid byte offset (%u), map size is " SIZE_FORMAT ".", byteoffset, _map_size); - const unsigned mask = 1 << (pos % 8); - if (v) { - _map[layer][byteoffset] |= mask; - } else { - _map[layer][byteoffset] &= ~mask; - } - } - - // Optimized case of is_any_bit_set_in_region for 32/64bit aligned access: - // pos is 32/64 aligned and num_bits is 32/64. - // This is the typical case when coalescing to medium chunks, whose size is - // 32 or 64 times the specialized chunk size (depending on class or non class - // case), so they occupy 64 bits which should be 64bit aligned, because - // chunks are chunk-size aligned. - template - bool is_any_bit_set_in_region_3264(unsigned pos, unsigned num_bits, unsigned layer) const { - assert(_map_size > 0, "not initialized"); - assert(layer == 0 || layer == 1, "Invalid layer %d.", layer); - assert(pos % (sizeof(T) * 8) == 0, "Bit position must be aligned (%u).", pos); - assert(num_bits == (sizeof(T) * 8), "Number of bits incorrect (%u).", num_bits); - const size_t byteoffset = pos / 8; - assert(byteoffset <= (_map_size - sizeof(T)), - "Invalid byte offset (" SIZE_FORMAT "), map size is " SIZE_FORMAT ".", byteoffset, _map_size); - const T w = *(T*)(_map[layer] + byteoffset); - return w > 0 ? true : false; - } - - // Returns true if any bit in region [pos1, pos1 + num_bits) is set in bit-layer layer. - bool is_any_bit_set_in_region(unsigned pos, unsigned num_bits, unsigned layer) const { - if (pos % 32 == 0 && num_bits == 32) { - return is_any_bit_set_in_region_3264(pos, num_bits, layer); - } else if (pos % 64 == 0 && num_bits == 64) { - return is_any_bit_set_in_region_3264(pos, num_bits, layer); - } else { - for (unsigned n = 0; n < num_bits; n ++) { - if (get_bit_at_position(pos + n, layer)) { - return true; - } - } - } - return false; - } - - // Returns true if any bit in region [p, p+word_size) is set in bit-layer layer. - bool is_any_bit_set_in_region(MetaWord* p, size_t word_size, unsigned layer) const { - assert(word_size % _smallest_chunk_word_size == 0, - "Region size " SIZE_FORMAT " not a multiple of smallest chunk size.", word_size); - const unsigned pos = get_bitpos_for_address(p); - const unsigned num_bits = (unsigned) (word_size / _smallest_chunk_word_size); - return is_any_bit_set_in_region(pos, num_bits, layer); - } - - // Optimized case of set_bits_of_region for 32/64bit aligned access: - // pos is 32/64 aligned and num_bits is 32/64. - // This is the typical case when coalescing to medium chunks, whose size - // is 32 or 64 times the specialized chunk size (depending on class or non - // class case), so they occupy 64 bits which should be 64bit aligned, - // because chunks are chunk-size aligned. - template - void set_bits_of_region_T(unsigned pos, unsigned num_bits, unsigned layer, bool v) { - assert(pos % (sizeof(T) * 8) == 0, "Bit position must be aligned to %u (%u).", - (unsigned)(sizeof(T) * 8), pos); - assert(num_bits == (sizeof(T) * 8), "Number of bits incorrect (%u), expected %u.", - num_bits, (unsigned)(sizeof(T) * 8)); - const size_t byteoffset = pos / 8; - assert(byteoffset <= (_map_size - sizeof(T)), - "invalid byte offset (" SIZE_FORMAT "), map size is " SIZE_FORMAT ".", byteoffset, _map_size); - T* const pw = (T*)(_map[layer] + byteoffset); - *pw = v ? all_ones::value : (T) 0; - } - - // Set all bits in a region starting at pos to a value. - void set_bits_of_region(unsigned pos, unsigned num_bits, unsigned layer, bool v) { - assert(_map_size > 0, "not initialized"); - assert(layer == 0 || layer == 1, "Invalid layer %d.", layer); - if (pos % 32 == 0 && num_bits == 32) { - set_bits_of_region_T(pos, num_bits, layer, v); - } else if (pos % 64 == 0 && num_bits == 64) { - set_bits_of_region_T(pos, num_bits, layer, v); - } else { - for (unsigned n = 0; n < num_bits; n ++) { - set_bit_at_position(pos + n, layer, v); - } - } - } - - // Helper: sets all bits in a region [p, p+word_size). - void set_bits_of_region(MetaWord* p, size_t word_size, unsigned layer, bool v) { - assert(word_size % _smallest_chunk_word_size == 0, - "Region size " SIZE_FORMAT " not a multiple of smallest chunk size.", word_size); - const unsigned pos = get_bitpos_for_address(p); - const unsigned num_bits = (unsigned) (word_size / _smallest_chunk_word_size); - set_bits_of_region(pos, num_bits, layer, v); - } - - // Helper: given an address, return the bit position representing that address. - unsigned get_bitpos_for_address(const MetaWord* p) const { - assert(_reference_address != NULL, "not initialized"); - assert(p >= _reference_address && p < _reference_address + _word_size, - "Address %p out of range for occupancy map [%p..%p).", - p, _reference_address, _reference_address + _word_size); - assert(is_aligned(p, _smallest_chunk_word_size * sizeof(MetaWord)), - "Address not aligned (%p).", p); - const ptrdiff_t d = (p - _reference_address) / _smallest_chunk_word_size; - assert(d >= 0 && (size_t)d < _map_size * 8, "Sanity."); - return (unsigned) d; - } - - public: - - OccupancyMap(const MetaWord* reference_address, size_t word_size, size_t smallest_chunk_word_size) : - _reference_address(reference_address), _word_size(word_size), - _smallest_chunk_word_size(smallest_chunk_word_size) { - assert(reference_address != NULL, "invalid reference address"); - assert(is_aligned(reference_address, smallest_chunk_word_size), - "Reference address not aligned to smallest chunk size."); - assert(is_aligned(word_size, smallest_chunk_word_size), - "Word_size shall be a multiple of the smallest chunk size."); - // Calculate bitmap size: one bit per smallest_chunk_word_size'd area. - size_t num_bits = word_size / smallest_chunk_word_size; - _map_size = (num_bits + 7) / 8; - assert(_map_size * 8 >= num_bits, "sanity"); - _map[0] = (uint8_t*) os::malloc(_map_size, mtInternal); - _map[1] = (uint8_t*) os::malloc(_map_size, mtInternal); - assert(_map[0] != NULL && _map[1] != NULL, "Occupancy Map: allocation failed."); - memset(_map[1], 0, _map_size); - memset(_map[0], 0, _map_size); - // Sanity test: the first respectively last possible chunk start address in - // the covered range shall map to the first and last bit in the bitmap. - assert(get_bitpos_for_address(reference_address) == 0, - "First chunk address in range must map to fist bit in bitmap."); - assert(get_bitpos_for_address(reference_address + word_size - smallest_chunk_word_size) == num_bits - 1, - "Last chunk address in range must map to last bit in bitmap."); - } - - ~OccupancyMap() { - os::free(_map[0]); - os::free(_map[1]); - } - - // Returns true if at address x a chunk is starting. - bool chunk_starts_at_address(MetaWord* p) const { - const unsigned pos = get_bitpos_for_address(p); - return get_bit_at_position(pos, layer_chunk_start_map); - } - - void set_chunk_starts_at_address(MetaWord* p, bool v) { - const unsigned pos = get_bitpos_for_address(p); - set_bit_at_position(pos, layer_chunk_start_map, v); - } - - // Removes all chunk-start-bits inside a region, typically as a - // result of a chunk merge. - void wipe_chunk_start_bits_in_region(MetaWord* p, size_t word_size) { - set_bits_of_region(p, word_size, layer_chunk_start_map, false); - } - - // Returns true if there are life (in use) chunks in the region limited - // by [p, p+word_size). - bool is_region_in_use(MetaWord* p, size_t word_size) const { - return is_any_bit_set_in_region(p, word_size, layer_in_use_map); - } - - // Marks the region starting at p with the size word_size as in use - // or free, depending on v. - void set_region_in_use(MetaWord* p, size_t word_size, bool v) { - set_bits_of_region(p, word_size, layer_in_use_map, v); - } - -#ifdef ASSERT - // Verify occupancy map for the address range [from, to). - // We need to tell it the address range, because the memory the - // occupancy map is covering may not be fully comitted yet. - void verify(MetaWord* from, MetaWord* to) { - Metachunk* chunk = NULL; - int nth_bit_for_chunk = 0; - MetaWord* chunk_end = NULL; - for (MetaWord* p = from; p < to; p += _smallest_chunk_word_size) { - const unsigned pos = get_bitpos_for_address(p); - // Check the chunk-starts-info: - if (get_bit_at_position(pos, layer_chunk_start_map)) { - // Chunk start marked in bitmap. - chunk = (Metachunk*) p; - if (chunk_end != NULL) { - assert(chunk_end == p, "Unexpected chunk start found at %p (expected " - "the next chunk to start at %p).", p, chunk_end); - } - assert(chunk->is_valid_sentinel(), "Invalid chunk at address %p.", p); - if (chunk->get_chunk_type() != HumongousIndex) { - guarantee(is_aligned(p, chunk->word_size()), "Chunk %p not aligned.", p); - } - chunk_end = p + chunk->word_size(); - nth_bit_for_chunk = 0; - assert(chunk_end <= to, "Chunk end overlaps test address range."); - } else { - // No chunk start marked in bitmap. - assert(chunk != NULL, "Chunk should start at start of address range."); - assert(p < chunk_end, "Did not find expected chunk start at %p.", p); - nth_bit_for_chunk ++; - } - // Check the in-use-info: - const bool in_use_bit = get_bit_at_position(pos, layer_in_use_map); - if (in_use_bit) { - assert(!chunk->is_tagged_free(), "Chunk %p: marked in-use in map but is free (bit %u).", - chunk, nth_bit_for_chunk); - } else { - assert(chunk->is_tagged_free(), "Chunk %p: marked free in map but is in use (bit %u).", - chunk, nth_bit_for_chunk); - } - } - } - - // Verify that a given chunk is correctly accounted for in the bitmap. - void verify_for_chunk(Metachunk* chunk) { - assert(chunk_starts_at_address((MetaWord*) chunk), - "No chunk start marked in map for chunk %p.", chunk); - // For chunks larger than the minimal chunk size, no other chunk - // must start in its area. - if (chunk->word_size() > _smallest_chunk_word_size) { - assert(!is_any_bit_set_in_region(((MetaWord*) chunk) + _smallest_chunk_word_size, - chunk->word_size() - _smallest_chunk_word_size, layer_chunk_start_map), - "No chunk must start within another chunk."); - } - if (!chunk->is_tagged_free()) { - assert(is_region_in_use((MetaWord*)chunk, chunk->word_size()), - "Chunk %p is in use but marked as free in map (%d %d).", - chunk, chunk->get_chunk_type(), chunk->get_origin()); - } else { - assert(!is_region_in_use((MetaWord*)chunk, chunk->word_size()), - "Chunk %p is free but marked as in-use in map (%d %d).", - chunk, chunk->get_chunk_type(), chunk->get_origin()); - } - } - -#endif // ASSERT - -}; - -// A VirtualSpaceList node. -class VirtualSpaceNode : public CHeapObj { - friend class VirtualSpaceList; - - // Link to next VirtualSpaceNode - VirtualSpaceNode* _next; - - // Whether this node is contained in class or metaspace. - const bool _is_class; - - // total in the VirtualSpace - MemRegion _reserved; - ReservedSpace _rs; - VirtualSpace _virtual_space; - MetaWord* _top; - // count of chunks contained in this VirtualSpace - uintx _container_count; - - OccupancyMap* _occupancy_map; - - // Convenience functions to access the _virtual_space - char* low() const { return virtual_space()->low(); } - char* high() const { return virtual_space()->high(); } - - // The first Metachunk will be allocated at the bottom of the - // VirtualSpace - Metachunk* first_chunk() { return (Metachunk*) bottom(); } - - // Committed but unused space in the virtual space - size_t free_words_in_vs() const; - - // True if this node belongs to class metaspace. - bool is_class() const { return _is_class; } - - // Helper function for take_from_committed: allocate padding chunks - // until top is at the given address. - void allocate_padding_chunks_until_top_is_at(MetaWord* target_top); - - public: - - VirtualSpaceNode(bool is_class, size_t byte_size); - VirtualSpaceNode(bool is_class, ReservedSpace rs) : - _is_class(is_class), _top(NULL), _next(NULL), _rs(rs), _container_count(0), _occupancy_map(NULL) {} - ~VirtualSpaceNode(); - - // Convenience functions for logical bottom and end - MetaWord* bottom() const { return (MetaWord*) _virtual_space.low(); } - MetaWord* end() const { return (MetaWord*) _virtual_space.high(); } - - const OccupancyMap* occupancy_map() const { return _occupancy_map; } - OccupancyMap* occupancy_map() { return _occupancy_map; } - - bool contains(const void* ptr) { return ptr >= low() && ptr < high(); } - - size_t reserved_words() const { return _virtual_space.reserved_size() / BytesPerWord; } - size_t committed_words() const { return _virtual_space.actual_committed_size() / BytesPerWord; } - - bool is_pre_committed() const { return _virtual_space.special(); } - - // address of next available space in _virtual_space; - // Accessors - VirtualSpaceNode* next() { return _next; } - void set_next(VirtualSpaceNode* v) { _next = v; } - - void set_reserved(MemRegion const v) { _reserved = v; } - void set_top(MetaWord* v) { _top = v; } - - // Accessors - MemRegion* reserved() { return &_reserved; } - VirtualSpace* virtual_space() const { return (VirtualSpace*) &_virtual_space; } - - // Returns true if "word_size" is available in the VirtualSpace - bool is_available(size_t word_size) { return word_size <= pointer_delta(end(), _top, sizeof(MetaWord)); } - - MetaWord* top() const { return _top; } - void inc_top(size_t word_size) { _top += word_size; } - - uintx container_count() { return _container_count; } - void inc_container_count(); - void dec_container_count(); -#ifdef ASSERT - uintx container_count_slow(); - void verify_container_count(); -#endif - - // used and capacity in this single entry in the list - size_t used_words_in_vs() const; - size_t capacity_words_in_vs() const; - - bool initialize(); - - // get space from the virtual space - Metachunk* take_from_committed(size_t chunk_word_size); - - // Allocate a chunk from the virtual space and return it. - Metachunk* get_chunk_vs(size_t chunk_word_size); - - // Expands/shrinks the committed space in a virtual space. Delegates - // to Virtualspace - bool expand_by(size_t min_words, size_t preferred_words); - - // In preparation for deleting this node, remove all the chunks - // in the node from any freelist. - void purge(ChunkManager* chunk_manager); - - // If an allocation doesn't fit in the current node a new node is created. - // Allocate chunks out of the remaining committed space in this node - // to avoid wasting that memory. - // This always adds up because all the chunk sizes are multiples of - // the smallest chunk size. - void retire(ChunkManager* chunk_manager); - - - void print_on(outputStream* st) const { print_on(st, K); } - void print_on(outputStream* st, size_t scale) const; - void print_map(outputStream* st, bool is_class) const; - - // Debug support - DEBUG_ONLY(void mangle();) - // Verify counters, all chunks in this list node and the occupancy map. - DEBUG_ONLY(void verify();) - // Verify that all free chunks in this node are ideally merged - // (there not should be multiple small chunks where a large chunk could exist.) - DEBUG_ONLY(void verify_free_chunks_are_ideally_merged();) - -}; - -#define assert_is_aligned(value, alignment) \ - assert(is_aligned((value), (alignment)), \ - SIZE_FORMAT_HEX " is not aligned to " \ - SIZE_FORMAT, (size_t)(uintptr_t)value, (alignment)) - -#define assert_counter(expected_value, real_value, msg) \ - assert( (expected_value) == (real_value), \ - "Counter mismatch (%s): expected " SIZE_FORMAT \ - ", but got: " SIZE_FORMAT ".", msg, expected_value, \ - real_value); - -// Decide if large pages should be committed when the memory is reserved. -static bool should_commit_large_pages_when_reserving(size_t bytes) { - if (UseLargePages && UseLargePagesInMetaspace && !os::can_commit_large_page_memory()) { - size_t words = bytes / BytesPerWord; - bool is_class = false; // We never reserve large pages for the class space. - if (MetaspaceGC::can_expand(words, is_class) && - MetaspaceGC::allowed_expansion() >= words) { - return true; - } - } - - return false; -} - - // byte_size is the size of the associated virtualspace. -VirtualSpaceNode::VirtualSpaceNode(bool is_class, size_t bytes) : - _is_class(is_class), _top(NULL), _next(NULL), _rs(), _container_count(0), _occupancy_map(NULL) { - assert_is_aligned(bytes, Metaspace::reserve_alignment()); - bool large_pages = should_commit_large_pages_when_reserving(bytes); - _rs = ReservedSpace(bytes, Metaspace::reserve_alignment(), large_pages); - - if (_rs.is_reserved()) { - assert(_rs.base() != NULL, "Catch if we get a NULL address"); - assert(_rs.size() != 0, "Catch if we get a 0 size"); - assert_is_aligned(_rs.base(), Metaspace::reserve_alignment()); - assert_is_aligned(_rs.size(), Metaspace::reserve_alignment()); - - MemTracker::record_virtual_memory_type((address)_rs.base(), mtClass); - } -} - -void VirtualSpaceNode::purge(ChunkManager* chunk_manager) { - DEBUG_ONLY(this->verify();) - Metachunk* chunk = first_chunk(); - Metachunk* invalid_chunk = (Metachunk*) top(); - while (chunk < invalid_chunk ) { - assert(chunk->is_tagged_free(), "Should be tagged free"); - MetaWord* next = ((MetaWord*)chunk) + chunk->word_size(); - chunk_manager->remove_chunk(chunk); - chunk->remove_sentinel(); - assert(chunk->next() == NULL && - chunk->prev() == NULL, - "Was not removed from its list"); - chunk = (Metachunk*) next; - } -} - -void VirtualSpaceNode::print_map(outputStream* st, bool is_class) const { - - if (bottom() == top()) { - return; - } - - const size_t spec_chunk_size = is_class ? ClassSpecializedChunk : SpecializedChunk; - const size_t small_chunk_size = is_class ? ClassSmallChunk : SmallChunk; - const size_t med_chunk_size = is_class ? ClassMediumChunk : MediumChunk; - - int line_len = 100; - const size_t section_len = align_up(spec_chunk_size * line_len, med_chunk_size); - line_len = (int)(section_len / spec_chunk_size); - - static const int NUM_LINES = 4; - - char* lines[NUM_LINES]; - for (int i = 0; i < NUM_LINES; i ++) { - lines[i] = (char*)os::malloc(line_len, mtInternal); - } - int pos = 0; - const MetaWord* p = bottom(); - const Metachunk* chunk = (const Metachunk*)p; - const MetaWord* chunk_end = p + chunk->word_size(); - while (p < top()) { - if (pos == line_len) { - pos = 0; - for (int i = 0; i < NUM_LINES; i ++) { - st->fill_to(22); - st->print_raw(lines[i], line_len); - st->cr(); - } - } - if (pos == 0) { - st->print(PTR_FORMAT ":", p2i(p)); - } - if (p == chunk_end) { - chunk = (Metachunk*)p; - chunk_end = p + chunk->word_size(); - } - // line 1: chunk starting points (a dot if that area is a chunk start). - lines[0][pos] = p == (const MetaWord*)chunk ? '.' : ' '; - - // Line 2: chunk type (x=spec, s=small, m=medium, h=humongous), uppercase if - // chunk is in use. - const bool chunk_is_free = ((Metachunk*)chunk)->is_tagged_free(); - if (chunk->word_size() == spec_chunk_size) { - lines[1][pos] = chunk_is_free ? 'x' : 'X'; - } else if (chunk->word_size() == small_chunk_size) { - lines[1][pos] = chunk_is_free ? 's' : 'S'; - } else if (chunk->word_size() == med_chunk_size) { - lines[1][pos] = chunk_is_free ? 'm' : 'M'; - } else if (chunk->word_size() > med_chunk_size) { - lines[1][pos] = chunk_is_free ? 'h' : 'H'; - } else { - ShouldNotReachHere(); - } - - // Line 3: chunk origin - const ChunkOrigin origin = chunk->get_origin(); - lines[2][pos] = origin == origin_normal ? ' ' : '0' + (int) origin; - - // Line 4: Virgin chunk? Virgin chunks are chunks created as a byproduct of padding or splitting, - // but were never used. - lines[3][pos] = chunk->get_use_count() > 0 ? ' ' : 'v'; - - p += spec_chunk_size; - pos ++; - } - if (pos > 0) { - for (int i = 0; i < NUM_LINES; i ++) { - st->fill_to(22); - st->print_raw(lines[i], line_len); - st->cr(); - } - } - for (int i = 0; i < NUM_LINES; i ++) { - os::free(lines[i]); - } -} - - -#ifdef ASSERT -uintx VirtualSpaceNode::container_count_slow() { - uintx count = 0; - Metachunk* chunk = first_chunk(); - Metachunk* invalid_chunk = (Metachunk*) top(); - while (chunk < invalid_chunk ) { - MetaWord* next = ((MetaWord*)chunk) + chunk->word_size(); - do_verify_chunk(chunk); - // Don't count the chunks on the free lists. Those are - // still part of the VirtualSpaceNode but not currently - // counted. - if (!chunk->is_tagged_free()) { - count++; - } - chunk = (Metachunk*) next; - } - return count; -} -#endif - -#ifdef ASSERT -// Verify counters, all chunks in this list node and the occupancy map. -void VirtualSpaceNode::verify() { - uintx num_in_use_chunks = 0; - Metachunk* chunk = first_chunk(); - Metachunk* invalid_chunk = (Metachunk*) top(); - - // Iterate the chunks in this node and verify each chunk. - while (chunk < invalid_chunk ) { - DEBUG_ONLY(do_verify_chunk(chunk);) - if (!chunk->is_tagged_free()) { - num_in_use_chunks ++; - } - MetaWord* next = ((MetaWord*)chunk) + chunk->word_size(); - chunk = (Metachunk*) next; - } - assert(_container_count == num_in_use_chunks, "Container count mismatch (real: " UINTX_FORMAT - ", counter: " UINTX_FORMAT ".", num_in_use_chunks, _container_count); - // Also verify the occupancy map. - occupancy_map()->verify(this->bottom(), this->top()); -} -#endif // ASSERT - -#ifdef ASSERT -// Verify that all free chunks in this node are ideally merged -// (there not should be multiple small chunks where a large chunk could exist.) -void VirtualSpaceNode::verify_free_chunks_are_ideally_merged() { - Metachunk* chunk = first_chunk(); - Metachunk* invalid_chunk = (Metachunk*) top(); - // Shorthands. - const size_t size_med = (is_class() ? ClassMediumChunk : MediumChunk) * BytesPerWord; - const size_t size_small = (is_class() ? ClassSmallChunk : SmallChunk) * BytesPerWord; - int num_free_chunks_since_last_med_boundary = -1; - int num_free_chunks_since_last_small_boundary = -1; - while (chunk < invalid_chunk ) { - // Test for missed chunk merge opportunities: count number of free chunks since last chunk boundary. - // Reset the counter when encountering a non-free chunk. - if (chunk->get_chunk_type() != HumongousIndex) { - if (chunk->is_tagged_free()) { - // Count successive free, non-humongous chunks. - if (is_aligned(chunk, size_small)) { - assert(num_free_chunks_since_last_small_boundary <= 1, - "Missed chunk merge opportunity at " PTR_FORMAT " for chunk size " SIZE_FORMAT_HEX ".", p2i(chunk) - size_small, size_small); - num_free_chunks_since_last_small_boundary = 0; - } else if (num_free_chunks_since_last_small_boundary != -1) { - num_free_chunks_since_last_small_boundary ++; - } - if (is_aligned(chunk, size_med)) { - assert(num_free_chunks_since_last_med_boundary <= 1, - "Missed chunk merge opportunity at " PTR_FORMAT " for chunk size " SIZE_FORMAT_HEX ".", p2i(chunk) - size_med, size_med); - num_free_chunks_since_last_med_boundary = 0; - } else if (num_free_chunks_since_last_med_boundary != -1) { - num_free_chunks_since_last_med_boundary ++; - } - } else { - // Encountering a non-free chunk, reset counters. - num_free_chunks_since_last_med_boundary = -1; - num_free_chunks_since_last_small_boundary = -1; - } - } else { - // One cannot merge areas with a humongous chunk in the middle. Reset counters. - num_free_chunks_since_last_med_boundary = -1; - num_free_chunks_since_last_small_boundary = -1; - } - - MetaWord* next = ((MetaWord*)chunk) + chunk->word_size(); - chunk = (Metachunk*) next; - } -} -#endif // ASSERT - -// List of VirtualSpaces for metadata allocation. -class VirtualSpaceList : public CHeapObj { - friend class VirtualSpaceNode; - - enum VirtualSpaceSizes { - VirtualSpaceSize = 256 * K - }; - - // Head of the list - VirtualSpaceNode* _virtual_space_list; - // virtual space currently being used for allocations - VirtualSpaceNode* _current_virtual_space; - - // Is this VirtualSpaceList used for the compressed class space - bool _is_class; - - // Sum of reserved and committed memory in the virtual spaces - size_t _reserved_words; - size_t _committed_words; - - // Number of virtual spaces - size_t _virtual_space_count; - - ~VirtualSpaceList(); - - VirtualSpaceNode* virtual_space_list() const { return _virtual_space_list; } - - void set_virtual_space_list(VirtualSpaceNode* v) { - _virtual_space_list = v; - } - void set_current_virtual_space(VirtualSpaceNode* v) { - _current_virtual_space = v; - } - - void link_vs(VirtualSpaceNode* new_entry); - - // Get another virtual space and add it to the list. This - // is typically prompted by a failed attempt to allocate a chunk - // and is typically followed by the allocation of a chunk. - bool create_new_virtual_space(size_t vs_word_size); - - // Chunk up the unused committed space in the current - // virtual space and add the chunks to the free list. - void retire_current_virtual_space(); - - public: - VirtualSpaceList(size_t word_size); - VirtualSpaceList(ReservedSpace rs); - - size_t free_bytes(); - - Metachunk* get_new_chunk(size_t chunk_word_size, - size_t suggested_commit_granularity); - - bool expand_node_by(VirtualSpaceNode* node, - size_t min_words, - size_t preferred_words); - - bool expand_by(size_t min_words, - size_t preferred_words); - - VirtualSpaceNode* current_virtual_space() { - return _current_virtual_space; - } - - bool is_class() const { return _is_class; } - - bool initialization_succeeded() { return _virtual_space_list != NULL; } - - size_t reserved_words() { return _reserved_words; } - size_t reserved_bytes() { return reserved_words() * BytesPerWord; } - size_t committed_words() { return _committed_words; } - size_t committed_bytes() { return committed_words() * BytesPerWord; } - - void inc_reserved_words(size_t v); - void dec_reserved_words(size_t v); - void inc_committed_words(size_t v); - void dec_committed_words(size_t v); - void inc_virtual_space_count(); - void dec_virtual_space_count(); - - bool contains(const void* ptr); - - // Unlink empty VirtualSpaceNodes and free it. - void purge(ChunkManager* chunk_manager); - - void print_on(outputStream* st) const { print_on(st, K); } - void print_on(outputStream* st, size_t scale) const; - void print_map(outputStream* st) const; - - class VirtualSpaceListIterator : public StackObj { - VirtualSpaceNode* _virtual_spaces; - public: - VirtualSpaceListIterator(VirtualSpaceNode* virtual_spaces) : - _virtual_spaces(virtual_spaces) {} - - bool repeat() { - return _virtual_spaces != NULL; - } - - VirtualSpaceNode* get_next() { - VirtualSpaceNode* result = _virtual_spaces; - if (_virtual_spaces != NULL) { - _virtual_spaces = _virtual_spaces->next(); - } - return result; - } - }; -}; - -class Metadebug : AllStatic { - // Debugging support for Metaspaces - static int _allocation_fail_alot_count; - - public: - - static void init_allocation_fail_alot_count(); -#ifdef ASSERT - static bool test_metadata_failure(); -#endif -}; - -int Metadebug::_allocation_fail_alot_count = 0; - - -// SpaceManager - used by Metaspace to handle allocations -class SpaceManager : public CHeapObj { - friend class ClassLoaderMetaspace; - friend class Metadebug; - - private: - - // protects allocations - Mutex* const _lock; - - // Type of metadata allocated. - const Metaspace::MetadataType _mdtype; - - // Type of metaspace - const Metaspace::MetaspaceType _space_type; - - // List of chunks in use by this SpaceManager. Allocations - // are done from the current chunk. The list is used for deallocating - // chunks when the SpaceManager is freed. - Metachunk* _chunk_list; - Metachunk* _current_chunk; - - // Maximum number of small chunks to allocate to a SpaceManager - static uint const _small_chunk_limit; - - // Maximum number of specialize chunks to allocate for anonymous and delegating - // metadata space to a SpaceManager - static uint const _anon_and_delegating_metadata_specialize_chunk_limit; - - // Some running counters, but lets keep their number small to not add to much to - // the per-classloader footprint. - // Note: capacity = used + free + waste + overhead. We do not keep running counters for - // free and waste. Their sum can be deduced from the three other values. - size_t _overhead_words; - size_t _capacity_words; - size_t _used_words; - uintx _num_chunks_by_type[NumberOfInUseLists]; - - // Free lists of blocks are per SpaceManager since they - // are assumed to be in chunks in use by the SpaceManager - // and all chunks in use by a SpaceManager are freed when - // the class loader using the SpaceManager is collected. - BlockFreelist* _block_freelists; - - private: - // Accessors - Metachunk* chunk_list() const { return _chunk_list; } - - BlockFreelist* block_freelists() const { return _block_freelists; } - - Metaspace::MetadataType mdtype() { return _mdtype; } - - VirtualSpaceList* vs_list() const { return Metaspace::get_space_list(_mdtype); } - ChunkManager* chunk_manager() const { return Metaspace::get_chunk_manager(_mdtype); } - - Metachunk* current_chunk() const { return _current_chunk; } - void set_current_chunk(Metachunk* v) { - _current_chunk = v; - } - - Metachunk* find_current_chunk(size_t word_size); - - // Add chunk to the list of chunks in use - void add_chunk(Metachunk* v, bool make_current); - void retire_current_chunk(); - - Mutex* lock() const { return _lock; } - - // Adds to the given statistic object. Expects to be locked with lock(). - void add_to_statistics_locked(SpaceManagerStatistics* out) const; - - // Verify internal counters against the current state. Expects to be locked with lock(). - DEBUG_ONLY(void verify_metrics_locked() const;) - - public: - SpaceManager(Metaspace::MetadataType mdtype, - Metaspace::MetaspaceType space_type, - Mutex* lock); - ~SpaceManager(); - - enum ChunkMultiples { - MediumChunkMultiple = 4 - }; - - static size_t specialized_chunk_size(bool is_class) { return is_class ? ClassSpecializedChunk : SpecializedChunk; } - static size_t small_chunk_size(bool is_class) { return is_class ? ClassSmallChunk : SmallChunk; } - static size_t medium_chunk_size(bool is_class) { return is_class ? ClassMediumChunk : MediumChunk; } - - static size_t smallest_chunk_size(bool is_class) { return specialized_chunk_size(is_class); } - - // Accessors - bool is_class() const { return _mdtype == Metaspace::ClassType; } - - size_t specialized_chunk_size() const { return specialized_chunk_size(is_class()); } - size_t small_chunk_size() const { return small_chunk_size(is_class()); } - size_t medium_chunk_size() const { return medium_chunk_size(is_class()); } - - size_t smallest_chunk_size() const { return smallest_chunk_size(is_class()); } - - size_t medium_chunk_bunch() const { return medium_chunk_size() * MediumChunkMultiple; } - - bool is_humongous(size_t word_size) { return word_size > medium_chunk_size(); } - - size_t capacity_words() const { return _capacity_words; } - size_t used_words() const { return _used_words; } - size_t overhead_words() const { return _overhead_words; } - - // Adjust local, global counters after a new chunk has been added. - void account_for_new_chunk(const Metachunk* new_chunk); - - // Adjust local, global counters after space has been allocated from the current chunk. - void account_for_allocation(size_t words); - - // Adjust global counters just before the SpaceManager dies, after all its chunks - // have been returned to the freelist. - void account_for_spacemanager_death(); - - // Adjust the initial chunk size to match one of the fixed chunk list sizes, - // or return the unadjusted size if the requested size is humongous. - static size_t adjust_initial_chunk_size(size_t requested, bool is_class_space); - size_t adjust_initial_chunk_size(size_t requested) const; - - // Get the initial chunks size for this metaspace type. - size_t get_initial_chunk_size(Metaspace::MetaspaceType type) const; - - // Todo: remove this once we have counters by chunk type. - uintx num_chunks_by_type(ChunkIndex chunk_type) const { return _num_chunks_by_type[chunk_type]; } - - Metachunk* get_new_chunk(size_t chunk_word_size); - - // Block allocation and deallocation. - // Allocates a block from the current chunk - MetaWord* allocate(size_t word_size); - - // Helper for allocations - MetaWord* allocate_work(size_t word_size); - - // Returns a block to the per manager freelist - void deallocate(MetaWord* p, size_t word_size); - - // Based on the allocation size and a minimum chunk size, - // returned chunk size (for expanding space for chunk allocation). - size_t calc_chunk_size(size_t allocation_word_size); - - // Called when an allocation from the current chunk fails. - // Gets a new chunk (may require getting a new virtual space), - // and allocates from that chunk. - MetaWord* grow_and_allocate(size_t word_size); - - // Notify memory usage to MemoryService. - void track_metaspace_memory_usage(); - - // debugging support. - - void print_on(outputStream* st) const; - void locked_print_chunks_in_use_on(outputStream* st) const; - - void verify(); - void verify_chunk_size(Metachunk* chunk); - - // This adjusts the size given to be greater than the minimum allocation size in - // words for data in metaspace. Esentially the minimum size is currently 3 words. - size_t get_allocation_word_size(size_t word_size) { - size_t byte_size = word_size * BytesPerWord; - - size_t raw_bytes_size = MAX2(byte_size, sizeof(Metablock)); - raw_bytes_size = align_up(raw_bytes_size, Metachunk::object_alignment()); - - size_t raw_word_size = raw_bytes_size / BytesPerWord; - assert(raw_word_size * BytesPerWord == raw_bytes_size, "Size problem"); - - return raw_word_size; - } - - // Adds to the given statistic object. - void add_to_statistics(SpaceManagerStatistics* out) const; - - // Verify internal counters against the current state. - DEBUG_ONLY(void verify_metrics() const;) - -}; - -uint const SpaceManager::_small_chunk_limit = 4; -uint const SpaceManager::_anon_and_delegating_metadata_specialize_chunk_limit = 4; - -void VirtualSpaceNode::inc_container_count() { - assert_lock_strong(MetaspaceExpand_lock); - _container_count++; -} - -void VirtualSpaceNode::dec_container_count() { - assert_lock_strong(MetaspaceExpand_lock); - _container_count--; -} - -#ifdef ASSERT -void VirtualSpaceNode::verify_container_count() { - assert(_container_count == container_count_slow(), - "Inconsistency in container_count _container_count " UINTX_FORMAT - " container_count_slow() " UINTX_FORMAT, _container_count, container_count_slow()); -} -#endif - // BlockFreelist methods -BlockFreelist::BlockFreelist() : _dictionary(new BlockTreeDictionary()), _small_blocks(NULL) {} - -BlockFreelist::~BlockFreelist() { - delete _dictionary; - if (_small_blocks != NULL) { - delete _small_blocks; - } -} - -void BlockFreelist::return_block(MetaWord* p, size_t word_size) { - assert(word_size >= SmallBlocks::small_block_min_size(), "never return dark matter"); - - Metablock* free_chunk = ::new (p) Metablock(word_size); - if (word_size < SmallBlocks::small_block_max_size()) { - small_blocks()->return_block(free_chunk, word_size); - } else { - dictionary()->return_chunk(free_chunk); -} - log_trace(gc, metaspace, freelist, blocks)("returning block at " INTPTR_FORMAT " size = " - SIZE_FORMAT, p2i(free_chunk), word_size); -} - -MetaWord* BlockFreelist::get_block(size_t word_size) { - assert(word_size >= SmallBlocks::small_block_min_size(), "never get dark matter"); - - // Try small_blocks first. - if (word_size < SmallBlocks::small_block_max_size()) { - // Don't create small_blocks() until needed. small_blocks() allocates the small block list for - // this space manager. - MetaWord* new_block = (MetaWord*) small_blocks()->get_block(word_size); - if (new_block != NULL) { - log_trace(gc, metaspace, freelist, blocks)("getting block at " INTPTR_FORMAT " size = " SIZE_FORMAT, - p2i(new_block), word_size); - return new_block; - } - } - - if (word_size < BlockFreelist::min_dictionary_size()) { - // If allocation in small blocks fails, this is Dark Matter. Too small for dictionary. - return NULL; - } - - Metablock* free_block = dictionary()->get_chunk(word_size); - if (free_block == NULL) { - return NULL; - } - - const size_t block_size = free_block->size(); - if (block_size > WasteMultiplier * word_size) { - return_block((MetaWord*)free_block, block_size); - return NULL; - } - - MetaWord* new_block = (MetaWord*)free_block; - assert(block_size >= word_size, "Incorrect size of block from freelist"); - const size_t unused = block_size - word_size; - if (unused >= SmallBlocks::small_block_min_size()) { - return_block(new_block + word_size, unused); - } - - log_trace(gc, metaspace, freelist, blocks)("getting block at " INTPTR_FORMAT " size = " SIZE_FORMAT, - p2i(new_block), word_size); - return new_block; -} - -void BlockFreelist::print_on(outputStream* st) const { - dictionary()->print_free_lists(st); - if (_small_blocks != NULL) { - _small_blocks->print_on(st); - } -} - // VirtualSpaceNode methods -VirtualSpaceNode::~VirtualSpaceNode() { - _rs.release(); - if (_occupancy_map != NULL) { - delete _occupancy_map; - } -#ifdef ASSERT - size_t word_size = sizeof(*this) / BytesPerWord; - Copy::fill_to_words((HeapWord*) this, word_size, 0xf1f1f1f1); -#endif -} - -size_t VirtualSpaceNode::used_words_in_vs() const { - return pointer_delta(top(), bottom(), sizeof(MetaWord)); -} - -// Space committed in the VirtualSpace -size_t VirtualSpaceNode::capacity_words_in_vs() const { - return pointer_delta(end(), bottom(), sizeof(MetaWord)); -} - -size_t VirtualSpaceNode::free_words_in_vs() const { - return pointer_delta(end(), top(), sizeof(MetaWord)); -} - -// Given an address larger than top(), allocate padding chunks until top is at the given address. -void VirtualSpaceNode::allocate_padding_chunks_until_top_is_at(MetaWord* target_top) { - - assert(target_top > top(), "Sanity"); - - // Padding chunks are added to the freelist. - ChunkManager* const chunk_manager = Metaspace::get_chunk_manager(this->is_class()); - - // shorthands - const size_t spec_word_size = chunk_manager->specialized_chunk_word_size(); - const size_t small_word_size = chunk_manager->small_chunk_word_size(); - const size_t med_word_size = chunk_manager->medium_chunk_word_size(); - - while (top() < target_top) { - - // We could make this coding more generic, but right now we only deal with two possible chunk sizes - // for padding chunks, so it is not worth it. - size_t padding_chunk_word_size = small_word_size; - if (is_aligned(top(), small_word_size * sizeof(MetaWord)) == false) { - assert_is_aligned(top(), spec_word_size * sizeof(MetaWord)); // Should always hold true. - padding_chunk_word_size = spec_word_size; - } - MetaWord* here = top(); - assert_is_aligned(here, padding_chunk_word_size * sizeof(MetaWord)); - inc_top(padding_chunk_word_size); - - // Create new padding chunk. - ChunkIndex padding_chunk_type = get_chunk_type_by_size(padding_chunk_word_size, is_class()); - assert(padding_chunk_type == SpecializedIndex || padding_chunk_type == SmallIndex, "sanity"); - - Metachunk* const padding_chunk = - ::new (here) Metachunk(padding_chunk_type, is_class(), padding_chunk_word_size, this); - assert(padding_chunk == (Metachunk*)here, "Sanity"); - DEBUG_ONLY(padding_chunk->set_origin(origin_pad);) - log_trace(gc, metaspace, freelist)("Created padding chunk in %s at " - PTR_FORMAT ", size " SIZE_FORMAT_HEX ".", - (is_class() ? "class space " : "metaspace"), - p2i(padding_chunk), padding_chunk->word_size() * sizeof(MetaWord)); - - // Mark chunk start in occupancy map. - occupancy_map()->set_chunk_starts_at_address((MetaWord*)padding_chunk, true); - - // Chunks are born as in-use (see MetaChunk ctor). So, before returning - // the padding chunk to its chunk manager, mark it as in use (ChunkManager - // will assert that). - do_update_in_use_info_for_chunk(padding_chunk, true); - - // Return Chunk to freelist. - inc_container_count(); - chunk_manager->return_single_chunk(padding_chunk); - // Please note: at this point, ChunkManager::return_single_chunk() - // may already have merged the padding chunk with neighboring chunks, so - // it may have vanished at this point. Do not reference the padding - // chunk beyond this point. - } - - assert(top() == target_top, "Sanity"); - -} // allocate_padding_chunks_until_top_is_at() - -// Allocates the chunk from the virtual space only. -// This interface is also used internally for debugging. Not all -// chunks removed here are necessarily used for allocation. -Metachunk* VirtualSpaceNode::take_from_committed(size_t chunk_word_size) { - // Non-humongous chunks are to be allocated aligned to their chunk - // size. So, start addresses of medium chunks are aligned to medium - // chunk size, those of small chunks to small chunk size and so - // forth. This facilitates merging of free chunks and reduces - // fragmentation. Chunk sizes are spec < small < medium, with each - // larger chunk size being a multiple of the next smaller chunk - // size. - // Because of this alignment, me may need to create a number of padding - // chunks. These chunks are created and added to the freelist. - - // The chunk manager to which we will give our padding chunks. - ChunkManager* const chunk_manager = Metaspace::get_chunk_manager(this->is_class()); - - // shorthands - const size_t spec_word_size = chunk_manager->specialized_chunk_word_size(); - const size_t small_word_size = chunk_manager->small_chunk_word_size(); - const size_t med_word_size = chunk_manager->medium_chunk_word_size(); - - assert(chunk_word_size == spec_word_size || chunk_word_size == small_word_size || - chunk_word_size >= med_word_size, "Invalid chunk size requested."); - - // Chunk alignment (in bytes) == chunk size unless humongous. - // Humongous chunks are aligned to the smallest chunk size (spec). - const size_t required_chunk_alignment = (chunk_word_size > med_word_size ? - spec_word_size : chunk_word_size) * sizeof(MetaWord); - - // Do we have enough space to create the requested chunk plus - // any padding chunks needed? - MetaWord* const next_aligned = - static_cast(align_up(top(), required_chunk_alignment)); - if (!is_available((next_aligned - top()) + chunk_word_size)) { - return NULL; - } - - // Before allocating the requested chunk, allocate padding chunks if necessary. - // We only need to do this for small or medium chunks: specialized chunks are the - // smallest size, hence always aligned. Homungous chunks are allocated unaligned - // (implicitly, also aligned to smallest chunk size). - if ((chunk_word_size == med_word_size || chunk_word_size == small_word_size) && next_aligned > top()) { - log_trace(gc, metaspace, freelist)("Creating padding chunks in %s between %p and %p...", - (is_class() ? "class space " : "metaspace"), - top(), next_aligned); - allocate_padding_chunks_until_top_is_at(next_aligned); - // Now, top should be aligned correctly. - assert_is_aligned(top(), required_chunk_alignment); - } - - // Now, top should be aligned correctly. - assert_is_aligned(top(), required_chunk_alignment); - - // Bottom of the new chunk - MetaWord* chunk_limit = top(); - assert(chunk_limit != NULL, "Not safe to call this method"); - - // The virtual spaces are always expanded by the - // commit granularity to enforce the following condition. - // Without this the is_available check will not work correctly. - assert(_virtual_space.committed_size() == _virtual_space.actual_committed_size(), - "The committed memory doesn't match the expanded memory."); - - if (!is_available(chunk_word_size)) { - LogTarget(Debug, gc, metaspace, freelist) lt; - if (lt.is_enabled()) { - LogStream ls(lt); - ls.print("VirtualSpaceNode::take_from_committed() not available " SIZE_FORMAT " words ", chunk_word_size); - // Dump some information about the virtual space that is nearly full - print_on(&ls); - } - return NULL; - } - - // Take the space (bump top on the current virtual space). - inc_top(chunk_word_size); - - // Initialize the chunk - ChunkIndex chunk_type = get_chunk_type_by_size(chunk_word_size, is_class()); - Metachunk* result = ::new (chunk_limit) Metachunk(chunk_type, is_class(), chunk_word_size, this); - assert(result == (Metachunk*)chunk_limit, "Sanity"); - occupancy_map()->set_chunk_starts_at_address((MetaWord*)result, true); - do_update_in_use_info_for_chunk(result, true); - - inc_container_count(); - - if (VerifyMetaspace) { - DEBUG_ONLY(chunk_manager->locked_verify()); - DEBUG_ONLY(this->verify()); - } - - DEBUG_ONLY(do_verify_chunk(result)); - - result->inc_use_count(); - - return result; -} - - -// Expand the virtual space (commit more of the reserved space) -bool VirtualSpaceNode::expand_by(size_t min_words, size_t preferred_words) { - size_t min_bytes = min_words * BytesPerWord; - size_t preferred_bytes = preferred_words * BytesPerWord; - - size_t uncommitted = virtual_space()->reserved_size() - virtual_space()->actual_committed_size(); - - if (uncommitted < min_bytes) { - return false; - } - - size_t commit = MIN2(preferred_bytes, uncommitted); - bool result = virtual_space()->expand_by(commit, false); - - if (result) { - log_trace(gc, metaspace, freelist)("Expanded %s virtual space list node by " SIZE_FORMAT " words.", - (is_class() ? "class" : "non-class"), commit); - DEBUG_ONLY(Atomic::inc(&g_internal_statistics.num_committed_space_expanded)); - } else { - log_trace(gc, metaspace, freelist)("Failed to expand %s virtual space list node by " SIZE_FORMAT " words.", - (is_class() ? "class" : "non-class"), commit); - } - - assert(result, "Failed to commit memory"); - - return result; -} - -Metachunk* VirtualSpaceNode::get_chunk_vs(size_t chunk_word_size) { - assert_lock_strong(MetaspaceExpand_lock); - Metachunk* result = take_from_committed(chunk_word_size); - return result; -} - -bool VirtualSpaceNode::initialize() { - - if (!_rs.is_reserved()) { - return false; - } - - // These are necessary restriction to make sure that the virtual space always - // grows in steps of Metaspace::commit_alignment(). If both base and size are - // aligned only the middle alignment of the VirtualSpace is used. - assert_is_aligned(_rs.base(), Metaspace::commit_alignment()); - assert_is_aligned(_rs.size(), Metaspace::commit_alignment()); - - // ReservedSpaces marked as special will have the entire memory - // pre-committed. Setting a committed size will make sure that - // committed_size and actual_committed_size agrees. - size_t pre_committed_size = _rs.special() ? _rs.size() : 0; - - bool result = virtual_space()->initialize_with_granularity(_rs, pre_committed_size, - Metaspace::commit_alignment()); - if (result) { - assert(virtual_space()->committed_size() == virtual_space()->actual_committed_size(), - "Checking that the pre-committed memory was registered by the VirtualSpace"); - - set_top((MetaWord*)virtual_space()->low()); - set_reserved(MemRegion((HeapWord*)_rs.base(), - (HeapWord*)(_rs.base() + _rs.size()))); - - assert(reserved()->start() == (HeapWord*) _rs.base(), - "Reserved start was not set properly " PTR_FORMAT - " != " PTR_FORMAT, p2i(reserved()->start()), p2i(_rs.base())); - assert(reserved()->word_size() == _rs.size() / BytesPerWord, - "Reserved size was not set properly " SIZE_FORMAT - " != " SIZE_FORMAT, reserved()->word_size(), - _rs.size() / BytesPerWord); - } - - // Initialize Occupancy Map. - const size_t smallest_chunk_size = is_class() ? ClassSpecializedChunk : SpecializedChunk; - _occupancy_map = new OccupancyMap(bottom(), reserved_words(), smallest_chunk_size); - - return result; -} - -void VirtualSpaceNode::print_on(outputStream* st, size_t scale) const { - size_t used_words = used_words_in_vs(); - size_t commit_words = committed_words(); - size_t res_words = reserved_words(); - VirtualSpace* vs = virtual_space(); - - st->print("node @" PTR_FORMAT ": ", p2i(this)); - st->print("reserved="); - print_scaled_words(st, res_words, scale); - st->print(", committed="); - print_scaled_words_and_percentage(st, commit_words, res_words, scale); - st->print(", used="); - print_scaled_words_and_percentage(st, used_words, res_words, scale); - st->cr(); - st->print(" [" PTR_FORMAT ", " PTR_FORMAT ", " - PTR_FORMAT ", " PTR_FORMAT ")", - p2i(bottom()), p2i(top()), p2i(end()), - p2i(vs->high_boundary())); -} - -#ifdef ASSERT -void VirtualSpaceNode::mangle() { - size_t word_size = capacity_words_in_vs(); - Copy::fill_to_words((HeapWord*) low(), word_size, 0xf1f1f1f1); -} -#endif // ASSERT - -// VirtualSpaceList methods -// Space allocated from the VirtualSpace - -VirtualSpaceList::~VirtualSpaceList() { - VirtualSpaceListIterator iter(virtual_space_list()); - while (iter.repeat()) { - VirtualSpaceNode* vsl = iter.get_next(); - delete vsl; - } -} - -void VirtualSpaceList::inc_reserved_words(size_t v) { - assert_lock_strong(MetaspaceExpand_lock); - _reserved_words = _reserved_words + v; -} -void VirtualSpaceList::dec_reserved_words(size_t v) { - assert_lock_strong(MetaspaceExpand_lock); - _reserved_words = _reserved_words - v; -} - -#define assert_committed_below_limit() \ - assert(MetaspaceUtils::committed_bytes() <= MaxMetaspaceSize, \ - "Too much committed memory. Committed: " SIZE_FORMAT \ - " limit (MaxMetaspaceSize): " SIZE_FORMAT, \ - MetaspaceUtils::committed_bytes(), MaxMetaspaceSize); - -void VirtualSpaceList::inc_committed_words(size_t v) { - assert_lock_strong(MetaspaceExpand_lock); - _committed_words = _committed_words + v; - - assert_committed_below_limit(); -} -void VirtualSpaceList::dec_committed_words(size_t v) { - assert_lock_strong(MetaspaceExpand_lock); - _committed_words = _committed_words - v; - - assert_committed_below_limit(); -} - -void VirtualSpaceList::inc_virtual_space_count() { - assert_lock_strong(MetaspaceExpand_lock); - _virtual_space_count++; -} -void VirtualSpaceList::dec_virtual_space_count() { - assert_lock_strong(MetaspaceExpand_lock); - _virtual_space_count--; -} - -void ChunkManager::remove_chunk(Metachunk* chunk) { - size_t word_size = chunk->word_size(); - ChunkIndex index = list_index(word_size); - if (index != HumongousIndex) { - free_chunks(index)->remove_chunk(chunk); - } else { - humongous_dictionary()->remove_chunk(chunk); - } - - // Chunk has been removed from the chunks free list, update counters. - account_for_removed_chunk(chunk); -} - -bool ChunkManager::attempt_to_coalesce_around_chunk(Metachunk* chunk, ChunkIndex target_chunk_type) { - assert_lock_strong(MetaspaceExpand_lock); - assert(chunk != NULL, "invalid chunk pointer"); - // Check for valid merge combinations. - assert((chunk->get_chunk_type() == SpecializedIndex && - (target_chunk_type == SmallIndex || target_chunk_type == MediumIndex)) || - (chunk->get_chunk_type() == SmallIndex && target_chunk_type == MediumIndex), - "Invalid chunk merge combination."); - - const size_t target_chunk_word_size = - get_size_for_nonhumongous_chunktype(target_chunk_type, this->is_class()); - - // [ prospective merge region ) - MetaWord* const p_merge_region_start = - (MetaWord*) align_down(chunk, target_chunk_word_size * sizeof(MetaWord)); - MetaWord* const p_merge_region_end = - p_merge_region_start + target_chunk_word_size; - - // We need the VirtualSpaceNode containing this chunk and its occupancy map. - VirtualSpaceNode* const vsn = chunk->container(); - OccupancyMap* const ocmap = vsn->occupancy_map(); - - // The prospective chunk merge range must be completely contained by the - // committed range of the virtual space node. - if (p_merge_region_start < vsn->bottom() || p_merge_region_end > vsn->top()) { - return false; - } - - // Only attempt to merge this range if at its start a chunk starts and at its end - // a chunk ends. If a chunk (can only be humongous) straddles either start or end - // of that range, we cannot merge. - if (!ocmap->chunk_starts_at_address(p_merge_region_start)) { - return false; - } - if (p_merge_region_end < vsn->top() && - !ocmap->chunk_starts_at_address(p_merge_region_end)) { - return false; - } - - // Now check if the prospective merge area contains live chunks. If it does we cannot merge. - if (ocmap->is_region_in_use(p_merge_region_start, target_chunk_word_size)) { - return false; - } - - // Success! Remove all chunks in this region... - log_trace(gc, metaspace, freelist)("%s: coalescing chunks in area [%p-%p)...", - (is_class() ? "class space" : "metaspace"), - p_merge_region_start, p_merge_region_end); - - const int num_chunks_removed = - remove_chunks_in_area(p_merge_region_start, target_chunk_word_size); - - // ... and create a single new bigger chunk. - Metachunk* const p_new_chunk = - ::new (p_merge_region_start) Metachunk(target_chunk_type, is_class(), target_chunk_word_size, vsn); - assert(p_new_chunk == (Metachunk*)p_merge_region_start, "Sanity"); - p_new_chunk->set_origin(origin_merge); - - log_trace(gc, metaspace, freelist)("%s: created coalesced chunk at %p, size " SIZE_FORMAT_HEX ".", - (is_class() ? "class space" : "metaspace"), - p_new_chunk, p_new_chunk->word_size() * sizeof(MetaWord)); - - // Fix occupancy map: remove old start bits of the small chunks and set new start bit. - ocmap->wipe_chunk_start_bits_in_region(p_merge_region_start, target_chunk_word_size); - ocmap->set_chunk_starts_at_address(p_merge_region_start, true); - - // Mark chunk as free. Note: it is not necessary to update the occupancy - // map in-use map, because the old chunks were also free, so nothing - // should have changed. - p_new_chunk->set_is_tagged_free(true); - - // Add new chunk to its freelist. - ChunkList* const list = free_chunks(target_chunk_type); - list->return_chunk_at_head(p_new_chunk); - - // And adjust ChunkManager:: _free_chunks_count (_free_chunks_total - // should not have changed, because the size of the space should be the same) - _free_chunks_count -= num_chunks_removed; - _free_chunks_count ++; - - // VirtualSpaceNode::container_count does not have to be modified: - // it means "number of active (non-free) chunks", so merging free chunks - // should not affect that count. - - // At the end of a chunk merge, run verification tests. - if (VerifyMetaspace) { - DEBUG_ONLY(this->locked_verify()); - DEBUG_ONLY(vsn->verify()); - } - - return true; -} - -// Remove all chunks in the given area - the chunks are supposed to be free - -// from their corresponding freelists. Mark them as invalid. -// - This does not correct the occupancy map. -// - This does not adjust the counters in ChunkManager. -// - Does not adjust container count counter in containing VirtualSpaceNode -// Returns number of chunks removed. -int ChunkManager::remove_chunks_in_area(MetaWord* p, size_t word_size) { - assert(p != NULL && word_size > 0, "Invalid range."); - const size_t smallest_chunk_size = get_size_for_nonhumongous_chunktype(SpecializedIndex, is_class()); - assert_is_aligned(word_size, smallest_chunk_size); - - Metachunk* const start = (Metachunk*) p; - const Metachunk* const end = (Metachunk*)(p + word_size); - Metachunk* cur = start; - int num_removed = 0; - while (cur < end) { - Metachunk* next = (Metachunk*)(((MetaWord*)cur) + cur->word_size()); - DEBUG_ONLY(do_verify_chunk(cur)); - assert(cur->get_chunk_type() != HumongousIndex, "Unexpected humongous chunk found at %p.", cur); - assert(cur->is_tagged_free(), "Chunk expected to be free (%p)", cur); - log_trace(gc, metaspace, freelist)("%s: removing chunk %p, size " SIZE_FORMAT_HEX ".", - (is_class() ? "class space" : "metaspace"), - cur, cur->word_size() * sizeof(MetaWord)); - cur->remove_sentinel(); - // Note: cannot call ChunkManager::remove_chunk, because that - // modifies the counters in ChunkManager, which we do not want. So - // we call remove_chunk on the freelist directly (see also the - // splitting function which does the same). - ChunkList* const list = free_chunks(list_index(cur->word_size())); - list->remove_chunk(cur); - num_removed ++; - cur = next; - } - return num_removed; -} - -// Walk the list of VirtualSpaceNodes and delete -// nodes with a 0 container_count. Remove Metachunks in -// the node from their respective freelists. -void VirtualSpaceList::purge(ChunkManager* chunk_manager) { - assert(SafepointSynchronize::is_at_safepoint(), "must be called at safepoint for contains to work"); - assert_lock_strong(MetaspaceExpand_lock); - // Don't use a VirtualSpaceListIterator because this - // list is being changed and a straightforward use of an iterator is not safe. - VirtualSpaceNode* purged_vsl = NULL; - VirtualSpaceNode* prev_vsl = virtual_space_list(); - VirtualSpaceNode* next_vsl = prev_vsl; - while (next_vsl != NULL) { - VirtualSpaceNode* vsl = next_vsl; - DEBUG_ONLY(vsl->verify_container_count();) - next_vsl = vsl->next(); - // Don't free the current virtual space since it will likely - // be needed soon. - if (vsl->container_count() == 0 && vsl != current_virtual_space()) { - log_trace(gc, metaspace, freelist)("Purging VirtualSpaceNode " PTR_FORMAT " (capacity: " SIZE_FORMAT - ", used: " SIZE_FORMAT ").", p2i(vsl), vsl->capacity_words_in_vs(), vsl->used_words_in_vs()); - DEBUG_ONLY(Atomic::inc(&g_internal_statistics.num_vsnodes_purged)); - // Unlink it from the list - if (prev_vsl == vsl) { - // This is the case of the current node being the first node. - assert(vsl == virtual_space_list(), "Expected to be the first node"); - set_virtual_space_list(vsl->next()); - } else { - prev_vsl->set_next(vsl->next()); - } - - vsl->purge(chunk_manager); - dec_reserved_words(vsl->reserved_words()); - dec_committed_words(vsl->committed_words()); - dec_virtual_space_count(); - purged_vsl = vsl; - delete vsl; - } else { - prev_vsl = vsl; - } - } -#ifdef ASSERT - if (purged_vsl != NULL) { - // List should be stable enough to use an iterator here. - VirtualSpaceListIterator iter(virtual_space_list()); - while (iter.repeat()) { - VirtualSpaceNode* vsl = iter.get_next(); - assert(vsl != purged_vsl, "Purge of vsl failed"); - } - } -#endif -} - - -// This function looks at the mmap regions in the metaspace without locking. -// The chunks are added with store ordering and not deleted except for at -// unloading time during a safepoint. -bool VirtualSpaceList::contains(const void* ptr) { - // List should be stable enough to use an iterator here because removing virtual - // space nodes is only allowed at a safepoint. - VirtualSpaceListIterator iter(virtual_space_list()); - while (iter.repeat()) { - VirtualSpaceNode* vsn = iter.get_next(); - if (vsn->contains(ptr)) { - return true; - } - } - return false; -} - -void VirtualSpaceList::retire_current_virtual_space() { - assert_lock_strong(MetaspaceExpand_lock); - - VirtualSpaceNode* vsn = current_virtual_space(); - - ChunkManager* cm = is_class() ? Metaspace::chunk_manager_class() : - Metaspace::chunk_manager_metadata(); - - vsn->retire(cm); -} - -void VirtualSpaceNode::retire(ChunkManager* chunk_manager) { - DEBUG_ONLY(verify_container_count();) - assert(this->is_class() == chunk_manager->is_class(), "Wrong ChunkManager?"); - for (int i = (int)MediumIndex; i >= (int)ZeroIndex; --i) { - ChunkIndex index = (ChunkIndex)i; - size_t chunk_size = chunk_manager->size_by_index(index); - - while (free_words_in_vs() >= chunk_size) { - Metachunk* chunk = get_chunk_vs(chunk_size); - // Chunk will be allocated aligned, so allocation may require - // additional padding chunks. That may cause above allocation to - // fail. Just ignore the failed allocation and continue with the - // next smaller chunk size. As the VirtualSpaceNode comitted - // size should be a multiple of the smallest chunk size, we - // should always be able to fill the VirtualSpace completely. - if (chunk == NULL) { - break; - } - chunk_manager->return_single_chunk(chunk); - } - DEBUG_ONLY(verify_container_count();) - } - assert(free_words_in_vs() == 0, "should be empty now"); -} - -VirtualSpaceList::VirtualSpaceList(size_t word_size) : - _is_class(false), - _virtual_space_list(NULL), - _current_virtual_space(NULL), - _reserved_words(0), - _committed_words(0), - _virtual_space_count(0) { - MutexLockerEx cl(MetaspaceExpand_lock, - Mutex::_no_safepoint_check_flag); - create_new_virtual_space(word_size); -} - -VirtualSpaceList::VirtualSpaceList(ReservedSpace rs) : - _is_class(true), - _virtual_space_list(NULL), - _current_virtual_space(NULL), - _reserved_words(0), - _committed_words(0), - _virtual_space_count(0) { - MutexLockerEx cl(MetaspaceExpand_lock, - Mutex::_no_safepoint_check_flag); - VirtualSpaceNode* class_entry = new VirtualSpaceNode(is_class(), rs); - bool succeeded = class_entry->initialize(); - if (succeeded) { - link_vs(class_entry); - } -} - -size_t VirtualSpaceList::free_bytes() { - return current_virtual_space()->free_words_in_vs() * BytesPerWord; -} - -// Allocate another meta virtual space and add it to the list. -bool VirtualSpaceList::create_new_virtual_space(size_t vs_word_size) { - assert_lock_strong(MetaspaceExpand_lock); - - if (is_class()) { - assert(false, "We currently don't support more than one VirtualSpace for" - " the compressed class space. The initialization of the" - " CCS uses another code path and should not hit this path."); - return false; - } - - if (vs_word_size == 0) { - assert(false, "vs_word_size should always be at least _reserve_alignment large."); - return false; - } - - // Reserve the space - size_t vs_byte_size = vs_word_size * BytesPerWord; - assert_is_aligned(vs_byte_size, Metaspace::reserve_alignment()); - - // Allocate the meta virtual space and initialize it. - VirtualSpaceNode* new_entry = new VirtualSpaceNode(is_class(), vs_byte_size); - if (!new_entry->initialize()) { - delete new_entry; - return false; - } else { - assert(new_entry->reserved_words() == vs_word_size, - "Reserved memory size differs from requested memory size"); - // ensure lock-free iteration sees fully initialized node - OrderAccess::storestore(); - link_vs(new_entry); - DEBUG_ONLY(Atomic::inc(&g_internal_statistics.num_vsnodes_created)); - return true; - } -} - -void VirtualSpaceList::link_vs(VirtualSpaceNode* new_entry) { - if (virtual_space_list() == NULL) { - set_virtual_space_list(new_entry); - } else { - current_virtual_space()->set_next(new_entry); - } - set_current_virtual_space(new_entry); - inc_reserved_words(new_entry->reserved_words()); - inc_committed_words(new_entry->committed_words()); - inc_virtual_space_count(); -#ifdef ASSERT - new_entry->mangle(); -#endif - LogTarget(Trace, gc, metaspace) lt; - if (lt.is_enabled()) { - LogStream ls(lt); - VirtualSpaceNode* vsl = current_virtual_space(); - ResourceMark rm; - vsl->print_on(&ls); - } -} - -bool VirtualSpaceList::expand_node_by(VirtualSpaceNode* node, - size_t min_words, - size_t preferred_words) { - size_t before = node->committed_words(); - - bool result = node->expand_by(min_words, preferred_words); - - size_t after = node->committed_words(); - - // after and before can be the same if the memory was pre-committed. - assert(after >= before, "Inconsistency"); - inc_committed_words(after - before); - - return result; -} - -bool VirtualSpaceList::expand_by(size_t min_words, size_t preferred_words) { - assert_is_aligned(min_words, Metaspace::commit_alignment_words()); - assert_is_aligned(preferred_words, Metaspace::commit_alignment_words()); - assert(min_words <= preferred_words, "Invalid arguments"); - - const char* const class_or_not = (is_class() ? "class" : "non-class"); - - if (!MetaspaceGC::can_expand(min_words, this->is_class())) { - log_trace(gc, metaspace, freelist)("Cannot expand %s virtual space list.", - class_or_not); - return false; - } - - size_t allowed_expansion_words = MetaspaceGC::allowed_expansion(); - if (allowed_expansion_words < min_words) { - log_trace(gc, metaspace, freelist)("Cannot expand %s virtual space list (must try gc first).", - class_or_not); - return false; - } - - size_t max_expansion_words = MIN2(preferred_words, allowed_expansion_words); - - // Commit more memory from the the current virtual space. - bool vs_expanded = expand_node_by(current_virtual_space(), - min_words, - max_expansion_words); - if (vs_expanded) { - log_trace(gc, metaspace, freelist)("Expanded %s virtual space list.", - class_or_not); - return true; - } - log_trace(gc, metaspace, freelist)("%s virtual space list: retire current node.", - class_or_not); - retire_current_virtual_space(); - - // Get another virtual space. - size_t grow_vs_words = MAX2((size_t)VirtualSpaceSize, preferred_words); - grow_vs_words = align_up(grow_vs_words, Metaspace::reserve_alignment_words()); - - if (create_new_virtual_space(grow_vs_words)) { - if (current_virtual_space()->is_pre_committed()) { - // The memory was pre-committed, so we are done here. - assert(min_words <= current_virtual_space()->committed_words(), - "The new VirtualSpace was pre-committed, so it" - "should be large enough to fit the alloc request."); - return true; - } - - return expand_node_by(current_virtual_space(), - min_words, - max_expansion_words); - } - - return false; -} - -// Given a chunk, calculate the largest possible padding space which -// could be required when allocating it. -static size_t largest_possible_padding_size_for_chunk(size_t chunk_word_size, bool is_class) { - const ChunkIndex chunk_type = get_chunk_type_by_size(chunk_word_size, is_class); - if (chunk_type != HumongousIndex) { - // Normal, non-humongous chunks are allocated at chunk size - // boundaries, so the largest padding space required would be that - // minus the smallest chunk size. - const size_t smallest_chunk_size = is_class ? ClassSpecializedChunk : SpecializedChunk; - return chunk_word_size - smallest_chunk_size; - } else { - // Humongous chunks are allocated at smallest-chunksize - // boundaries, so there is no padding required. - return 0; - } -} - - -Metachunk* VirtualSpaceList::get_new_chunk(size_t chunk_word_size, size_t suggested_commit_granularity) { - - // Allocate a chunk out of the current virtual space. - Metachunk* next = current_virtual_space()->get_chunk_vs(chunk_word_size); - - if (next != NULL) { - return next; - } - - // The expand amount is currently only determined by the requested sizes - // and not how much committed memory is left in the current virtual space. - - // We must have enough space for the requested size and any - // additional reqired padding chunks. - const size_t size_for_padding = largest_possible_padding_size_for_chunk(chunk_word_size, this->is_class()); - - size_t min_word_size = align_up(chunk_word_size + size_for_padding, Metaspace::commit_alignment_words()); - size_t preferred_word_size = align_up(suggested_commit_granularity, Metaspace::commit_alignment_words()); - if (min_word_size >= preferred_word_size) { - // Can happen when humongous chunks are allocated. - preferred_word_size = min_word_size; - } - - bool expanded = expand_by(min_word_size, preferred_word_size); - if (expanded) { - next = current_virtual_space()->get_chunk_vs(chunk_word_size); - assert(next != NULL, "The allocation was expected to succeed after the expansion"); - } - - return next; -} - -void VirtualSpaceList::print_on(outputStream* st, size_t scale) const { - st->print_cr(SIZE_FORMAT " nodes, current node: " PTR_FORMAT, - _virtual_space_count, p2i(_current_virtual_space)); - VirtualSpaceListIterator iter(virtual_space_list()); - while (iter.repeat()) { - st->cr(); - VirtualSpaceNode* node = iter.get_next(); - node->print_on(st, scale); - } -} - -void VirtualSpaceList::print_map(outputStream* st) const { - VirtualSpaceNode* list = virtual_space_list(); - VirtualSpaceListIterator iter(list); - unsigned i = 0; - while (iter.repeat()) { - st->print_cr("Node %u:", i); - VirtualSpaceNode* node = iter.get_next(); - node->print_map(st, this->is_class()); - i ++; - } -} - // MetaspaceGC methods // VM_CollectForMetadataAllocation is the vm operation used to GC. @@ -2615,972 +330,6 @@ } } -// Metadebug methods - -void Metadebug::init_allocation_fail_alot_count() { - if (MetadataAllocationFailALot) { - _allocation_fail_alot_count = - 1+(long)((double)MetadataAllocationFailALotInterval*os::random()/(max_jint+1.0)); - } -} - -#ifdef ASSERT -bool Metadebug::test_metadata_failure() { - if (MetadataAllocationFailALot && - Threads::is_vm_complete()) { - if (_allocation_fail_alot_count > 0) { - _allocation_fail_alot_count--; - } else { - log_trace(gc, metaspace, freelist)("Metadata allocation failing for MetadataAllocationFailALot"); - init_allocation_fail_alot_count(); - return true; - } - } - return false; -} -#endif - -// ChunkManager methods -size_t ChunkManager::free_chunks_total_words() { - return _free_chunks_total; -} - -size_t ChunkManager::free_chunks_total_bytes() { - return free_chunks_total_words() * BytesPerWord; -} - -// Update internal accounting after a chunk was added -void ChunkManager::account_for_added_chunk(const Metachunk* c) { - assert_lock_strong(MetaspaceExpand_lock); - _free_chunks_count ++; - _free_chunks_total += c->word_size(); -} - -// Update internal accounting after a chunk was removed -void ChunkManager::account_for_removed_chunk(const Metachunk* c) { - assert_lock_strong(MetaspaceExpand_lock); - assert(_free_chunks_count >= 1, - "ChunkManager::_free_chunks_count: about to go negative (" SIZE_FORMAT ").", _free_chunks_count); - assert(_free_chunks_total >= c->word_size(), - "ChunkManager::_free_chunks_total: about to go negative" - "(now: " SIZE_FORMAT ", decrement value: " SIZE_FORMAT ").", _free_chunks_total, c->word_size()); - _free_chunks_count --; - _free_chunks_total -= c->word_size(); -} - -size_t ChunkManager::free_chunks_count() { -#ifdef ASSERT - if (!UseConcMarkSweepGC && !MetaspaceExpand_lock->is_locked()) { - MutexLockerEx cl(MetaspaceExpand_lock, - Mutex::_no_safepoint_check_flag); - // This lock is only needed in debug because the verification - // of the _free_chunks_totals walks the list of free chunks - slow_locked_verify_free_chunks_count(); - } -#endif - return _free_chunks_count; -} - -ChunkIndex ChunkManager::list_index(size_t size) { - return get_chunk_type_by_size(size, is_class()); -} - -size_t ChunkManager::size_by_index(ChunkIndex index) const { - index_bounds_check(index); - assert(index != HumongousIndex, "Do not call for humongous chunks."); - return get_size_for_nonhumongous_chunktype(index, is_class()); -} - -void ChunkManager::locked_verify_free_chunks_total() { - assert_lock_strong(MetaspaceExpand_lock); - assert(sum_free_chunks() == _free_chunks_total, - "_free_chunks_total " SIZE_FORMAT " is not the" - " same as sum " SIZE_FORMAT, _free_chunks_total, - sum_free_chunks()); -} - -void ChunkManager::locked_verify_free_chunks_count() { - assert_lock_strong(MetaspaceExpand_lock); - assert(sum_free_chunks_count() == _free_chunks_count, - "_free_chunks_count " SIZE_FORMAT " is not the" - " same as sum " SIZE_FORMAT, _free_chunks_count, - sum_free_chunks_count()); -} - -void ChunkManager::verify() { - MutexLockerEx cl(MetaspaceExpand_lock, - Mutex::_no_safepoint_check_flag); - locked_verify(); -} - -void ChunkManager::locked_verify() { - locked_verify_free_chunks_count(); - locked_verify_free_chunks_total(); - for (ChunkIndex i = ZeroIndex; i < NumberOfFreeLists; i = next_chunk_index(i)) { - ChunkList* list = free_chunks(i); - if (list != NULL) { - Metachunk* chunk = list->head(); - while (chunk) { - DEBUG_ONLY(do_verify_chunk(chunk);) - assert(chunk->is_tagged_free(), "Chunk should be tagged as free."); - chunk = chunk->next(); - } - } - } -} - -void ChunkManager::locked_print_free_chunks(outputStream* st) { - assert_lock_strong(MetaspaceExpand_lock); - st->print_cr("Free chunk total " SIZE_FORMAT " count " SIZE_FORMAT, - _free_chunks_total, _free_chunks_count); -} - -void ChunkManager::locked_print_sum_free_chunks(outputStream* st) { - assert_lock_strong(MetaspaceExpand_lock); - st->print_cr("Sum free chunk total " SIZE_FORMAT " count " SIZE_FORMAT, - sum_free_chunks(), sum_free_chunks_count()); -} - -ChunkList* ChunkManager::free_chunks(ChunkIndex index) { - assert(index == SpecializedIndex || index == SmallIndex || index == MediumIndex, - "Bad index: %d", (int)index); - - return &_free_chunks[index]; -} - -// These methods that sum the free chunk lists are used in printing -// methods that are used in product builds. -size_t ChunkManager::sum_free_chunks() { - assert_lock_strong(MetaspaceExpand_lock); - size_t result = 0; - for (ChunkIndex i = ZeroIndex; i < NumberOfFreeLists; i = next_chunk_index(i)) { - ChunkList* list = free_chunks(i); - - if (list == NULL) { - continue; - } - - result = result + list->count() * list->size(); - } - result = result + humongous_dictionary()->total_size(); - return result; -} - -size_t ChunkManager::sum_free_chunks_count() { - assert_lock_strong(MetaspaceExpand_lock); - size_t count = 0; - for (ChunkIndex i = ZeroIndex; i < NumberOfFreeLists; i = next_chunk_index(i)) { - ChunkList* list = free_chunks(i); - if (list == NULL) { - continue; - } - count = count + list->count(); - } - count = count + humongous_dictionary()->total_free_blocks(); - return count; -} - -ChunkList* ChunkManager::find_free_chunks_list(size_t word_size) { - ChunkIndex index = list_index(word_size); - assert(index < HumongousIndex, "No humongous list"); - return free_chunks(index); -} - -// Helper for chunk splitting: given a target chunk size and a larger free chunk, -// split up the larger chunk into n smaller chunks, at least one of which should be -// the target chunk of target chunk size. The smaller chunks, including the target -// chunk, are returned to the freelist. The pointer to the target chunk is returned. -// Note that this chunk is supposed to be removed from the freelist right away. -Metachunk* ChunkManager::split_chunk(size_t target_chunk_word_size, Metachunk* larger_chunk) { - assert(larger_chunk->word_size() > target_chunk_word_size, "Sanity"); - - const ChunkIndex larger_chunk_index = larger_chunk->get_chunk_type(); - const ChunkIndex target_chunk_index = get_chunk_type_by_size(target_chunk_word_size, is_class()); - - MetaWord* const region_start = (MetaWord*)larger_chunk; - const size_t region_word_len = larger_chunk->word_size(); - MetaWord* const region_end = region_start + region_word_len; - VirtualSpaceNode* const vsn = larger_chunk->container(); - OccupancyMap* const ocmap = vsn->occupancy_map(); - - // Any larger non-humongous chunk size is a multiple of any smaller chunk size. - // Since non-humongous chunks are aligned to their chunk size, the larger chunk should start - // at an address suitable to place the smaller target chunk. - assert_is_aligned(region_start, target_chunk_word_size); - - // Remove old chunk. - free_chunks(larger_chunk_index)->remove_chunk(larger_chunk); - larger_chunk->remove_sentinel(); - - // Prevent access to the old chunk from here on. - larger_chunk = NULL; - // ... and wipe it. - DEBUG_ONLY(memset(region_start, 0xfe, region_word_len * BytesPerWord)); - - // In its place create first the target chunk... - MetaWord* p = region_start; - Metachunk* target_chunk = ::new (p) Metachunk(target_chunk_index, is_class(), target_chunk_word_size, vsn); - assert(target_chunk == (Metachunk*)p, "Sanity"); - target_chunk->set_origin(origin_split); - - // Note: we do not need to mark its start in the occupancy map - // because it coincides with the old chunk start. - - // Mark chunk as free and return to the freelist. - do_update_in_use_info_for_chunk(target_chunk, false); - free_chunks(target_chunk_index)->return_chunk_at_head(target_chunk); - - // This chunk should now be valid and can be verified. - DEBUG_ONLY(do_verify_chunk(target_chunk)); - - // In the remaining space create the remainder chunks. - p += target_chunk->word_size(); - assert(p < region_end, "Sanity"); - - while (p < region_end) { - - // Find the largest chunk size which fits the alignment requirements at address p. - ChunkIndex this_chunk_index = prev_chunk_index(larger_chunk_index); - size_t this_chunk_word_size = 0; - for(;;) { - this_chunk_word_size = get_size_for_nonhumongous_chunktype(this_chunk_index, is_class()); - if (is_aligned(p, this_chunk_word_size * BytesPerWord)) { - break; - } else { - this_chunk_index = prev_chunk_index(this_chunk_index); - assert(this_chunk_index >= target_chunk_index, "Sanity"); - } - } - - assert(this_chunk_word_size >= target_chunk_word_size, "Sanity"); - assert(is_aligned(p, this_chunk_word_size * BytesPerWord), "Sanity"); - assert(p + this_chunk_word_size <= region_end, "Sanity"); - - // Create splitting chunk. - Metachunk* this_chunk = ::new (p) Metachunk(this_chunk_index, is_class(), this_chunk_word_size, vsn); - assert(this_chunk == (Metachunk*)p, "Sanity"); - this_chunk->set_origin(origin_split); - ocmap->set_chunk_starts_at_address(p, true); - do_update_in_use_info_for_chunk(this_chunk, false); - - // This chunk should be valid and can be verified. - DEBUG_ONLY(do_verify_chunk(this_chunk)); - - // Return this chunk to freelist and correct counter. - free_chunks(this_chunk_index)->return_chunk_at_head(this_chunk); - _free_chunks_count ++; - - log_trace(gc, metaspace, freelist)("Created chunk at " PTR_FORMAT ", word size " - SIZE_FORMAT_HEX " (%s), in split region [" PTR_FORMAT "..." PTR_FORMAT ").", - p2i(this_chunk), this_chunk->word_size(), chunk_size_name(this_chunk_index), - p2i(region_start), p2i(region_end)); - - p += this_chunk_word_size; - - } - - return target_chunk; -} - -Metachunk* ChunkManager::free_chunks_get(size_t word_size) { - assert_lock_strong(MetaspaceExpand_lock); - - slow_locked_verify(); - - Metachunk* chunk = NULL; - bool we_did_split_a_chunk = false; - - if (list_index(word_size) != HumongousIndex) { - - ChunkList* free_list = find_free_chunks_list(word_size); - assert(free_list != NULL, "Sanity check"); - - chunk = free_list->head(); - - if (chunk == NULL) { - // Split large chunks into smaller chunks if there are no smaller chunks, just large chunks. - // This is the counterpart of the coalescing-upon-chunk-return. - - ChunkIndex target_chunk_index = get_chunk_type_by_size(word_size, is_class()); - - // Is there a larger chunk we could split? - Metachunk* larger_chunk = NULL; - ChunkIndex larger_chunk_index = next_chunk_index(target_chunk_index); - while (larger_chunk == NULL && larger_chunk_index < NumberOfFreeLists) { - larger_chunk = free_chunks(larger_chunk_index)->head(); - if (larger_chunk == NULL) { - larger_chunk_index = next_chunk_index(larger_chunk_index); - } - } - - if (larger_chunk != NULL) { - assert(larger_chunk->word_size() > word_size, "Sanity"); - assert(larger_chunk->get_chunk_type() == larger_chunk_index, "Sanity"); - - // We found a larger chunk. Lets split it up: - // - remove old chunk - // - in its place, create new smaller chunks, with at least one chunk - // being of target size, the others sized as large as possible. This - // is to make sure the resulting chunks are "as coalesced as possible" - // (similar to VirtualSpaceNode::retire()). - // Note: during this operation both ChunkManager and VirtualSpaceNode - // are temporarily invalid, so be careful with asserts. - - log_trace(gc, metaspace, freelist)("%s: splitting chunk " PTR_FORMAT - ", word size " SIZE_FORMAT_HEX " (%s), to get a chunk of word size " SIZE_FORMAT_HEX " (%s)...", - (is_class() ? "class space" : "metaspace"), p2i(larger_chunk), larger_chunk->word_size(), - chunk_size_name(larger_chunk_index), word_size, chunk_size_name(target_chunk_index)); - - chunk = split_chunk(word_size, larger_chunk); - - // This should have worked. - assert(chunk != NULL, "Sanity"); - assert(chunk->word_size() == word_size, "Sanity"); - assert(chunk->is_tagged_free(), "Sanity"); - - we_did_split_a_chunk = true; - - } - } - - if (chunk == NULL) { - return NULL; - } - - // Remove the chunk as the head of the list. - free_list->remove_chunk(chunk); - - log_trace(gc, metaspace, freelist)("ChunkManager::free_chunks_get: free_list: " PTR_FORMAT " chunks left: " SSIZE_FORMAT ".", - p2i(free_list), free_list->count()); - - } else { - chunk = humongous_dictionary()->get_chunk(word_size); - - if (chunk == NULL) { - return NULL; - } - - log_debug(gc, metaspace, alloc)("Free list allocate humongous chunk size " SIZE_FORMAT " for requested size " SIZE_FORMAT " waste " SIZE_FORMAT, - chunk->word_size(), word_size, chunk->word_size() - word_size); - } - - // Chunk has been removed from the chunk manager; update counters. - account_for_removed_chunk(chunk); - do_update_in_use_info_for_chunk(chunk, true); - chunk->container()->inc_container_count(); - chunk->inc_use_count(); - - // Remove it from the links to this freelist - chunk->set_next(NULL); - chunk->set_prev(NULL); - - // Run some verifications (some more if we did a chunk split) -#ifdef ASSERT - if (VerifyMetaspace) { - locked_verify(); - VirtualSpaceNode* const vsn = chunk->container(); - vsn->verify(); - if (we_did_split_a_chunk) { - vsn->verify_free_chunks_are_ideally_merged(); - } - } -#endif - - return chunk; -} - -Metachunk* ChunkManager::chunk_freelist_allocate(size_t word_size) { - assert_lock_strong(MetaspaceExpand_lock); - slow_locked_verify(); - - // Take from the beginning of the list - Metachunk* chunk = free_chunks_get(word_size); - if (chunk == NULL) { - return NULL; - } - - assert((word_size <= chunk->word_size()) || - (list_index(chunk->word_size()) == HumongousIndex), - "Non-humongous variable sized chunk"); - LogTarget(Debug, gc, metaspace, freelist) lt; - if (lt.is_enabled()) { - size_t list_count; - if (list_index(word_size) < HumongousIndex) { - ChunkList* list = find_free_chunks_list(word_size); - list_count = list->count(); - } else { - list_count = humongous_dictionary()->total_count(); - } - LogStream ls(lt); - ls.print("ChunkManager::chunk_freelist_allocate: " PTR_FORMAT " chunk " PTR_FORMAT " size " SIZE_FORMAT " count " SIZE_FORMAT " ", - p2i(this), p2i(chunk), chunk->word_size(), list_count); - ResourceMark rm; - locked_print_free_chunks(&ls); - } - - return chunk; -} - -void ChunkManager::return_single_chunk(Metachunk* chunk) { - const ChunkIndex index = chunk->get_chunk_type(); - assert_lock_strong(MetaspaceExpand_lock); - DEBUG_ONLY(do_verify_chunk(chunk);) - assert(chunk != NULL, "Expected chunk."); - assert(chunk->container() != NULL, "Container should have been set."); - assert(chunk->is_tagged_free() == false, "Chunk should be in use."); - index_bounds_check(index); - - // Note: mangle *before* returning the chunk to the freelist or dictionary. It does not - // matter for the freelist (non-humongous chunks), but the humongous chunk dictionary - // keeps tree node pointers in the chunk payload area which mangle will overwrite. - DEBUG_ONLY(chunk->mangle(badMetaWordVal);) - - if (index != HumongousIndex) { - // Return non-humongous chunk to freelist. - ChunkList* list = free_chunks(index); - assert(list->size() == chunk->word_size(), "Wrong chunk type."); - list->return_chunk_at_head(chunk); - log_trace(gc, metaspace, freelist)("returned one %s chunk at " PTR_FORMAT " to freelist.", - chunk_size_name(index), p2i(chunk)); - } else { - // Return humongous chunk to dictionary. - assert(chunk->word_size() > free_chunks(MediumIndex)->size(), "Wrong chunk type."); - assert(chunk->word_size() % free_chunks(SpecializedIndex)->size() == 0, - "Humongous chunk has wrong alignment."); - _humongous_dictionary.return_chunk(chunk); - log_trace(gc, metaspace, freelist)("returned one %s chunk at " PTR_FORMAT " (word size " SIZE_FORMAT ") to freelist.", - chunk_size_name(index), p2i(chunk), chunk->word_size()); - } - chunk->container()->dec_container_count(); - do_update_in_use_info_for_chunk(chunk, false); - - // Chunk has been added; update counters. - account_for_added_chunk(chunk); - - // Attempt coalesce returned chunks with its neighboring chunks: - // if this chunk is small or special, attempt to coalesce to a medium chunk. - if (index == SmallIndex || index == SpecializedIndex) { - if (!attempt_to_coalesce_around_chunk(chunk, MediumIndex)) { - // This did not work. But if this chunk is special, we still may form a small chunk? - if (index == SpecializedIndex) { - if (!attempt_to_coalesce_around_chunk(chunk, SmallIndex)) { - // give up. - } - } - } - } - -} - -void ChunkManager::return_chunk_list(Metachunk* chunks) { - if (chunks == NULL) { - return; - } - LogTarget(Trace, gc, metaspace, freelist) log; - if (log.is_enabled()) { // tracing - log.print("returning list of chunks..."); - } - unsigned num_chunks_returned = 0; - size_t size_chunks_returned = 0; - Metachunk* cur = chunks; - while (cur != NULL) { - // Capture the next link before it is changed - // by the call to return_chunk_at_head(); - Metachunk* next = cur->next(); - if (log.is_enabled()) { // tracing - num_chunks_returned ++; - size_chunks_returned += cur->word_size(); - } - return_single_chunk(cur); - cur = next; - } - if (log.is_enabled()) { // tracing - log.print("returned %u chunks to freelist, total word size " SIZE_FORMAT ".", - num_chunks_returned, size_chunks_returned); - } -} - -void ChunkManager::collect_statistics(ChunkManagerStatistics* out) const { - MutexLockerEx cl(MetaspaceExpand_lock, Mutex::_no_safepoint_check_flag); - for (ChunkIndex i = ZeroIndex; i < NumberOfInUseLists; i = next_chunk_index(i)) { - out->chunk_stats(i).add(num_free_chunks(i), size_free_chunks_in_bytes(i) / sizeof(MetaWord)); - } -} - -// SpaceManager methods - -size_t SpaceManager::adjust_initial_chunk_size(size_t requested, bool is_class_space) { - size_t chunk_sizes[] = { - specialized_chunk_size(is_class_space), - small_chunk_size(is_class_space), - medium_chunk_size(is_class_space) - }; - - // Adjust up to one of the fixed chunk sizes ... - for (size_t i = 0; i < ARRAY_SIZE(chunk_sizes); i++) { - if (requested <= chunk_sizes[i]) { - return chunk_sizes[i]; - } - } - - // ... or return the size as a humongous chunk. - return requested; -} - -size_t SpaceManager::adjust_initial_chunk_size(size_t requested) const { - return adjust_initial_chunk_size(requested, is_class()); -} - -size_t SpaceManager::get_initial_chunk_size(Metaspace::MetaspaceType type) const { - size_t requested; - - if (is_class()) { - switch (type) { - case Metaspace::BootMetaspaceType: requested = Metaspace::first_class_chunk_word_size(); break; - case Metaspace::AnonymousMetaspaceType: requested = ClassSpecializedChunk; break; - case Metaspace::ReflectionMetaspaceType: requested = ClassSpecializedChunk; break; - default: requested = ClassSmallChunk; break; - } - } else { - switch (type) { - case Metaspace::BootMetaspaceType: requested = Metaspace::first_chunk_word_size(); break; - case Metaspace::AnonymousMetaspaceType: requested = SpecializedChunk; break; - case Metaspace::ReflectionMetaspaceType: requested = SpecializedChunk; break; - default: requested = SmallChunk; break; - } - } - - // Adjust to one of the fixed chunk sizes (unless humongous) - const size_t adjusted = adjust_initial_chunk_size(requested); - - assert(adjusted != 0, "Incorrect initial chunk size. Requested: " - SIZE_FORMAT " adjusted: " SIZE_FORMAT, requested, adjusted); - - return adjusted; -} - -void SpaceManager::locked_print_chunks_in_use_on(outputStream* st) const { - - for (ChunkIndex i = ZeroIndex; i < NumberOfInUseLists; i = next_chunk_index(i)) { - st->print("SpaceManager: " UINTX_FORMAT " %s chunks.", - num_chunks_by_type(i), chunk_size_name(i)); - } - - chunk_manager()->locked_print_free_chunks(st); -} - -size_t SpaceManager::calc_chunk_size(size_t word_size) { - - // Decide between a small chunk and a medium chunk. Up to - // _small_chunk_limit small chunks can be allocated. - // After that a medium chunk is preferred. - size_t chunk_word_size; - - // Special case for anonymous metadata space. - // Anonymous metadata space is usually small, with majority within 1K - 2K range and - // rarely about 4K (64-bits JVM). - // Instead of jumping to SmallChunk after initial chunk exhausted, keeping allocation - // from SpecializeChunk up to _anon_or_delegating_metadata_specialize_chunk_limit (4) - // reduces space waste from 60+% to around 30%. - if ((_space_type == Metaspace::AnonymousMetaspaceType || _space_type == Metaspace::ReflectionMetaspaceType) && - _mdtype == Metaspace::NonClassType && - num_chunks_by_type(SpecializedIndex) < _anon_and_delegating_metadata_specialize_chunk_limit && - word_size + Metachunk::overhead() <= SpecializedChunk) { - return SpecializedChunk; - } - - if (num_chunks_by_type(MediumIndex) == 0 && - num_chunks_by_type(SmallIndex) < _small_chunk_limit) { - chunk_word_size = (size_t) small_chunk_size(); - if (word_size + Metachunk::overhead() > small_chunk_size()) { - chunk_word_size = medium_chunk_size(); - } - } else { - chunk_word_size = medium_chunk_size(); - } - - // Might still need a humongous chunk. Enforce - // humongous allocations sizes to be aligned up to - // the smallest chunk size. - size_t if_humongous_sized_chunk = - align_up(word_size + Metachunk::overhead(), - smallest_chunk_size()); - chunk_word_size = - MAX2((size_t) chunk_word_size, if_humongous_sized_chunk); - - assert(!SpaceManager::is_humongous(word_size) || - chunk_word_size == if_humongous_sized_chunk, - "Size calculation is wrong, word_size " SIZE_FORMAT - " chunk_word_size " SIZE_FORMAT, - word_size, chunk_word_size); - Log(gc, metaspace, alloc) log; - if (log.is_debug() && SpaceManager::is_humongous(word_size)) { - log.debug("Metadata humongous allocation:"); - log.debug(" word_size " PTR_FORMAT, word_size); - log.debug(" chunk_word_size " PTR_FORMAT, chunk_word_size); - log.debug(" chunk overhead " PTR_FORMAT, Metachunk::overhead()); - } - return chunk_word_size; -} - -void SpaceManager::track_metaspace_memory_usage() { - if (is_init_completed()) { - if (is_class()) { - MemoryService::track_compressed_class_memory_usage(); - } - MemoryService::track_metaspace_memory_usage(); - } -} - -MetaWord* SpaceManager::grow_and_allocate(size_t word_size) { - assert_lock_strong(_lock); - assert(vs_list()->current_virtual_space() != NULL, - "Should have been set"); - assert(current_chunk() == NULL || - current_chunk()->allocate(word_size) == NULL, - "Don't need to expand"); - MutexLockerEx cl(MetaspaceExpand_lock, Mutex::_no_safepoint_check_flag); - - if (log_is_enabled(Trace, gc, metaspace, freelist)) { - size_t words_left = 0; - size_t words_used = 0; - if (current_chunk() != NULL) { - words_left = current_chunk()->free_word_size(); - words_used = current_chunk()->used_word_size(); - } - log_trace(gc, metaspace, freelist)("SpaceManager::grow_and_allocate for " SIZE_FORMAT " words " SIZE_FORMAT " words used " SIZE_FORMAT " words left", - word_size, words_used, words_left); - } - - // Get another chunk - size_t chunk_word_size = calc_chunk_size(word_size); - Metachunk* next = get_new_chunk(chunk_word_size); - - MetaWord* mem = NULL; - - // If a chunk was available, add it to the in-use chunk list - // and do an allocation from it. - if (next != NULL) { - // Add to this manager's list of chunks in use. - // If the new chunk is humongous, it was created to serve a single large allocation. In that - // case it usually makes no sense to make it the current chunk, since the next allocation would - // need to allocate a new chunk anyway, while we would now prematurely retire a perfectly - // good chunk which could be used for more normal allocations. - bool make_current = true; - if (next->get_chunk_type() == HumongousIndex && - current_chunk() != NULL) { - make_current = false; - } - add_chunk(next, make_current); - mem = next->allocate(word_size); - } - - // Track metaspace memory usage statistic. - track_metaspace_memory_usage(); - - return mem; -} - -void SpaceManager::print_on(outputStream* st) const { - SpaceManagerStatistics stat; - add_to_statistics(&stat); // will lock _lock. - stat.print_on(st, 1*K, false); -} - -SpaceManager::SpaceManager(Metaspace::MetadataType mdtype, - Metaspace::MetaspaceType space_type, - Mutex* lock) : - _mdtype(mdtype), - _space_type(space_type), - _capacity_words(0), - _used_words(0), - _overhead_words(0), - _block_freelists(NULL), - _lock(lock), - _chunk_list(NULL), - _current_chunk(NULL) -{ - Metadebug::init_allocation_fail_alot_count(); - memset(_num_chunks_by_type, 0, sizeof(_num_chunks_by_type)); - log_trace(gc, metaspace, freelist)("SpaceManager(): " PTR_FORMAT, p2i(this)); -} - -void SpaceManager::account_for_new_chunk(const Metachunk* new_chunk) { - - assert_lock_strong(MetaspaceExpand_lock); - - _capacity_words += new_chunk->word_size(); - _overhead_words += Metachunk::overhead(); - DEBUG_ONLY(new_chunk->verify()); - _num_chunks_by_type[new_chunk->get_chunk_type()] ++; - - // Adjust global counters: - MetaspaceUtils::inc_capacity(mdtype(), new_chunk->word_size()); - MetaspaceUtils::inc_overhead(mdtype(), Metachunk::overhead()); -} - -void SpaceManager::account_for_allocation(size_t words) { - // Note: we should be locked with the ClassloaderData-specific metaspace lock. - // We may or may not be locked with the global metaspace expansion lock. - assert_lock_strong(lock()); - - // Add to the per SpaceManager totals. This can be done non-atomically. - _used_words += words; - - // Adjust global counters. This will be done atomically. - MetaspaceUtils::inc_used(mdtype(), words); -} - -void SpaceManager::account_for_spacemanager_death() { - - assert_lock_strong(MetaspaceExpand_lock); - - MetaspaceUtils::dec_capacity(mdtype(), _capacity_words); - MetaspaceUtils::dec_overhead(mdtype(), _overhead_words); - MetaspaceUtils::dec_used(mdtype(), _used_words); -} - -SpaceManager::~SpaceManager() { - - // This call this->_lock which can't be done while holding MetaspaceExpand_lock - DEBUG_ONLY(verify_metrics()); - - MutexLockerEx fcl(MetaspaceExpand_lock, - Mutex::_no_safepoint_check_flag); - - chunk_manager()->slow_locked_verify(); - - account_for_spacemanager_death(); - - Log(gc, metaspace, freelist) log; - if (log.is_trace()) { - log.trace("~SpaceManager(): " PTR_FORMAT, p2i(this)); - ResourceMark rm; - LogStream ls(log.trace()); - locked_print_chunks_in_use_on(&ls); - if (block_freelists() != NULL) { - block_freelists()->print_on(&ls); - } - } - - // Add all the chunks in use by this space manager - // to the global list of free chunks. - - // Follow each list of chunks-in-use and add them to the - // free lists. Each list is NULL terminated. - chunk_manager()->return_chunk_list(chunk_list()); -#ifdef ASSERT - _chunk_list = NULL; - _current_chunk = NULL; -#endif - - chunk_manager()->slow_locked_verify(); - - if (_block_freelists != NULL) { - delete _block_freelists; - } -} - -void SpaceManager::deallocate(MetaWord* p, size_t word_size) { - assert_lock_strong(lock()); - // Allocations and deallocations are in raw_word_size - size_t raw_word_size = get_allocation_word_size(word_size); - // Lazily create a block_freelist - if (block_freelists() == NULL) { - _block_freelists = new BlockFreelist(); - } - block_freelists()->return_block(p, raw_word_size); - DEBUG_ONLY(Atomic::inc(&g_internal_statistics.num_deallocs)); -} - -// Adds a chunk to the list of chunks in use. -void SpaceManager::add_chunk(Metachunk* new_chunk, bool make_current) { - - assert_lock_strong(_lock); - assert(new_chunk != NULL, "Should not be NULL"); - assert(new_chunk->next() == NULL, "Should not be on a list"); - - new_chunk->reset_empty(); - - // Find the correct list and and set the current - // chunk for that list. - ChunkIndex index = chunk_manager()->list_index(new_chunk->word_size()); - - if (make_current) { - // If we are to make the chunk current, retire the old current chunk and replace - // it with the new chunk. - retire_current_chunk(); - set_current_chunk(new_chunk); - } - - // Add the new chunk at the head of its respective chunk list. - new_chunk->set_next(_chunk_list); - _chunk_list = new_chunk; - - // Adjust counters. - account_for_new_chunk(new_chunk); - - assert(new_chunk->is_empty(), "Not ready for reuse"); - Log(gc, metaspace, freelist) log; - if (log.is_trace()) { - log.trace("SpaceManager::added chunk: "); - ResourceMark rm; - LogStream ls(log.trace()); - new_chunk->print_on(&ls); - chunk_manager()->locked_print_free_chunks(&ls); - } -} - -void SpaceManager::retire_current_chunk() { - if (current_chunk() != NULL) { - size_t remaining_words = current_chunk()->free_word_size(); - if (remaining_words >= SmallBlocks::small_block_min_size()) { - MetaWord* ptr = current_chunk()->allocate(remaining_words); - deallocate(ptr, remaining_words); - account_for_allocation(remaining_words); - } - } -} - -Metachunk* SpaceManager::get_new_chunk(size_t chunk_word_size) { - // Get a chunk from the chunk freelist - Metachunk* next = chunk_manager()->chunk_freelist_allocate(chunk_word_size); - - if (next == NULL) { - next = vs_list()->get_new_chunk(chunk_word_size, - medium_chunk_bunch()); - } - - Log(gc, metaspace, alloc) log; - if (log.is_debug() && next != NULL && - SpaceManager::is_humongous(next->word_size())) { - log.debug(" new humongous chunk word size " PTR_FORMAT, next->word_size()); - } - - return next; -} - -MetaWord* SpaceManager::allocate(size_t word_size) { - MutexLockerEx cl(lock(), Mutex::_no_safepoint_check_flag); - size_t raw_word_size = get_allocation_word_size(word_size); - BlockFreelist* fl = block_freelists(); - MetaWord* p = NULL; - - DEBUG_ONLY(if (VerifyMetaspace) verify_metrics_locked()); - - // Allocation from the dictionary is expensive in the sense that - // the dictionary has to be searched for a size. Don't allocate - // from the dictionary until it starts to get fat. Is this - // a reasonable policy? Maybe an skinny dictionary is fast enough - // for allocations. Do some profiling. JJJ - if (fl != NULL && fl->total_size() > allocation_from_dictionary_limit) { - p = fl->get_block(raw_word_size); - if (p != NULL) { - DEBUG_ONLY(Atomic::inc(&g_internal_statistics.num_allocs_from_deallocated_blocks)); - } - } - if (p == NULL) { - p = allocate_work(raw_word_size); - } - - return p; -} - -// Returns the address of spaced allocated for "word_size". -// This methods does not know about blocks (Metablocks) -MetaWord* SpaceManager::allocate_work(size_t word_size) { - assert_lock_strong(lock()); -#ifdef ASSERT - if (Metadebug::test_metadata_failure()) { - return NULL; - } -#endif - // Is there space in the current chunk? - MetaWord* result = NULL; - - if (current_chunk() != NULL) { - result = current_chunk()->allocate(word_size); - } - - if (result == NULL) { - result = grow_and_allocate(word_size); - } - - if (result != NULL) { - account_for_allocation(word_size); - } - - return result; -} - -void SpaceManager::verify() { - Metachunk* curr = chunk_list(); - while (curr != NULL) { - DEBUG_ONLY(do_verify_chunk(curr);) - assert(curr->is_tagged_free() == false, "Chunk should be tagged as in use."); - curr = curr->next(); - } -} - -void SpaceManager::verify_chunk_size(Metachunk* chunk) { - assert(is_humongous(chunk->word_size()) || - chunk->word_size() == medium_chunk_size() || - chunk->word_size() == small_chunk_size() || - chunk->word_size() == specialized_chunk_size(), - "Chunk size is wrong"); - return; -} - -void SpaceManager::add_to_statistics_locked(SpaceManagerStatistics* out) const { - assert_lock_strong(lock()); - Metachunk* chunk = chunk_list(); - while (chunk != NULL) { - UsedChunksStatistics& chunk_stat = out->chunk_stats(chunk->get_chunk_type()); - chunk_stat.add_num(1); - chunk_stat.add_cap(chunk->word_size()); - chunk_stat.add_overhead(Metachunk::overhead()); - chunk_stat.add_used(chunk->used_word_size() - Metachunk::overhead()); - if (chunk != current_chunk()) { - chunk_stat.add_waste(chunk->free_word_size()); - } else { - chunk_stat.add_free(chunk->free_word_size()); - } - chunk = chunk->next(); - } - if (block_freelists() != NULL) { - out->add_free_blocks_info(block_freelists()->num_blocks(), block_freelists()->total_size()); - } -} - -void SpaceManager::add_to_statistics(SpaceManagerStatistics* out) const { - MutexLockerEx cl(lock(), Mutex::_no_safepoint_check_flag); - add_to_statistics_locked(out); -} - -#ifdef ASSERT -void SpaceManager::verify_metrics_locked() const { - assert_lock_strong(lock()); - - SpaceManagerStatistics stat; - add_to_statistics_locked(&stat); - - UsedChunksStatistics chunk_stats = stat.totals(); - - DEBUG_ONLY(chunk_stats.check_sanity()); - - assert_counter(_capacity_words, chunk_stats.cap(), "SpaceManager::_capacity_words"); - assert_counter(_used_words, chunk_stats.used(), "SpaceManager::_used_words"); - assert_counter(_overhead_words, chunk_stats.overhead(), "SpaceManager::_overhead_words"); -} - -void SpaceManager::verify_metrics() const { - MutexLockerEx cl(lock(), Mutex::_no_safepoint_check_flag); - verify_metrics_locked(); -} -#endif // ASSERT - - - // MetaspaceUtils size_t MetaspaceUtils::_capacity_words [Metaspace:: MetadataTypeCount] = {0, 0}; size_t MetaspaceUtils::_overhead_words [Metaspace:: MetadataTypeCount] = {0, 0}; @@ -3733,72 +482,6 @@ } } -class PrintCLDMetaspaceInfoClosure : public CLDClosure { -private: - outputStream* const _out; - const size_t _scale; - const bool _do_print; - const bool _break_down_by_chunktype; - -public: - - uintx _num_loaders; - ClassLoaderMetaspaceStatistics _stats_total; - - uintx _num_loaders_by_spacetype [Metaspace::MetaspaceTypeCount]; - ClassLoaderMetaspaceStatistics _stats_by_spacetype [Metaspace::MetaspaceTypeCount]; - -public: - PrintCLDMetaspaceInfoClosure(outputStream* out, size_t scale, bool do_print, bool break_down_by_chunktype) - : _out(out), _scale(scale), _do_print(do_print), _break_down_by_chunktype(break_down_by_chunktype) - , _num_loaders(0) - { - memset(_num_loaders_by_spacetype, 0, sizeof(_num_loaders_by_spacetype)); - } - - void do_cld(ClassLoaderData* cld) { - - assert(SafepointSynchronize::is_at_safepoint(), "Must be at a safepoint"); - - ClassLoaderMetaspace* msp = cld->metaspace_or_null(); - if (msp == NULL) { - return; - } - - // Collect statistics for this class loader metaspace - ClassLoaderMetaspaceStatistics this_cld_stat; - msp->add_to_statistics(&this_cld_stat); - - // And add it to the running totals - _stats_total.add(this_cld_stat); - _num_loaders ++; - _stats_by_spacetype[msp->space_type()].add(this_cld_stat); - _num_loaders_by_spacetype[msp->space_type()] ++; - - // Optionally, print. - if (_do_print) { - - _out->print(UINTX_FORMAT_W(4) ": ", _num_loaders); - - if (cld->is_anonymous()) { - _out->print("ClassLoaderData " PTR_FORMAT " for anonymous class", p2i(cld)); - } else { - ResourceMark rm; - _out->print("ClassLoaderData " PTR_FORMAT " for %s", p2i(cld), cld->loader_name()); - } - - if (cld->is_unloading()) { - _out->print(" (unloading)"); - } - - this_cld_stat.print_on(_out, _scale, _break_down_by_chunktype); - _out->cr(); - - } - - } // do_cld - -}; void MetaspaceUtils::print_vs(outputStream* out, size_t scale) { const size_t reserved_nonclass_words = reserved_bytes(Metaspace::NonClassType) / sizeof(MetaWord); @@ -4839,24 +1522,6 @@ add_to_statistics_locked(out); } -#ifdef ASSERT -static void do_verify_chunk(Metachunk* chunk) { - guarantee(chunk != NULL, "Sanity"); - // Verify chunk itself; then verify that it is consistent with the - // occupany map of its containing node. - chunk->verify(); - VirtualSpaceNode* const vsn = chunk->container(); - OccupancyMap* const ocmap = vsn->occupancy_map(); - ocmap->verify_for_chunk(chunk); -} -#endif - -static void do_update_in_use_info_for_chunk(Metachunk* chunk, bool inuse) { - chunk->set_is_tagged_free(!inuse); - OccupancyMap* const ocmap = chunk->container()->occupancy_map(); - ocmap->set_region_in_use((MetaWord*)chunk, chunk->word_size(), inuse); -} - /////////////// Unit tests /////////////// #ifndef PRODUCT @@ -5081,99 +1746,8 @@ } }; -// The following test is placed here instead of a gtest / unittest file -// because the ChunkManager class is only available in this file. -void ChunkManager_test_list_index() { - { - // Test previous bug where a query for a humongous class metachunk, - // incorrectly matched the non-class medium metachunk size. - { - ChunkManager manager(true); - - assert(MediumChunk > ClassMediumChunk, "Precondition for test"); - - ChunkIndex index = manager.list_index(MediumChunk); - - assert(index == HumongousIndex, - "Requested size is larger than ClassMediumChunk," - " so should return HumongousIndex. Got index: %d", (int)index); - } - - // Check the specified sizes as well. - { - ChunkManager manager(true); - assert(manager.list_index(ClassSpecializedChunk) == SpecializedIndex, "sanity"); - assert(manager.list_index(ClassSmallChunk) == SmallIndex, "sanity"); - assert(manager.list_index(ClassMediumChunk) == MediumIndex, "sanity"); - assert(manager.list_index(ClassMediumChunk + ClassSpecializedChunk) == HumongousIndex, "sanity"); - } - { - ChunkManager manager(false); - assert(manager.list_index(SpecializedChunk) == SpecializedIndex, "sanity"); - assert(manager.list_index(SmallChunk) == SmallIndex, "sanity"); - assert(manager.list_index(MediumChunk) == MediumIndex, "sanity"); - assert(manager.list_index(MediumChunk + SpecializedChunk) == HumongousIndex, "sanity"); - } - - } - -} - #endif // !PRODUCT -#ifdef ASSERT - -// The following test is placed here instead of a gtest / unittest file -// because the ChunkManager class is only available in this file. -class SpaceManagerTest : AllStatic { - friend void SpaceManager_test_adjust_initial_chunk_size(); - - static void test_adjust_initial_chunk_size(bool is_class) { - const size_t smallest = SpaceManager::smallest_chunk_size(is_class); - const size_t normal = SpaceManager::small_chunk_size(is_class); - const size_t medium = SpaceManager::medium_chunk_size(is_class); - -#define test_adjust_initial_chunk_size(value, expected, is_class_value) \ - do { \ - size_t v = value; \ - size_t e = expected; \ - assert(SpaceManager::adjust_initial_chunk_size(v, (is_class_value)) == e, \ - "Expected: " SIZE_FORMAT " got: " SIZE_FORMAT, e, v); \ - } while (0) - - // Smallest (specialized) - test_adjust_initial_chunk_size(1, smallest, is_class); - test_adjust_initial_chunk_size(smallest - 1, smallest, is_class); - test_adjust_initial_chunk_size(smallest, smallest, is_class); - - // Small - test_adjust_initial_chunk_size(smallest + 1, normal, is_class); - test_adjust_initial_chunk_size(normal - 1, normal, is_class); - test_adjust_initial_chunk_size(normal, normal, is_class); - - // Medium - test_adjust_initial_chunk_size(normal + 1, medium, is_class); - test_adjust_initial_chunk_size(medium - 1, medium, is_class); - test_adjust_initial_chunk_size(medium, medium, is_class); - - // Humongous - test_adjust_initial_chunk_size(medium + 1, medium + 1, is_class); - -#undef test_adjust_initial_chunk_size - } - - static void test_adjust_initial_chunk_size() { - test_adjust_initial_chunk_size(false); - test_adjust_initial_chunk_size(true); - } -}; - -void SpaceManager_test_adjust_initial_chunk_size() { - SpaceManagerTest::test_adjust_initial_chunk_size(); -} - -#endif // ASSERT - struct chunkmanager_statistics_t { int num_specialized_chunks; int num_small_chunks; diff --git a/src/hotspot/share/memory/metaspace.hpp b/src/hotspot/share/memory/metaspace.hpp --- a/src/hotspot/share/memory/metaspace.hpp +++ b/src/hotspot/share/memory/metaspace.hpp @@ -55,23 +55,23 @@ // +-------------------+ // -class ChunkManager; class ClassLoaderData; -class Metablock; -class Metachunk; class MetaspaceTracer; class MetaWord; class Mutex; class outputStream; -class PrintCLDMetaspaceInfoClosure; -class SpaceManager; -class VirtualSpaceList; + class CollectedHeap; namespace metaspace { -namespace internals { + class ChunkManager; class ClassLoaderMetaspaceStatistics; -}} + class Metablock; + class Metachunk; + class PrintCLDMetaspaceInfoClosure; + class SpaceManager; + class VirtualSpaceList; +} // Metaspaces each have a SpaceManager and allocations // are done by the SpaceManager. Allocations are done @@ -131,31 +131,31 @@ DEBUG_ONLY(static bool _frozen;) // Virtual Space lists for both classes and other metadata - static VirtualSpaceList* _space_list; - static VirtualSpaceList* _class_space_list; + static metaspace::VirtualSpaceList* _space_list; + static metaspace::VirtualSpaceList* _class_space_list; - static ChunkManager* _chunk_manager_metadata; - static ChunkManager* _chunk_manager_class; + static metaspace::ChunkManager* _chunk_manager_metadata; + static metaspace::ChunkManager* _chunk_manager_class; static const MetaspaceTracer* _tracer; public: - static VirtualSpaceList* space_list() { return _space_list; } - static VirtualSpaceList* class_space_list() { return _class_space_list; } - static VirtualSpaceList* get_space_list(MetadataType mdtype) { + static metaspace::VirtualSpaceList* space_list() { return _space_list; } + static metaspace::VirtualSpaceList* class_space_list() { return _class_space_list; } + static metaspace::VirtualSpaceList* get_space_list(MetadataType mdtype) { assert(mdtype != MetadataTypeCount, "MetadaTypeCount can't be used as mdtype"); return mdtype == ClassType ? class_space_list() : space_list(); } - static ChunkManager* chunk_manager_metadata() { return _chunk_manager_metadata; } - static ChunkManager* chunk_manager_class() { return _chunk_manager_class; } - static ChunkManager* get_chunk_manager(MetadataType mdtype) { + static metaspace::ChunkManager* chunk_manager_metadata() { return _chunk_manager_metadata; } + static metaspace::ChunkManager* chunk_manager_class() { return _chunk_manager_class; } + static metaspace::ChunkManager* get_chunk_manager(MetadataType mdtype) { assert(mdtype != MetadataTypeCount, "MetadaTypeCount can't be used as mdtype"); return mdtype == ClassType ? chunk_manager_class() : chunk_manager_metadata(); } // convenience function - static ChunkManager* get_chunk_manager(bool is_class) { + static metaspace::ChunkManager* get_chunk_manager(bool is_class) { return is_class ? chunk_manager_class() : chunk_manager_metadata(); } @@ -231,7 +231,7 @@ friend class CollectedHeap; // For expand_and_allocate() friend class Metaspace; friend class MetaspaceUtils; - friend class PrintCLDMetaspaceInfoClosure; + friend class metaspace::PrintCLDMetaspaceInfoClosure; friend class VM_CollectForMetadataAllocation; // For expand_and_allocate() private: @@ -242,16 +242,16 @@ // special cases such as the boot class loader, reflection // class loader and anonymous class loader. void initialize_first_chunk(Metaspace::MetaspaceType type, Metaspace::MetadataType mdtype); - Metachunk* get_initialization_chunk(Metaspace::MetaspaceType type, Metaspace::MetadataType mdtype); + metaspace::Metachunk* get_initialization_chunk(Metaspace::MetaspaceType type, Metaspace::MetadataType mdtype); const Metaspace::MetaspaceType _space_type; Mutex* const _lock; - SpaceManager* _vsm; - SpaceManager* _class_vsm; + metaspace::SpaceManager* _vsm; + metaspace::SpaceManager* _class_vsm; - SpaceManager* vsm() const { return _vsm; } - SpaceManager* class_vsm() const { return _class_vsm; } - SpaceManager* get_space_manager(Metaspace::MetadataType mdtype) { + metaspace::SpaceManager* vsm() const { return _vsm; } + metaspace::SpaceManager* class_vsm() const { return _class_vsm; } + metaspace::SpaceManager* get_space_manager(Metaspace::MetadataType mdtype) { assert(mdtype != Metaspace::MetadataTypeCount, "MetadaTypeCount can't be used as mdtype"); return mdtype == Metaspace::ClassType ? class_vsm() : vsm(); } @@ -263,15 +263,15 @@ size_t class_chunk_size(size_t word_size); // Adds to the given statistic object. Must be locked with CLD metaspace lock. - void add_to_statistics_locked(metaspace::internals::ClassLoaderMetaspaceStatistics* out) const; + void add_to_statistics_locked(metaspace::ClassLoaderMetaspaceStatistics* out) const; + + Metaspace::MetaspaceType space_type() const { return _space_type; } public: ClassLoaderMetaspace(Mutex* lock, Metaspace::MetaspaceType type); ~ClassLoaderMetaspace(); - Metaspace::MetaspaceType space_type() const { return _space_type; } - // Allocate space for metadata of type mdtype. This is space // within a Metachunk and is used by // allocate(ClassLoaderData*, size_t, bool, MetadataType, TRAPS) @@ -287,14 +287,14 @@ void verify(); // Adds to the given statistic object. Will lock with CLD metaspace lock. - void add_to_statistics(metaspace::internals::ClassLoaderMetaspaceStatistics* out) const; + void add_to_statistics(metaspace::ClassLoaderMetaspaceStatistics* out) const; }; // ClassLoaderMetaspace class MetaspaceUtils : AllStatic { // Spacemanager updates running counters. - friend class SpaceManager; + friend class metaspace::SpaceManager; // Running counters for statistics concerning in-use chunks. // Note: capacity = used + free + waste + overhead. Note that we do not @@ -328,7 +328,7 @@ // Collect used metaspace statistics. This involves walking the CLDG. The resulting // output will be the accumulated values for all live metaspaces. // Note: method does not do any locking. - static void collect_statistics(metaspace::internals::ClassLoaderMetaspaceStatistics* out); + static void collect_statistics(metaspace::ClassLoaderMetaspaceStatistics* out); // Used by MetaspaceCounters static size_t free_chunks_total_words(); diff --git a/src/hotspot/share/memory/metaspace/blockFreelist.cpp b/src/hotspot/share/memory/metaspace/blockFreelist.cpp new file mode 100644 --- /dev/null +++ b/src/hotspot/share/memory/metaspace/blockFreelist.cpp @@ -0,0 +1,109 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ +#include "precompiled.hpp" + +#include "logging/log.hpp" +#include "memory/binaryTreeDictionary.inline.hpp" +#include "memory/metaspace/blockFreelist.hpp" +#include "utilities/ostream.hpp" +#include "utilities/globalDefinitions.hpp" + + +namespace metaspace { + + +BlockFreelist::BlockFreelist() : _dictionary(new BlockTreeDictionary()), _small_blocks(NULL) {} + +BlockFreelist::~BlockFreelist() { + delete _dictionary; + if (_small_blocks != NULL) { + delete _small_blocks; + } +} + +void BlockFreelist::return_block(MetaWord* p, size_t word_size) { + assert(word_size >= SmallBlocks::small_block_min_size(), "never return dark matter"); + + Metablock* free_chunk = ::new (p) Metablock(word_size); + if (word_size < SmallBlocks::small_block_max_size()) { + small_blocks()->return_block(free_chunk, word_size); + } else { + dictionary()->return_chunk(free_chunk); +} + log_trace(gc, metaspace, freelist, blocks)("returning block at " INTPTR_FORMAT " size = " + SIZE_FORMAT, p2i(free_chunk), word_size); +} + +MetaWord* BlockFreelist::get_block(size_t word_size) { + assert(word_size >= SmallBlocks::small_block_min_size(), "never get dark matter"); + + // Try small_blocks first. + if (word_size < SmallBlocks::small_block_max_size()) { + // Don't create small_blocks() until needed. small_blocks() allocates the small block list for + // this space manager. + MetaWord* new_block = (MetaWord*) small_blocks()->get_block(word_size); + if (new_block != NULL) { + log_trace(gc, metaspace, freelist, blocks)("getting block at " INTPTR_FORMAT " size = " SIZE_FORMAT, + p2i(new_block), word_size); + return new_block; + } + } + + if (word_size < BlockFreelist::min_dictionary_size()) { + // If allocation in small blocks fails, this is Dark Matter. Too small for dictionary. + return NULL; + } + + Metablock* free_block = dictionary()->get_chunk(word_size); + if (free_block == NULL) { + return NULL; + } + + const size_t block_size = free_block->size(); + if (block_size > WasteMultiplier * word_size) { + return_block((MetaWord*)free_block, block_size); + return NULL; + } + + MetaWord* new_block = (MetaWord*)free_block; + assert(block_size >= word_size, "Incorrect size of block from freelist"); + const size_t unused = block_size - word_size; + if (unused >= SmallBlocks::small_block_min_size()) { + return_block(new_block + word_size, unused); + } + + log_trace(gc, metaspace, freelist, blocks)("getting block at " INTPTR_FORMAT " size = " SIZE_FORMAT, + p2i(new_block), word_size); + return new_block; +} + +void BlockFreelist::print_on(outputStream* st) const { + dictionary()->print_free_lists(st); + if (_small_blocks != NULL) { + _small_blocks->print_on(st); + } +} + +} // namespace metaspace + diff --git a/src/hotspot/share/memory/metaspace/blockFreelist.hpp b/src/hotspot/share/memory/metaspace/blockFreelist.hpp new file mode 100644 --- /dev/null +++ b/src/hotspot/share/memory/metaspace/blockFreelist.hpp @@ -0,0 +1,93 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_MEMORY_METASPACE_BLOCKFREELIST_HPP_ +#define SHARE_MEMORY_METASPACE_BLOCKFREELIST_HPP_ + +#include "memory/allocation.hpp" +#include "memory/binaryTreeDictionary.hpp" +#include "memory/freeList.hpp" +#include "memory/metaspace/smallBlocks.hpp" +#include "memory/metaspace/metablock.hpp" +#include "utilities/globalDefinitions.hpp" + +namespace metaspace { + +typedef BinaryTreeDictionary > BlockTreeDictionary; + +// Used to manage the free list of Metablocks (a block corresponds +// to the allocation of a quantum of metadata). +class BlockFreelist : public CHeapObj { + + BlockTreeDictionary* const _dictionary; + SmallBlocks* _small_blocks; + + // Only allocate and split from freelist if the size of the allocation + // is at least 1/4th the size of the available block. + const static int WasteMultiplier = 4; + + // Accessors + BlockTreeDictionary* dictionary() const { return _dictionary; } + SmallBlocks* small_blocks() { + if (_small_blocks == NULL) { + _small_blocks = new SmallBlocks(); + } + return _small_blocks; + } + + public: + + BlockFreelist(); + ~BlockFreelist(); + + // Get and return a block to the free list + MetaWord* get_block(size_t word_size); + void return_block(MetaWord* p, size_t word_size); + + // Returns the total size, in words, of all blocks kept in this structure. + size_t total_size() const { + size_t result = dictionary()->total_size(); + if (_small_blocks != NULL) { + result = result + _small_blocks->total_size(); + } + return result; + } + + // Returns the number of all blocks kept in this structure. + uintx num_blocks() const { + uintx result = dictionary()->total_free_blocks(); + if (_small_blocks != NULL) { + result = result + _small_blocks->total_num_blocks(); + } + return result; + } + + static size_t min_dictionary_size() { return TreeChunk >::min_size(); } + void print_on(outputStream* st) const; +}; + +} // namespace metaspace + +#endif /* SHARE_MEMORY_METASPACE_BLOCKFREELIST_HPP_ */ + diff --git a/src/hotspot/share/memory/metaspace/chunkManager.cpp b/src/hotspot/share/memory/metaspace/chunkManager.cpp new file mode 100644 --- /dev/null +++ b/src/hotspot/share/memory/metaspace/chunkManager.cpp @@ -0,0 +1,654 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ +#include "precompiled.hpp" + +#include "logging/log.hpp" +#include "logging/logStream.hpp" +#include "memory/binaryTreeDictionary.inline.hpp" +#include "memory/freeList.inline.hpp" +#include "memory/metaspace/chunkManager.hpp" +#include "memory/metaspace/metachunk.hpp" +#include "memory/metaspace/metaspaceCommon.hpp" +#include "memory/metaspace/metaspaceStatistics.hpp" +#include "memory/metaspace/occupancyMap.hpp" +#include "memory/metaspace/virtualSpaceNode.hpp" +#include "runtime/mutexLocker.hpp" +#include "utilities/debug.hpp" +#include "utilities/globalDefinitions.hpp" +#include "utilities/ostream.hpp" + +namespace metaspace { + +void ChunkManager::remove_chunk(Metachunk* chunk) { + size_t word_size = chunk->word_size(); + ChunkIndex index = list_index(word_size); + if (index != HumongousIndex) { + free_chunks(index)->remove_chunk(chunk); + } else { + humongous_dictionary()->remove_chunk(chunk); + } + + // Chunk has been removed from the chunks free list, update counters. + account_for_removed_chunk(chunk); +} + +bool ChunkManager::attempt_to_coalesce_around_chunk(Metachunk* chunk, ChunkIndex target_chunk_type) { + assert_lock_strong(MetaspaceExpand_lock); + assert(chunk != NULL, "invalid chunk pointer"); + // Check for valid merge combinations. + assert((chunk->get_chunk_type() == SpecializedIndex && + (target_chunk_type == SmallIndex || target_chunk_type == MediumIndex)) || + (chunk->get_chunk_type() == SmallIndex && target_chunk_type == MediumIndex), + "Invalid chunk merge combination."); + + const size_t target_chunk_word_size = + get_size_for_nonhumongous_chunktype(target_chunk_type, this->is_class()); + + // [ prospective merge region ) + MetaWord* const p_merge_region_start = + (MetaWord*) align_down(chunk, target_chunk_word_size * sizeof(MetaWord)); + MetaWord* const p_merge_region_end = + p_merge_region_start + target_chunk_word_size; + + // We need the VirtualSpaceNode containing this chunk and its occupancy map. + VirtualSpaceNode* const vsn = chunk->container(); + OccupancyMap* const ocmap = vsn->occupancy_map(); + + // The prospective chunk merge range must be completely contained by the + // committed range of the virtual space node. + if (p_merge_region_start < vsn->bottom() || p_merge_region_end > vsn->top()) { + return false; + } + + // Only attempt to merge this range if at its start a chunk starts and at its end + // a chunk ends. If a chunk (can only be humongous) straddles either start or end + // of that range, we cannot merge. + if (!ocmap->chunk_starts_at_address(p_merge_region_start)) { + return false; + } + if (p_merge_region_end < vsn->top() && + !ocmap->chunk_starts_at_address(p_merge_region_end)) { + return false; + } + + // Now check if the prospective merge area contains live chunks. If it does we cannot merge. + if (ocmap->is_region_in_use(p_merge_region_start, target_chunk_word_size)) { + return false; + } + + // Success! Remove all chunks in this region... + log_trace(gc, metaspace, freelist)("%s: coalescing chunks in area [%p-%p)...", + (is_class() ? "class space" : "metaspace"), + p_merge_region_start, p_merge_region_end); + + const int num_chunks_removed = + remove_chunks_in_area(p_merge_region_start, target_chunk_word_size); + + // ... and create a single new bigger chunk. + Metachunk* const p_new_chunk = + ::new (p_merge_region_start) Metachunk(target_chunk_type, is_class(), target_chunk_word_size, vsn); + assert(p_new_chunk == (Metachunk*)p_merge_region_start, "Sanity"); + p_new_chunk->set_origin(origin_merge); + + log_trace(gc, metaspace, freelist)("%s: created coalesced chunk at %p, size " SIZE_FORMAT_HEX ".", + (is_class() ? "class space" : "metaspace"), + p_new_chunk, p_new_chunk->word_size() * sizeof(MetaWord)); + + // Fix occupancy map: remove old start bits of the small chunks and set new start bit. + ocmap->wipe_chunk_start_bits_in_region(p_merge_region_start, target_chunk_word_size); + ocmap->set_chunk_starts_at_address(p_merge_region_start, true); + + // Mark chunk as free. Note: it is not necessary to update the occupancy + // map in-use map, because the old chunks were also free, so nothing + // should have changed. + p_new_chunk->set_is_tagged_free(true); + + // Add new chunk to its freelist. + ChunkList* const list = free_chunks(target_chunk_type); + list->return_chunk_at_head(p_new_chunk); + + // And adjust ChunkManager:: _free_chunks_count (_free_chunks_total + // should not have changed, because the size of the space should be the same) + _free_chunks_count -= num_chunks_removed; + _free_chunks_count ++; + + // VirtualSpaceNode::container_count does not have to be modified: + // it means "number of active (non-free) chunks", so merging free chunks + // should not affect that count. + + // At the end of a chunk merge, run verification tests. + if (VerifyMetaspace) { + DEBUG_ONLY(this->locked_verify()); + DEBUG_ONLY(vsn->verify()); + } + + return true; +} + +// Remove all chunks in the given area - the chunks are supposed to be free - +// from their corresponding freelists. Mark them as invalid. +// - This does not correct the occupancy map. +// - This does not adjust the counters in ChunkManager. +// - Does not adjust container count counter in containing VirtualSpaceNode +// Returns number of chunks removed. +int ChunkManager::remove_chunks_in_area(MetaWord* p, size_t word_size) { + assert(p != NULL && word_size > 0, "Invalid range."); + const size_t smallest_chunk_size = get_size_for_nonhumongous_chunktype(SpecializedIndex, is_class()); + assert_is_aligned(word_size, smallest_chunk_size); + + Metachunk* const start = (Metachunk*) p; + const Metachunk* const end = (Metachunk*)(p + word_size); + Metachunk* cur = start; + int num_removed = 0; + while (cur < end) { + Metachunk* next = (Metachunk*)(((MetaWord*)cur) + cur->word_size()); + DEBUG_ONLY(do_verify_chunk(cur)); + assert(cur->get_chunk_type() != HumongousIndex, "Unexpected humongous chunk found at %p.", cur); + assert(cur->is_tagged_free(), "Chunk expected to be free (%p)", cur); + log_trace(gc, metaspace, freelist)("%s: removing chunk %p, size " SIZE_FORMAT_HEX ".", + (is_class() ? "class space" : "metaspace"), + cur, cur->word_size() * sizeof(MetaWord)); + cur->remove_sentinel(); + // Note: cannot call ChunkManager::remove_chunk, because that + // modifies the counters in ChunkManager, which we do not want. So + // we call remove_chunk on the freelist directly (see also the + // splitting function which does the same). + ChunkList* const list = free_chunks(list_index(cur->word_size())); + list->remove_chunk(cur); + num_removed ++; + cur = next; + } + return num_removed; +} + +size_t ChunkManager::free_chunks_total_words() { + return _free_chunks_total; +} + +size_t ChunkManager::free_chunks_total_bytes() { + return free_chunks_total_words() * BytesPerWord; +} + +// Update internal accounting after a chunk was added +void ChunkManager::account_for_added_chunk(const Metachunk* c) { + assert_lock_strong(MetaspaceExpand_lock); + _free_chunks_count ++; + _free_chunks_total += c->word_size(); +} + +// Update internal accounting after a chunk was removed +void ChunkManager::account_for_removed_chunk(const Metachunk* c) { + assert_lock_strong(MetaspaceExpand_lock); + assert(_free_chunks_count >= 1, + "ChunkManager::_free_chunks_count: about to go negative (" SIZE_FORMAT ").", _free_chunks_count); + assert(_free_chunks_total >= c->word_size(), + "ChunkManager::_free_chunks_total: about to go negative" + "(now: " SIZE_FORMAT ", decrement value: " SIZE_FORMAT ").", _free_chunks_total, c->word_size()); + _free_chunks_count --; + _free_chunks_total -= c->word_size(); +} + +size_t ChunkManager::free_chunks_count() { +#ifdef ASSERT + if (!UseConcMarkSweepGC && !MetaspaceExpand_lock->is_locked()) { + MutexLockerEx cl(MetaspaceExpand_lock, + Mutex::_no_safepoint_check_flag); + // This lock is only needed in debug because the verification + // of the _free_chunks_totals walks the list of free chunks + slow_locked_verify_free_chunks_count(); + } +#endif + return _free_chunks_count; +} + +ChunkIndex ChunkManager::list_index(size_t size) { + return get_chunk_type_by_size(size, is_class()); +} + +size_t ChunkManager::size_by_index(ChunkIndex index) const { + index_bounds_check(index); + assert(index != HumongousIndex, "Do not call for humongous chunks."); + return get_size_for_nonhumongous_chunktype(index, is_class()); +} + +void ChunkManager::locked_verify_free_chunks_total() { + assert_lock_strong(MetaspaceExpand_lock); + assert(sum_free_chunks() == _free_chunks_total, + "_free_chunks_total " SIZE_FORMAT " is not the" + " same as sum " SIZE_FORMAT, _free_chunks_total, + sum_free_chunks()); +} + +void ChunkManager::locked_verify_free_chunks_count() { + assert_lock_strong(MetaspaceExpand_lock); + assert(sum_free_chunks_count() == _free_chunks_count, + "_free_chunks_count " SIZE_FORMAT " is not the" + " same as sum " SIZE_FORMAT, _free_chunks_count, + sum_free_chunks_count()); +} + +void ChunkManager::verify() { + MutexLockerEx cl(MetaspaceExpand_lock, + Mutex::_no_safepoint_check_flag); + locked_verify(); +} + +void ChunkManager::locked_verify() { + locked_verify_free_chunks_count(); + locked_verify_free_chunks_total(); + for (ChunkIndex i = ZeroIndex; i < NumberOfFreeLists; i = next_chunk_index(i)) { + ChunkList* list = free_chunks(i); + if (list != NULL) { + Metachunk* chunk = list->head(); + while (chunk) { + DEBUG_ONLY(do_verify_chunk(chunk);) + assert(chunk->is_tagged_free(), "Chunk should be tagged as free."); + chunk = chunk->next(); + } + } + } +} + +void ChunkManager::locked_print_free_chunks(outputStream* st) { + assert_lock_strong(MetaspaceExpand_lock); + st->print_cr("Free chunk total " SIZE_FORMAT " count " SIZE_FORMAT, + _free_chunks_total, _free_chunks_count); +} + +void ChunkManager::locked_print_sum_free_chunks(outputStream* st) { + assert_lock_strong(MetaspaceExpand_lock); + st->print_cr("Sum free chunk total " SIZE_FORMAT " count " SIZE_FORMAT, + sum_free_chunks(), sum_free_chunks_count()); +} + +ChunkList* ChunkManager::free_chunks(ChunkIndex index) { + assert(index == SpecializedIndex || index == SmallIndex || index == MediumIndex, + "Bad index: %d", (int)index); + + return &_free_chunks[index]; +} + +// These methods that sum the free chunk lists are used in printing +// methods that are used in product builds. +size_t ChunkManager::sum_free_chunks() { + assert_lock_strong(MetaspaceExpand_lock); + size_t result = 0; + for (ChunkIndex i = ZeroIndex; i < NumberOfFreeLists; i = next_chunk_index(i)) { + ChunkList* list = free_chunks(i); + + if (list == NULL) { + continue; + } + + result = result + list->count() * list->size(); + } + result = result + humongous_dictionary()->total_size(); + return result; +} + +size_t ChunkManager::sum_free_chunks_count() { + assert_lock_strong(MetaspaceExpand_lock); + size_t count = 0; + for (ChunkIndex i = ZeroIndex; i < NumberOfFreeLists; i = next_chunk_index(i)) { + ChunkList* list = free_chunks(i); + if (list == NULL) { + continue; + } + count = count + list->count(); + } + count = count + humongous_dictionary()->total_free_blocks(); + return count; +} + +ChunkList* ChunkManager::find_free_chunks_list(size_t word_size) { + ChunkIndex index = list_index(word_size); + assert(index < HumongousIndex, "No humongous list"); + return free_chunks(index); +} + +// Helper for chunk splitting: given a target chunk size and a larger free chunk, +// split up the larger chunk into n smaller chunks, at least one of which should be +// the target chunk of target chunk size. The smaller chunks, including the target +// chunk, are returned to the freelist. The pointer to the target chunk is returned. +// Note that this chunk is supposed to be removed from the freelist right away. +Metachunk* ChunkManager::split_chunk(size_t target_chunk_word_size, Metachunk* larger_chunk) { + assert(larger_chunk->word_size() > target_chunk_word_size, "Sanity"); + + const ChunkIndex larger_chunk_index = larger_chunk->get_chunk_type(); + const ChunkIndex target_chunk_index = get_chunk_type_by_size(target_chunk_word_size, is_class()); + + MetaWord* const region_start = (MetaWord*)larger_chunk; + const size_t region_word_len = larger_chunk->word_size(); + MetaWord* const region_end = region_start + region_word_len; + VirtualSpaceNode* const vsn = larger_chunk->container(); + OccupancyMap* const ocmap = vsn->occupancy_map(); + + // Any larger non-humongous chunk size is a multiple of any smaller chunk size. + // Since non-humongous chunks are aligned to their chunk size, the larger chunk should start + // at an address suitable to place the smaller target chunk. + assert_is_aligned(region_start, target_chunk_word_size); + + // Remove old chunk. + free_chunks(larger_chunk_index)->remove_chunk(larger_chunk); + larger_chunk->remove_sentinel(); + + // Prevent access to the old chunk from here on. + larger_chunk = NULL; + // ... and wipe it. + DEBUG_ONLY(memset(region_start, 0xfe, region_word_len * BytesPerWord)); + + // In its place create first the target chunk... + MetaWord* p = region_start; + Metachunk* target_chunk = ::new (p) Metachunk(target_chunk_index, is_class(), target_chunk_word_size, vsn); + assert(target_chunk == (Metachunk*)p, "Sanity"); + target_chunk->set_origin(origin_split); + + // Note: we do not need to mark its start in the occupancy map + // because it coincides with the old chunk start. + + // Mark chunk as free and return to the freelist. + do_update_in_use_info_for_chunk(target_chunk, false); + free_chunks(target_chunk_index)->return_chunk_at_head(target_chunk); + + // This chunk should now be valid and can be verified. + DEBUG_ONLY(do_verify_chunk(target_chunk)); + + // In the remaining space create the remainder chunks. + p += target_chunk->word_size(); + assert(p < region_end, "Sanity"); + + while (p < region_end) { + + // Find the largest chunk size which fits the alignment requirements at address p. + ChunkIndex this_chunk_index = prev_chunk_index(larger_chunk_index); + size_t this_chunk_word_size = 0; + for(;;) { + this_chunk_word_size = get_size_for_nonhumongous_chunktype(this_chunk_index, is_class()); + if (is_aligned(p, this_chunk_word_size * BytesPerWord)) { + break; + } else { + this_chunk_index = prev_chunk_index(this_chunk_index); + assert(this_chunk_index >= target_chunk_index, "Sanity"); + } + } + + assert(this_chunk_word_size >= target_chunk_word_size, "Sanity"); + assert(is_aligned(p, this_chunk_word_size * BytesPerWord), "Sanity"); + assert(p + this_chunk_word_size <= region_end, "Sanity"); + + // Create splitting chunk. + Metachunk* this_chunk = ::new (p) Metachunk(this_chunk_index, is_class(), this_chunk_word_size, vsn); + assert(this_chunk == (Metachunk*)p, "Sanity"); + this_chunk->set_origin(origin_split); + ocmap->set_chunk_starts_at_address(p, true); + do_update_in_use_info_for_chunk(this_chunk, false); + + // This chunk should be valid and can be verified. + DEBUG_ONLY(do_verify_chunk(this_chunk)); + + // Return this chunk to freelist and correct counter. + free_chunks(this_chunk_index)->return_chunk_at_head(this_chunk); + _free_chunks_count ++; + + log_trace(gc, metaspace, freelist)("Created chunk at " PTR_FORMAT ", word size " + SIZE_FORMAT_HEX " (%s), in split region [" PTR_FORMAT "..." PTR_FORMAT ").", + p2i(this_chunk), this_chunk->word_size(), chunk_size_name(this_chunk_index), + p2i(region_start), p2i(region_end)); + + p += this_chunk_word_size; + + } + + return target_chunk; +} + +Metachunk* ChunkManager::free_chunks_get(size_t word_size) { + assert_lock_strong(MetaspaceExpand_lock); + + slow_locked_verify(); + + Metachunk* chunk = NULL; + bool we_did_split_a_chunk = false; + + if (list_index(word_size) != HumongousIndex) { + + ChunkList* free_list = find_free_chunks_list(word_size); + assert(free_list != NULL, "Sanity check"); + + chunk = free_list->head(); + + if (chunk == NULL) { + // Split large chunks into smaller chunks if there are no smaller chunks, just large chunks. + // This is the counterpart of the coalescing-upon-chunk-return. + + ChunkIndex target_chunk_index = get_chunk_type_by_size(word_size, is_class()); + + // Is there a larger chunk we could split? + Metachunk* larger_chunk = NULL; + ChunkIndex larger_chunk_index = next_chunk_index(target_chunk_index); + while (larger_chunk == NULL && larger_chunk_index < NumberOfFreeLists) { + larger_chunk = free_chunks(larger_chunk_index)->head(); + if (larger_chunk == NULL) { + larger_chunk_index = next_chunk_index(larger_chunk_index); + } + } + + if (larger_chunk != NULL) { + assert(larger_chunk->word_size() > word_size, "Sanity"); + assert(larger_chunk->get_chunk_type() == larger_chunk_index, "Sanity"); + + // We found a larger chunk. Lets split it up: + // - remove old chunk + // - in its place, create new smaller chunks, with at least one chunk + // being of target size, the others sized as large as possible. This + // is to make sure the resulting chunks are "as coalesced as possible" + // (similar to VirtualSpaceNode::retire()). + // Note: during this operation both ChunkManager and VirtualSpaceNode + // are temporarily invalid, so be careful with asserts. + + log_trace(gc, metaspace, freelist)("%s: splitting chunk " PTR_FORMAT + ", word size " SIZE_FORMAT_HEX " (%s), to get a chunk of word size " SIZE_FORMAT_HEX " (%s)...", + (is_class() ? "class space" : "metaspace"), p2i(larger_chunk), larger_chunk->word_size(), + chunk_size_name(larger_chunk_index), word_size, chunk_size_name(target_chunk_index)); + + chunk = split_chunk(word_size, larger_chunk); + + // This should have worked. + assert(chunk != NULL, "Sanity"); + assert(chunk->word_size() == word_size, "Sanity"); + assert(chunk->is_tagged_free(), "Sanity"); + + we_did_split_a_chunk = true; + + } + } + + if (chunk == NULL) { + return NULL; + } + + // Remove the chunk as the head of the list. + free_list->remove_chunk(chunk); + + log_trace(gc, metaspace, freelist)("ChunkManager::free_chunks_get: free_list: " PTR_FORMAT " chunks left: " SSIZE_FORMAT ".", + p2i(free_list), free_list->count()); + + } else { + chunk = humongous_dictionary()->get_chunk(word_size); + + if (chunk == NULL) { + return NULL; + } + + log_debug(gc, metaspace, alloc)("Free list allocate humongous chunk size " SIZE_FORMAT " for requested size " SIZE_FORMAT " waste " SIZE_FORMAT, + chunk->word_size(), word_size, chunk->word_size() - word_size); + } + + // Chunk has been removed from the chunk manager; update counters. + account_for_removed_chunk(chunk); + do_update_in_use_info_for_chunk(chunk, true); + chunk->container()->inc_container_count(); + chunk->inc_use_count(); + + // Remove it from the links to this freelist + chunk->set_next(NULL); + chunk->set_prev(NULL); + + // Run some verifications (some more if we did a chunk split) +#ifdef ASSERT + if (VerifyMetaspace) { + locked_verify(); + VirtualSpaceNode* const vsn = chunk->container(); + vsn->verify(); + if (we_did_split_a_chunk) { + vsn->verify_free_chunks_are_ideally_merged(); + } + } +#endif + + return chunk; +} + +Metachunk* ChunkManager::chunk_freelist_allocate(size_t word_size) { + assert_lock_strong(MetaspaceExpand_lock); + slow_locked_verify(); + + // Take from the beginning of the list + Metachunk* chunk = free_chunks_get(word_size); + if (chunk == NULL) { + return NULL; + } + + assert((word_size <= chunk->word_size()) || + (list_index(chunk->word_size()) == HumongousIndex), + "Non-humongous variable sized chunk"); + LogTarget(Debug, gc, metaspace, freelist) lt; + if (lt.is_enabled()) { + size_t list_count; + if (list_index(word_size) < HumongousIndex) { + ChunkList* list = find_free_chunks_list(word_size); + list_count = list->count(); + } else { + list_count = humongous_dictionary()->total_count(); + } + LogStream ls(lt); + ls.print("ChunkManager::chunk_freelist_allocate: " PTR_FORMAT " chunk " PTR_FORMAT " size " SIZE_FORMAT " count " SIZE_FORMAT " ", + p2i(this), p2i(chunk), chunk->word_size(), list_count); + ResourceMark rm; + locked_print_free_chunks(&ls); + } + + return chunk; +} + +void ChunkManager::return_single_chunk(Metachunk* chunk) { + const ChunkIndex index = chunk->get_chunk_type(); + assert_lock_strong(MetaspaceExpand_lock); + DEBUG_ONLY(do_verify_chunk(chunk);) + assert(chunk != NULL, "Expected chunk."); + assert(chunk->container() != NULL, "Container should have been set."); + assert(chunk->is_tagged_free() == false, "Chunk should be in use."); + index_bounds_check(index); + + // Note: mangle *before* returning the chunk to the freelist or dictionary. It does not + // matter for the freelist (non-humongous chunks), but the humongous chunk dictionary + // keeps tree node pointers in the chunk payload area which mangle will overwrite. + DEBUG_ONLY(chunk->mangle(badMetaWordVal);) + + if (index != HumongousIndex) { + // Return non-humongous chunk to freelist. + ChunkList* list = free_chunks(index); + assert(list->size() == chunk->word_size(), "Wrong chunk type."); + list->return_chunk_at_head(chunk); + log_trace(gc, metaspace, freelist)("returned one %s chunk at " PTR_FORMAT " to freelist.", + chunk_size_name(index), p2i(chunk)); + } else { + // Return humongous chunk to dictionary. + assert(chunk->word_size() > free_chunks(MediumIndex)->size(), "Wrong chunk type."); + assert(chunk->word_size() % free_chunks(SpecializedIndex)->size() == 0, + "Humongous chunk has wrong alignment."); + _humongous_dictionary.return_chunk(chunk); + log_trace(gc, metaspace, freelist)("returned one %s chunk at " PTR_FORMAT " (word size " SIZE_FORMAT ") to freelist.", + chunk_size_name(index), p2i(chunk), chunk->word_size()); + } + chunk->container()->dec_container_count(); + do_update_in_use_info_for_chunk(chunk, false); + + // Chunk has been added; update counters. + account_for_added_chunk(chunk); + + // Attempt coalesce returned chunks with its neighboring chunks: + // if this chunk is small or special, attempt to coalesce to a medium chunk. + if (index == SmallIndex || index == SpecializedIndex) { + if (!attempt_to_coalesce_around_chunk(chunk, MediumIndex)) { + // This did not work. But if this chunk is special, we still may form a small chunk? + if (index == SpecializedIndex) { + if (!attempt_to_coalesce_around_chunk(chunk, SmallIndex)) { + // give up. + } + } + } + } + +} + +void ChunkManager::return_chunk_list(Metachunk* chunks) { + if (chunks == NULL) { + return; + } + LogTarget(Trace, gc, metaspace, freelist) log; + if (log.is_enabled()) { // tracing + log.print("returning list of chunks..."); + } + unsigned num_chunks_returned = 0; + size_t size_chunks_returned = 0; + Metachunk* cur = chunks; + while (cur != NULL) { + // Capture the next link before it is changed + // by the call to return_chunk_at_head(); + Metachunk* next = cur->next(); + if (log.is_enabled()) { // tracing + num_chunks_returned ++; + size_chunks_returned += cur->word_size(); + } + return_single_chunk(cur); + cur = next; + } + if (log.is_enabled()) { // tracing + log.print("returned %u chunks to freelist, total word size " SIZE_FORMAT ".", + num_chunks_returned, size_chunks_returned); + } +} + +void ChunkManager::collect_statistics(ChunkManagerStatistics* out) const { + MutexLockerEx cl(MetaspaceExpand_lock, Mutex::_no_safepoint_check_flag); + for (ChunkIndex i = ZeroIndex; i < NumberOfInUseLists; i = next_chunk_index(i)) { + out->chunk_stats(i).add(num_free_chunks(i), size_free_chunks_in_bytes(i) / sizeof(MetaWord)); + } +} + +} // namespace metaspace + + + diff --git a/src/hotspot/share/memory/metaspace/chunkManager.hpp b/src/hotspot/share/memory/metaspace/chunkManager.hpp new file mode 100644 --- /dev/null +++ b/src/hotspot/share/memory/metaspace/chunkManager.hpp @@ -0,0 +1,229 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_MEMORY_METASPACE_CHUNKMANAGER_HPP +#define SHARE_MEMORY_METASPACE_CHUNKMANAGER_HPP + +#include "memory/allocation.hpp" +#include "memory/binaryTreeDictionary.hpp" +#include "memory/freeList.inline.hpp" +#include "memory/metaspace/metachunk.hpp" +#include "memory/metaspace/metaspaceStatistics.hpp" +#include "memory/metaspaceChunkFreeListSummary.hpp" +#include "utilities/globalDefinitions.hpp" + +class TestVirtualSpaceNodeTest; + +namespace metaspace { + +typedef class FreeList ChunkList; +typedef BinaryTreeDictionary > ChunkTreeDictionary; + +// Manages the global free lists of chunks. +class ChunkManager : public CHeapObj { + friend class ::TestVirtualSpaceNodeTest; + + // Free list of chunks of different sizes. + // SpecializedChunk + // SmallChunk + // MediumChunk + ChunkList _free_chunks[NumberOfFreeLists]; + + // Whether or not this is the class chunkmanager. + const bool _is_class; + + // Return non-humongous chunk list by its index. + ChunkList* free_chunks(ChunkIndex index); + + // Returns non-humongous chunk list for the given chunk word size. + ChunkList* find_free_chunks_list(size_t word_size); + + // HumongousChunk + ChunkTreeDictionary _humongous_dictionary; + + // Returns the humongous chunk dictionary. + ChunkTreeDictionary* humongous_dictionary() { + return &_humongous_dictionary; + } + + // Size, in metaspace words, of all chunks managed by this ChunkManager + size_t _free_chunks_total; + // Number of chunks in this ChunkManager + size_t _free_chunks_count; + + // Update counters after a chunk was added or removed removed. + void account_for_added_chunk(const Metachunk* c); + void account_for_removed_chunk(const Metachunk* c); + + size_t sum_free_chunks(); + size_t sum_free_chunks_count(); + + void locked_verify_free_chunks_total(); + void slow_locked_verify_free_chunks_total() { + if (VerifyMetaspace) { + locked_verify_free_chunks_total(); + } + } + void locked_verify_free_chunks_count(); + void slow_locked_verify_free_chunks_count() { + if (VerifyMetaspace) { + locked_verify_free_chunks_count(); + } + } + + // Given a pointer to a chunk, attempts to merge it with neighboring + // free chunks to form a bigger chunk. Returns true if successful. + bool attempt_to_coalesce_around_chunk(Metachunk* chunk, ChunkIndex target_chunk_type); + + // Helper for chunk merging: + // Given an address range with 1-n chunks which are all supposed to be + // free and hence currently managed by this ChunkManager, remove them + // from this ChunkManager and mark them as invalid. + // - This does not correct the occupancy map. + // - This does not adjust the counters in ChunkManager. + // - Does not adjust container count counter in containing VirtualSpaceNode. + // Returns number of chunks removed. + int remove_chunks_in_area(MetaWord* p, size_t word_size); + + // Helper for chunk splitting: given a target chunk size and a larger free chunk, + // split up the larger chunk into n smaller chunks, at least one of which should be + // the target chunk of target chunk size. The smaller chunks, including the target + // chunk, are returned to the freelist. The pointer to the target chunk is returned. + // Note that this chunk is supposed to be removed from the freelist right away. + Metachunk* split_chunk(size_t target_chunk_word_size, Metachunk* chunk); + + public: + + ChunkManager(bool is_class) + : _is_class(is_class), _free_chunks_total(0), _free_chunks_count(0) { + _free_chunks[SpecializedIndex].set_size(get_size_for_nonhumongous_chunktype(SpecializedIndex, is_class)); + _free_chunks[SmallIndex].set_size(get_size_for_nonhumongous_chunktype(SmallIndex, is_class)); + _free_chunks[MediumIndex].set_size(get_size_for_nonhumongous_chunktype(MediumIndex, is_class)); + } + + // Add or delete (return) a chunk to the global freelist. + Metachunk* chunk_freelist_allocate(size_t word_size); + + // Map a size to a list index assuming that there are lists + // for special, small, medium, and humongous chunks. + ChunkIndex list_index(size_t size); + + // Map a given index to the chunk size. + size_t size_by_index(ChunkIndex index) const; + + bool is_class() const { return _is_class; } + + // Convenience accessors. + size_t medium_chunk_word_size() const { return size_by_index(MediumIndex); } + size_t small_chunk_word_size() const { return size_by_index(SmallIndex); } + size_t specialized_chunk_word_size() const { return size_by_index(SpecializedIndex); } + + // Take a chunk from the ChunkManager. The chunk is expected to be in + // the chunk manager (the freelist if non-humongous, the dictionary if + // humongous). + void remove_chunk(Metachunk* chunk); + + // Return a single chunk of type index to the ChunkManager. + void return_single_chunk(Metachunk* chunk); + + // Add the simple linked list of chunks to the freelist of chunks + // of type index. + void return_chunk_list(Metachunk* chunk); + + // Total of the space in the free chunks list + size_t free_chunks_total_words(); + size_t free_chunks_total_bytes(); + + // Number of chunks in the free chunks list + size_t free_chunks_count(); + + // Remove from a list by size. Selects list based on size of chunk. + Metachunk* free_chunks_get(size_t chunk_word_size); + +#define index_bounds_check(index) \ + assert(is_valid_chunktype(index), "Bad index: %d", (int) index) + + size_t num_free_chunks(ChunkIndex index) const { + index_bounds_check(index); + + if (index == HumongousIndex) { + return _humongous_dictionary.total_free_blocks(); + } + + ssize_t count = _free_chunks[index].count(); + return count == -1 ? 0 : (size_t) count; + } + + size_t size_free_chunks_in_bytes(ChunkIndex index) const { + index_bounds_check(index); + + size_t word_size = 0; + if (index == HumongousIndex) { + word_size = _humongous_dictionary.total_size(); + } else { + const size_t size_per_chunk_in_words = _free_chunks[index].size(); + word_size = size_per_chunk_in_words * num_free_chunks(index); + } + + return word_size * BytesPerWord; + } + + MetaspaceChunkFreeListSummary chunk_free_list_summary() const { + return MetaspaceChunkFreeListSummary(num_free_chunks(SpecializedIndex), + num_free_chunks(SmallIndex), + num_free_chunks(MediumIndex), + num_free_chunks(HumongousIndex), + size_free_chunks_in_bytes(SpecializedIndex), + size_free_chunks_in_bytes(SmallIndex), + size_free_chunks_in_bytes(MediumIndex), + size_free_chunks_in_bytes(HumongousIndex)); + } + + // Debug support + void verify(); + void slow_verify() { + if (VerifyMetaspace) { + verify(); + } + } + void locked_verify(); + void slow_locked_verify() { + if (VerifyMetaspace) { + locked_verify(); + } + } + + void locked_print_free_chunks(outputStream* st); + void locked_print_sum_free_chunks(outputStream* st); + + // Fill in current statistic values to the given statistics object. + void collect_statistics(ChunkManagerStatistics* out) const; + +}; + +} // namespace metaspace + + +#endif /* SHARE_MEMORY_METASPACE_CHUNKMANAGER_HPP_ */ + diff --git a/src/hotspot/share/memory/metaspace/metaDebug.cpp b/src/hotspot/share/memory/metaspace/metaDebug.cpp new file mode 100644 --- /dev/null +++ b/src/hotspot/share/memory/metaspace/metaDebug.cpp @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "logging/log.hpp" +#include "memory/metaspace/metaDebug.hpp" +#include "runtime/os.hpp" +#include "runtime/thread.hpp" +#include "utilities/debug.hpp" +#include "utilities/globalDefinitions.hpp" + +namespace metaspace { + +int Metadebug::_allocation_fail_alot_count = 0; + +void Metadebug::init_allocation_fail_alot_count() { + if (MetadataAllocationFailALot) { + _allocation_fail_alot_count = + 1+(long)((double)MetadataAllocationFailALotInterval*os::random()/(max_jint+1.0)); + } +} + +#ifdef ASSERT +bool Metadebug::test_metadata_failure() { + if (MetadataAllocationFailALot && + Threads::is_vm_complete()) { + if (_allocation_fail_alot_count > 0) { + _allocation_fail_alot_count--; + } else { + log_trace(gc, metaspace, freelist)("Metadata allocation failing for MetadataAllocationFailALot"); + init_allocation_fail_alot_count(); + return true; + } + } + return false; +} +#endif + + +} // namespace metaspace + + diff --git a/src/hotspot/share/memory/metaspace/metaDebug.hpp b/src/hotspot/share/memory/metaspace/metaDebug.hpp new file mode 100644 --- /dev/null +++ b/src/hotspot/share/memory/metaspace/metaDebug.hpp @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_MEMORY_METASPACE_METADEBUG_HPP_ +#define SHARE_MEMORY_METASPACE_METADEBUG_HPP_ + +#include "memory/allocation.hpp" + +namespace metaspace { + +class Metadebug : AllStatic { + // Debugging support for Metaspaces + static int _allocation_fail_alot_count; + + public: + + static void init_allocation_fail_alot_count(); +#ifdef ASSERT + static bool test_metadata_failure(); +#endif +}; + +} // namespace metaspace + +#endif /* SHARE_MEMORY_METASPACE_METADEBUG_HPP_ */ + diff --git a/src/hotspot/share/memory/metaspace/metabase.hpp b/src/hotspot/share/memory/metaspace/metabase.hpp new file mode 100644 --- /dev/null +++ b/src/hotspot/share/memory/metaspace/metabase.hpp @@ -0,0 +1,82 @@ +/* + * Copyright (c) 2012, 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_MEMORY_METASPACE_METABASE_HPP_ +#define SHARE_MEMORY_METASPACE_METABASE_HPP_ + +#include "utilities/globalDefinitions.hpp" + +namespace metaspace { + +// Super class of Metablock and Metachunk to allow them to +// be put on the FreeList and in the BinaryTreeDictionary. +template +class Metabase { + size_t _word_size; + T* _next; + T* _prev; + + protected: + Metabase(size_t word_size) : _word_size(word_size), _next(NULL), _prev(NULL) {} + + public: + T* next() const { return _next; } + T* prev() const { return _prev; } + void set_next(T* v) { _next = v; assert(v != this, "Boom");} + void set_prev(T* v) { _prev = v; assert(v != this, "Boom");} + void clear_next() { set_next(NULL); } + void clear_prev() { set_prev(NULL); } + + size_t size() const volatile { return _word_size; } + void set_size(size_t v) { _word_size = v; } + + void link_next(T* ptr) { set_next(ptr); } + void link_prev(T* ptr) { set_prev(ptr); } + void link_after(T* ptr) { + link_next(ptr); + if (ptr != NULL) ptr->link_prev((T*)this); + } + + uintptr_t* end() const { return ((uintptr_t*) this) + size(); } + + bool cantCoalesce() const { return false; } + + // Debug support +#ifdef ASSERT + void* prev_addr() const { return (void*)&_prev; } + void* next_addr() const { return (void*)&_next; } + void* size_addr() const { return (void*)&_word_size; } +#endif + bool verify_chunk_in_free_list(T* tc) const { return true; } + bool verify_par_locked() { return true; } + + void assert_is_mangled() const {/* Don't check "\*/} + + bool is_free() { return true; } +}; + +} // namespace metaspace + +#endif /* SHARE_MEMORY_METASPACE_METABASE_HPP_ */ + diff --git a/src/hotspot/share/memory/metaspace/metablock.hpp b/src/hotspot/share/memory/metaspace/metablock.hpp new file mode 100644 --- /dev/null +++ b/src/hotspot/share/memory/metaspace/metablock.hpp @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2012, 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_MEMORY_METASPACE_METABLOCK_HPP_ +#define SHARE_MEMORY_METASPACE_METABLOCK_HPP_ + +#include "memory/metaspace/metabase.hpp" +#include "utilities/globalDefinitions.hpp" + +namespace metaspace { + +// Metablock is the unit of allocation from a Chunk. +// +// A Metablock may be reused by its SpaceManager but are never moved between +// SpaceManagers. There is no explicit link to the Metachunk +// from which it was allocated. Metablock may be deallocated and +// put on a freelist but the space is never freed, rather +// the Metachunk it is a part of will be deallocated when it's +// associated class loader is collected. + +class Metablock : public Metabase { + friend class VMStructs; + public: + Metablock(size_t word_size) : Metabase(word_size) {} +}; + +} // namespace metaspace + +#endif /* SHARE_MEMORY_METASPACE_METABLOCK_HPP_ */ + diff --git a/src/hotspot/share/memory/metachunk.cpp b/src/hotspot/share/memory/metaspace/metachunk.cpp rename from src/hotspot/share/memory/metachunk.cpp rename to src/hotspot/share/memory/metaspace/metachunk.cpp --- a/src/hotspot/share/memory/metachunk.cpp +++ b/src/hotspot/share/memory/metaspace/metachunk.cpp @@ -24,12 +24,14 @@ #include "precompiled.hpp" #include "memory/allocation.hpp" -#include "memory/metachunk.hpp" +#include "memory/metaspace/metachunk.hpp" +#include "memory/metaspace/occupancyMap.hpp" +#include "memory/metaspace/virtualSpaceNode.hpp" #include "utilities/align.hpp" #include "utilities/copy.hpp" #include "utilities/debug.hpp" -class VirtualSpaceNode; +namespace metaspace { size_t Metachunk::object_alignment() { // Must align pointers and sizes to 8, @@ -147,3 +149,24 @@ return "Invalid index"; } } + +#ifdef ASSERT +void do_verify_chunk(Metachunk* chunk) { + guarantee(chunk != NULL, "Sanity"); + // Verify chunk itself; then verify that it is consistent with the + // occupany map of its containing node. + chunk->verify(); + VirtualSpaceNode* const vsn = chunk->container(); + OccupancyMap* const ocmap = vsn->occupancy_map(); + ocmap->verify_for_chunk(chunk); +} +#endif + +void do_update_in_use_info_for_chunk(Metachunk* chunk, bool inuse) { + chunk->set_is_tagged_free(!inuse); + OccupancyMap* const ocmap = chunk->container()->occupancy_map(); + ocmap->set_region_in_use((MetaWord*)chunk, chunk->word_size(), inuse); +} + +} // namespace metaspace + diff --git a/src/hotspot/share/memory/metachunk.hpp b/src/hotspot/share/memory/metaspace/metachunk.hpp rename from src/hotspot/share/memory/metachunk.hpp rename to src/hotspot/share/memory/metaspace/metachunk.hpp --- a/src/hotspot/share/memory/metachunk.hpp +++ b/src/hotspot/share/memory/metaspace/metachunk.hpp @@ -24,58 +24,17 @@ #ifndef SHARE_VM_MEMORY_METACHUNK_HPP #define SHARE_VM_MEMORY_METACHUNK_HPP +#include "memory/metaspace/metabase.hpp" +#include "memory/metaspace/metaspaceCommon.hpp" #include "utilities/debug.hpp" #include "utilities/globalDefinitions.hpp" +class MetachunkTest; + +namespace metaspace { + class VirtualSpaceNode; -// Super class of Metablock and Metachunk to allow them to -// be put on the FreeList and in the BinaryTreeDictionary. -template -class Metabase { - size_t _word_size; - T* _next; - T* _prev; - - protected: - Metabase(size_t word_size) : _word_size(word_size), _next(NULL), _prev(NULL) {} - - public: - T* next() const { return _next; } - T* prev() const { return _prev; } - void set_next(T* v) { _next = v; assert(v != this, "Boom");} - void set_prev(T* v) { _prev = v; assert(v != this, "Boom");} - void clear_next() { set_next(NULL); } - void clear_prev() { set_prev(NULL); } - - size_t size() const volatile { return _word_size; } - void set_size(size_t v) { _word_size = v; } - - void link_next(T* ptr) { set_next(ptr); } - void link_prev(T* ptr) { set_prev(ptr); } - void link_after(T* ptr) { - link_next(ptr); - if (ptr != NULL) ptr->link_prev((T*)this); - } - - uintptr_t* end() const { return ((uintptr_t*) this) + size(); } - - bool cantCoalesce() const { return false; } - - // Debug support -#ifdef ASSERT - void* prev_addr() const { return (void*)&_prev; } - void* next_addr() const { return (void*)&_next; } - void* size_addr() const { return (void*)&_word_size; } -#endif - bool verify_chunk_in_free_list(T* tc) const { return true; } - bool verify_par_locked() { return true; } - - void assert_is_mangled() const {/* Don't check "\*/} - - bool is_free() { return true; } -}; - // Metachunk - Quantum of allocation from a Virtualspace // Metachunks are reused (when freed are put on a global freelist) and // have no permanent association to a SpaceManager. @@ -93,39 +52,6 @@ // | | | | // +--------------+ <- bottom --+ --+ -// ChunkIndex defines the type of chunk. -// Chunk types differ by size: specialized < small < medium, chunks -// larger than medium are humongous chunks of varying size. -enum ChunkIndex { - ZeroIndex = 0, - SpecializedIndex = ZeroIndex, - SmallIndex = SpecializedIndex + 1, - MediumIndex = SmallIndex + 1, - HumongousIndex = MediumIndex + 1, - NumberOfFreeLists = 3, - NumberOfInUseLists = 4 -}; - -// Utility functions. -size_t get_size_for_nonhumongous_chunktype(ChunkIndex chunk_type, bool is_class); -ChunkIndex get_chunk_type_by_size(size_t size, bool is_class); - -ChunkIndex next_chunk_index(ChunkIndex i); -ChunkIndex prev_chunk_index(ChunkIndex i); - -// Returns a descriptive name for a chunk type. -const char* chunk_size_name(ChunkIndex index); - -// Verify chunk type. -inline bool is_valid_chunktype(ChunkIndex index) { - return index == SpecializedIndex || index == SmallIndex || - index == MediumIndex || index == HumongousIndex; -} - -inline bool is_valid_nonhumongous_chunktype(ChunkIndex index) { - return is_valid_chunktype(index) && index != HumongousIndex; -} - enum ChunkOrigin { // Chunk normally born (via take_from_committed) origin_normal = 1, @@ -152,7 +78,9 @@ } class Metachunk : public Metabase { - friend class MetachunkTest; + + friend class ::MetachunkTest; + // The VirtualSpaceNode containing this chunk. VirtualSpaceNode* const _container; @@ -232,19 +160,14 @@ }; -// Metablock is the unit of allocation from a Chunk. -// -// A Metablock may be reused by its SpaceManager but are never moved between -// SpaceManagers. There is no explicit link to the Metachunk -// from which it was allocated. Metablock may be deallocated and -// put on a freelist but the space is never freed, rather -// the Metachunk it is a part of will be deallocated when it's -// associated class loader is collected. -class Metablock : public Metabase { - friend class VMStructs; - public: - Metablock(size_t word_size) : Metabase(word_size) {} -}; +// Helper function that does a bunch of checks for a chunk. +DEBUG_ONLY(void do_verify_chunk(Metachunk* chunk);) + +// Given a Metachunk, update its in-use information (both in the +// chunk and the occupancy map). +void do_update_in_use_info_for_chunk(Metachunk* chunk, bool inuse); + +} // namespace metaspace #endif // SHARE_VM_MEMORY_METACHUNK_HPP diff --git a/src/hotspot/share/memory/metaspace/metaspaceCommon.cpp b/src/hotspot/share/memory/metaspace/metaspaceCommon.cpp --- a/src/hotspot/share/memory/metaspace/metaspaceCommon.cpp +++ b/src/hotspot/share/memory/metaspace/metaspaceCommon.cpp @@ -26,11 +26,14 @@ #include "precompiled.hpp" #include "memory/metaspace/metaspaceCommon.hpp" +#include "memory/metaspace/virtualSpaceNode.hpp" +#include "utilities/debug.hpp" #include "utilities/globalDefinitions.hpp" #include "utilities/ostream.hpp" namespace metaspace { -namespace internals { + +DEBUG_ONLY(internal_statistics_t g_internal_statistics;) // Print a size, in words, scaled. void print_scaled_words(outputStream* st, size_t word_size, size_t scale, int width) { @@ -128,5 +131,70 @@ } } -} // namespace internals +// Returns size of this chunk type. +size_t get_size_for_nonhumongous_chunktype(ChunkIndex chunktype, bool is_class) { + assert(is_valid_nonhumongous_chunktype(chunktype), "invalid chunk type."); + size_t size = 0; + if (is_class) { + switch(chunktype) { + case SpecializedIndex: size = ClassSpecializedChunk; break; + case SmallIndex: size = ClassSmallChunk; break; + case MediumIndex: size = ClassMediumChunk; break; + default: + ShouldNotReachHere(); + } + } else { + switch(chunktype) { + case SpecializedIndex: size = SpecializedChunk; break; + case SmallIndex: size = SmallChunk; break; + case MediumIndex: size = MediumChunk; break; + default: + ShouldNotReachHere(); + } + } + return size; +} + +ChunkIndex get_chunk_type_by_size(size_t size, bool is_class) { + if (is_class) { + if (size == ClassSpecializedChunk) { + return SpecializedIndex; + } else if (size == ClassSmallChunk) { + return SmallIndex; + } else if (size == ClassMediumChunk) { + return MediumIndex; + } else if (size > ClassMediumChunk) { + // A valid humongous chunk size is a multiple of the smallest chunk size. + assert(is_aligned(size, ClassSpecializedChunk), "Invalid chunk size"); + return HumongousIndex; + } + } else { + if (size == SpecializedChunk) { + return SpecializedIndex; + } else if (size == SmallChunk) { + return SmallIndex; + } else if (size == MediumChunk) { + return MediumIndex; + } else if (size > MediumChunk) { + // A valid humongous chunk size is a multiple of the smallest chunk size. + assert(is_aligned(size, SpecializedChunk), "Invalid chunk size"); + return HumongousIndex; + } + } + ShouldNotReachHere(); + return (ChunkIndex)-1; +} + +ChunkIndex next_chunk_index(ChunkIndex i) { + assert(i < NumberOfInUseLists, "Out of bound"); + return (ChunkIndex) (i+1); +} + +ChunkIndex prev_chunk_index(ChunkIndex i) { + assert(i > ZeroIndex, "Out of bound"); + return (ChunkIndex) (i-1); +} + + } // namespace metaspace + diff --git a/src/hotspot/share/memory/metaspace/metaspaceCommon.hpp b/src/hotspot/share/memory/metaspace/metaspaceCommon.hpp --- a/src/hotspot/share/memory/metaspace/metaspaceCommon.hpp +++ b/src/hotspot/share/memory/metaspace/metaspaceCommon.hpp @@ -25,13 +25,21 @@ #ifndef SHARE_MEMORY_METASPACE_METASPACECOMMON_HPP_ #define SHARE_MEMORY_METASPACE_METASPACECOMMON_HPP_ +#include "utilities/debug.hpp" #include "utilities/globalDefinitions.hpp" - class outputStream; namespace metaspace { -namespace internals { + +enum ChunkSizes { // in words. + ClassSpecializedChunk = 128, + SpecializedChunk = 128, + ClassSmallChunk = 256, + SmallChunk = 512, + ClassMediumChunk = 4 * K, + MediumChunk = 8 * K +}; // Print a size, in words, scaled. void print_scaled_words(outputStream* st, size_t word_size, size_t scale = 0, int width = -1); @@ -50,7 +58,70 @@ // larger than 99% but not 100% are displayed as ">100%". void print_percentage(outputStream* st, size_t total, size_t part); -} // namespace internals + +#define assert_is_aligned(value, alignment) \ + assert(is_aligned((value), (alignment)), \ + SIZE_FORMAT_HEX " is not aligned to " \ + SIZE_FORMAT, (size_t)(uintptr_t)value, (alignment)) + +// Internal statistics. +#ifdef ASSERT +struct internal_statistics_t { + // Number of allocations. + uintx num_allocs; + // Number of times a ClassLoaderMetaspace was born... + uintx num_metaspace_births; + // ... and died. + uintx num_metaspace_deaths; + // Number of times VirtualSpaceListNodes were created... + uintx num_vsnodes_created; + // ... and purged. + uintx num_vsnodes_purged; + // Number of times we expanded the committed section of the space. + uintx num_committed_space_expanded; + // Number of deallocations + uintx num_deallocs; + // Number of deallocations triggered from outside ("real" deallocations). + uintx num_external_deallocs; + // Number of times an allocation was satisfied from deallocated blocks. + uintx num_allocs_from_deallocated_blocks; +}; +extern internal_statistics_t g_internal_statistics; +#endif + +// ChunkIndex defines the type of chunk. +// Chunk types differ by size: specialized < small < medium, chunks +// larger than medium are humongous chunks of varying size. +enum ChunkIndex { + ZeroIndex = 0, + SpecializedIndex = ZeroIndex, + SmallIndex = SpecializedIndex + 1, + MediumIndex = SmallIndex + 1, + HumongousIndex = MediumIndex + 1, + NumberOfFreeLists = 3, + NumberOfInUseLists = 4 +}; + +// Utility functions. +size_t get_size_for_nonhumongous_chunktype(ChunkIndex chunk_type, bool is_class); +ChunkIndex get_chunk_type_by_size(size_t size, bool is_class); + +ChunkIndex next_chunk_index(ChunkIndex i); +ChunkIndex prev_chunk_index(ChunkIndex i); +// Returns a descriptive name for a chunk type. +const char* chunk_size_name(ChunkIndex index); + +// Verify chunk type. +inline bool is_valid_chunktype(ChunkIndex index) { + return index == SpecializedIndex || index == SmallIndex || + index == MediumIndex || index == HumongousIndex; +} + +inline bool is_valid_nonhumongous_chunktype(ChunkIndex index) { + return is_valid_chunktype(index) && index != HumongousIndex; +} + } // namespace metaspace #endif /* SHARE_MEMORY_METASPACE_METASPACESTATISTICS_HPP_ */ + diff --git a/src/hotspot/share/memory/metaspace/metaspaceStatistics.cpp b/src/hotspot/share/memory/metaspace/metaspaceStatistics.cpp --- a/src/hotspot/share/memory/metaspace/metaspaceStatistics.cpp +++ b/src/hotspot/share/memory/metaspace/metaspaceStatistics.cpp @@ -24,7 +24,7 @@ */ #include "precompiled.hpp" -#include "memory/metachunk.hpp" +#include "memory/metaspace/metachunk.hpp" #include "memory/metaspace/metaspaceCommon.hpp" #include "memory/metaspace/metaspaceStatistics.hpp" #include "utilities/debug.hpp" @@ -32,7 +32,6 @@ #include "utilities/ostream.hpp" namespace metaspace { -namespace internals { // FreeChunksStatistics methods @@ -271,8 +270,7 @@ st->cr(); } - -} // end namespace internals } // end namespace metaspace + diff --git a/src/hotspot/share/memory/metaspace/metaspaceStatistics.hpp b/src/hotspot/share/memory/metaspace/metaspaceStatistics.hpp --- a/src/hotspot/share/memory/metaspace/metaspaceStatistics.hpp +++ b/src/hotspot/share/memory/metaspace/metaspaceStatistics.hpp @@ -27,13 +27,12 @@ #define SHARE_MEMORY_METASPACE_METASPACESTATISTICS_HPP_ #include "utilities/globalDefinitions.hpp" -#include "memory/metachunk.hpp" // for ChunkIndex enum #include "memory/metaspace.hpp" // for MetadataType enum +#include "memory/metaspace/metachunk.hpp" // for ChunkIndex enum class outputStream; namespace metaspace { -namespace internals { // Contains statistics for a number of free chunks. class FreeChunksStatistics { @@ -183,7 +182,7 @@ }; // ClassLoaderMetaspaceStatistics -} // namespace internals } // namespace metaspace #endif /* SHARE_MEMORY_METASPACE_METASPACESTATISTICS_HPP_ */ + diff --git a/src/hotspot/share/memory/metaspace/occupancyMap.cpp b/src/hotspot/share/memory/metaspace/occupancyMap.cpp new file mode 100644 --- /dev/null +++ b/src/hotspot/share/memory/metaspace/occupancyMap.cpp @@ -0,0 +1,135 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2018, SAP. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "utilities/debug.hpp" +#include "utilities/globalDefinitions.hpp" +#include "memory/metaspace/metachunk.hpp" +#include "memory/metaspace/occupancyMap.hpp" +#include "runtime/os.hpp" + +namespace metaspace { + +OccupancyMap::OccupancyMap(const MetaWord* reference_address, size_t word_size, size_t smallest_chunk_word_size) : + _reference_address(reference_address), _word_size(word_size), + _smallest_chunk_word_size(smallest_chunk_word_size) +{ + assert(reference_address != NULL, "invalid reference address"); + assert(is_aligned(reference_address, smallest_chunk_word_size), + "Reference address not aligned to smallest chunk size."); + assert(is_aligned(word_size, smallest_chunk_word_size), + "Word_size shall be a multiple of the smallest chunk size."); + // Calculate bitmap size: one bit per smallest_chunk_word_size'd area. + size_t num_bits = word_size / smallest_chunk_word_size; + _map_size = (num_bits + 7) / 8; + assert(_map_size * 8 >= num_bits, "sanity"); + _map[0] = (uint8_t*) os::malloc(_map_size, mtInternal); + _map[1] = (uint8_t*) os::malloc(_map_size, mtInternal); + assert(_map[0] != NULL && _map[1] != NULL, "Occupancy Map: allocation failed."); + memset(_map[1], 0, _map_size); + memset(_map[0], 0, _map_size); + // Sanity test: the first respectively last possible chunk start address in + // the covered range shall map to the first and last bit in the bitmap. + assert(get_bitpos_for_address(reference_address) == 0, + "First chunk address in range must map to fist bit in bitmap."); + assert(get_bitpos_for_address(reference_address + word_size - smallest_chunk_word_size) == num_bits - 1, + "Last chunk address in range must map to last bit in bitmap."); +} + +OccupancyMap::~OccupancyMap() { + os::free(_map[0]); + os::free(_map[1]); +} + +#ifdef ASSERT +// Verify occupancy map for the address range [from, to). +// We need to tell it the address range, because the memory the +// occupancy map is covering may not be fully comitted yet. +void OccupancyMap::verify(MetaWord* from, MetaWord* to) { + Metachunk* chunk = NULL; + int nth_bit_for_chunk = 0; + MetaWord* chunk_end = NULL; + for (MetaWord* p = from; p < to; p += _smallest_chunk_word_size) { + const unsigned pos = get_bitpos_for_address(p); + // Check the chunk-starts-info: + if (get_bit_at_position(pos, layer_chunk_start_map)) { + // Chunk start marked in bitmap. + chunk = (Metachunk*) p; + if (chunk_end != NULL) { + assert(chunk_end == p, "Unexpected chunk start found at %p (expected " + "the next chunk to start at %p).", p, chunk_end); + } + assert(chunk->is_valid_sentinel(), "Invalid chunk at address %p.", p); + if (chunk->get_chunk_type() != HumongousIndex) { + guarantee(is_aligned(p, chunk->word_size()), "Chunk %p not aligned.", p); + } + chunk_end = p + chunk->word_size(); + nth_bit_for_chunk = 0; + assert(chunk_end <= to, "Chunk end overlaps test address range."); + } else { + // No chunk start marked in bitmap. + assert(chunk != NULL, "Chunk should start at start of address range."); + assert(p < chunk_end, "Did not find expected chunk start at %p.", p); + nth_bit_for_chunk ++; + } + // Check the in-use-info: + const bool in_use_bit = get_bit_at_position(pos, layer_in_use_map); + if (in_use_bit) { + assert(!chunk->is_tagged_free(), "Chunk %p: marked in-use in map but is free (bit %u).", + chunk, nth_bit_for_chunk); + } else { + assert(chunk->is_tagged_free(), "Chunk %p: marked free in map but is in use (bit %u).", + chunk, nth_bit_for_chunk); + } + } +} + +// Verify that a given chunk is correctly accounted for in the bitmap. +void OccupancyMap::verify_for_chunk(Metachunk* chunk) { + assert(chunk_starts_at_address((MetaWord*) chunk), + "No chunk start marked in map for chunk %p.", chunk); + // For chunks larger than the minimal chunk size, no other chunk + // must start in its area. + if (chunk->word_size() > _smallest_chunk_word_size) { + assert(!is_any_bit_set_in_region(((MetaWord*) chunk) + _smallest_chunk_word_size, + chunk->word_size() - _smallest_chunk_word_size, layer_chunk_start_map), + "No chunk must start within another chunk."); + } + if (!chunk->is_tagged_free()) { + assert(is_region_in_use((MetaWord*)chunk, chunk->word_size()), + "Chunk %p is in use but marked as free in map (%d %d).", + chunk, chunk->get_chunk_type(), chunk->get_origin()); + } else { + assert(!is_region_in_use((MetaWord*)chunk, chunk->word_size()), + "Chunk %p is free but marked as in-use in map (%d %d).", + chunk, chunk->get_chunk_type(), chunk->get_origin()); + } +} + +#endif // ASSERT + +} // namespace metaspace + + diff --git a/src/hotspot/share/memory/metaspace/occupancyMap.hpp b/src/hotspot/share/memory/metaspace/occupancyMap.hpp new file mode 100644 --- /dev/null +++ b/src/hotspot/share/memory/metaspace/occupancyMap.hpp @@ -0,0 +1,243 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2018, SAP. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_MEMORY_METASPACE_OCCUPANCYMAP_HPP_ +#define SHARE_MEMORY_METASPACE_OCCUPANCYMAP_HPP_ + +#include "memory/allocation.hpp" +#include "utilities/debug.hpp" +#include "utilities/globalDefinitions.hpp" + + +namespace metaspace { + +class Metachunk; + +// Helper for Occupancy Bitmap. A type trait to give an all-bits-are-one-unsigned constant. +template struct all_ones { static const T value; }; +template <> struct all_ones { static const uint64_t value = 0xFFFFFFFFFFFFFFFFULL; }; +template <> struct all_ones { static const uint32_t value = 0xFFFFFFFF; }; + +// The OccupancyMap is a bitmap which, for a given VirtualSpaceNode, +// keeps information about +// - where a chunk starts +// - whether a chunk is in-use or free +// A bit in this bitmap represents one range of memory in the smallest +// chunk size (SpecializedChunk or ClassSpecializedChunk). +class OccupancyMap : public CHeapObj { + + // The address range this map covers. + const MetaWord* const _reference_address; + const size_t _word_size; + + // The word size of a specialized chunk, aka the number of words one + // bit in this map represents. + const size_t _smallest_chunk_word_size; + + // map data + // Data are organized in two bit layers: + // The first layer is the chunk-start-map. Here, a bit is set to mark + // the corresponding region as the head of a chunk. + // The second layer is the in-use-map. Here, a set bit indicates that + // the corresponding belongs to a chunk which is in use. + uint8_t* _map[2]; + + enum { layer_chunk_start_map = 0, layer_in_use_map = 1 }; + + // length, in bytes, of bitmap data + size_t _map_size; + + // Returns true if bit at position pos at bit-layer layer is set. + bool get_bit_at_position(unsigned pos, unsigned layer) const { + assert(layer == 0 || layer == 1, "Invalid layer %d", layer); + const unsigned byteoffset = pos / 8; + assert(byteoffset < _map_size, + "invalid byte offset (%u), map size is " SIZE_FORMAT ".", byteoffset, _map_size); + const unsigned mask = 1 << (pos % 8); + return (_map[layer][byteoffset] & mask) > 0; + } + + // Changes bit at position pos at bit-layer layer to value v. + void set_bit_at_position(unsigned pos, unsigned layer, bool v) { + assert(layer == 0 || layer == 1, "Invalid layer %d", layer); + const unsigned byteoffset = pos / 8; + assert(byteoffset < _map_size, + "invalid byte offset (%u), map size is " SIZE_FORMAT ".", byteoffset, _map_size); + const unsigned mask = 1 << (pos % 8); + if (v) { + _map[layer][byteoffset] |= mask; + } else { + _map[layer][byteoffset] &= ~mask; + } + } + + // Optimized case of is_any_bit_set_in_region for 32/64bit aligned access: + // pos is 32/64 aligned and num_bits is 32/64. + // This is the typical case when coalescing to medium chunks, whose size is + // 32 or 64 times the specialized chunk size (depending on class or non class + // case), so they occupy 64 bits which should be 64bit aligned, because + // chunks are chunk-size aligned. + template + bool is_any_bit_set_in_region_3264(unsigned pos, unsigned num_bits, unsigned layer) const { + assert(_map_size > 0, "not initialized"); + assert(layer == 0 || layer == 1, "Invalid layer %d.", layer); + assert(pos % (sizeof(T) * 8) == 0, "Bit position must be aligned (%u).", pos); + assert(num_bits == (sizeof(T) * 8), "Number of bits incorrect (%u).", num_bits); + const size_t byteoffset = pos / 8; + assert(byteoffset <= (_map_size - sizeof(T)), + "Invalid byte offset (" SIZE_FORMAT "), map size is " SIZE_FORMAT ".", byteoffset, _map_size); + const T w = *(T*)(_map[layer] + byteoffset); + return w > 0 ? true : false; + } + + // Returns true if any bit in region [pos1, pos1 + num_bits) is set in bit-layer layer. + bool is_any_bit_set_in_region(unsigned pos, unsigned num_bits, unsigned layer) const { + if (pos % 32 == 0 && num_bits == 32) { + return is_any_bit_set_in_region_3264(pos, num_bits, layer); + } else if (pos % 64 == 0 && num_bits == 64) { + return is_any_bit_set_in_region_3264(pos, num_bits, layer); + } else { + for (unsigned n = 0; n < num_bits; n ++) { + if (get_bit_at_position(pos + n, layer)) { + return true; + } + } + } + return false; + } + + // Returns true if any bit in region [p, p+word_size) is set in bit-layer layer. + bool is_any_bit_set_in_region(MetaWord* p, size_t word_size, unsigned layer) const { + assert(word_size % _smallest_chunk_word_size == 0, + "Region size " SIZE_FORMAT " not a multiple of smallest chunk size.", word_size); + const unsigned pos = get_bitpos_for_address(p); + const unsigned num_bits = (unsigned) (word_size / _smallest_chunk_word_size); + return is_any_bit_set_in_region(pos, num_bits, layer); + } + + // Optimized case of set_bits_of_region for 32/64bit aligned access: + // pos is 32/64 aligned and num_bits is 32/64. + // This is the typical case when coalescing to medium chunks, whose size + // is 32 or 64 times the specialized chunk size (depending on class or non + // class case), so they occupy 64 bits which should be 64bit aligned, + // because chunks are chunk-size aligned. + template + void set_bits_of_region_T(unsigned pos, unsigned num_bits, unsigned layer, bool v) { + assert(pos % (sizeof(T) * 8) == 0, "Bit position must be aligned to %u (%u).", + (unsigned)(sizeof(T) * 8), pos); + assert(num_bits == (sizeof(T) * 8), "Number of bits incorrect (%u), expected %u.", + num_bits, (unsigned)(sizeof(T) * 8)); + const size_t byteoffset = pos / 8; + assert(byteoffset <= (_map_size - sizeof(T)), + "invalid byte offset (" SIZE_FORMAT "), map size is " SIZE_FORMAT ".", byteoffset, _map_size); + T* const pw = (T*)(_map[layer] + byteoffset); + *pw = v ? all_ones::value : (T) 0; + } + + // Set all bits in a region starting at pos to a value. + void set_bits_of_region(unsigned pos, unsigned num_bits, unsigned layer, bool v) { + assert(_map_size > 0, "not initialized"); + assert(layer == 0 || layer == 1, "Invalid layer %d.", layer); + if (pos % 32 == 0 && num_bits == 32) { + set_bits_of_region_T(pos, num_bits, layer, v); + } else if (pos % 64 == 0 && num_bits == 64) { + set_bits_of_region_T(pos, num_bits, layer, v); + } else { + for (unsigned n = 0; n < num_bits; n ++) { + set_bit_at_position(pos + n, layer, v); + } + } + } + + // Helper: sets all bits in a region [p, p+word_size). + void set_bits_of_region(MetaWord* p, size_t word_size, unsigned layer, bool v) { + assert(word_size % _smallest_chunk_word_size == 0, + "Region size " SIZE_FORMAT " not a multiple of smallest chunk size.", word_size); + const unsigned pos = get_bitpos_for_address(p); + const unsigned num_bits = (unsigned) (word_size / _smallest_chunk_word_size); + set_bits_of_region(pos, num_bits, layer, v); + } + + // Helper: given an address, return the bit position representing that address. + unsigned get_bitpos_for_address(const MetaWord* p) const { + assert(_reference_address != NULL, "not initialized"); + assert(p >= _reference_address && p < _reference_address + _word_size, + "Address %p out of range for occupancy map [%p..%p).", + p, _reference_address, _reference_address + _word_size); + assert(is_aligned(p, _smallest_chunk_word_size * sizeof(MetaWord)), + "Address not aligned (%p).", p); + const ptrdiff_t d = (p - _reference_address) / _smallest_chunk_word_size; + assert(d >= 0 && (size_t)d < _map_size * 8, "Sanity."); + return (unsigned) d; + } + + public: + + OccupancyMap(const MetaWord* reference_address, size_t word_size, size_t smallest_chunk_word_size); + ~OccupancyMap(); + + // Returns true if at address x a chunk is starting. + bool chunk_starts_at_address(MetaWord* p) const { + const unsigned pos = get_bitpos_for_address(p); + return get_bit_at_position(pos, layer_chunk_start_map); + } + + void set_chunk_starts_at_address(MetaWord* p, bool v) { + const unsigned pos = get_bitpos_for_address(p); + set_bit_at_position(pos, layer_chunk_start_map, v); + } + + // Removes all chunk-start-bits inside a region, typically as a + // result of a chunk merge. + void wipe_chunk_start_bits_in_region(MetaWord* p, size_t word_size) { + set_bits_of_region(p, word_size, layer_chunk_start_map, false); + } + + // Returns true if there are life (in use) chunks in the region limited + // by [p, p+word_size). + bool is_region_in_use(MetaWord* p, size_t word_size) const { + return is_any_bit_set_in_region(p, word_size, layer_in_use_map); + } + + // Marks the region starting at p with the size word_size as in use + // or free, depending on v. + void set_region_in_use(MetaWord* p, size_t word_size, bool v) { + set_bits_of_region(p, word_size, layer_in_use_map, v); + } + + // Verify occupancy map for the address range [from, to). + // We need to tell it the address range, because the memory the + // occupancy map is covering may not be fully comitted yet. + DEBUG_ONLY(void verify(MetaWord* from, MetaWord* to);) + + // Verify that a given chunk is correctly accounted for in the bitmap. + DEBUG_ONLY(void verify_for_chunk(Metachunk* chunk);) + +}; + +} // namespace metaspace + +#endif /* SHARE_MEMORY_METASPACE_OCCUPANCYMAP_HPP_ */ + diff --git a/src/hotspot/share/memory/metaspace/printCLDMetaspaceInfoClosure.cpp b/src/hotspot/share/memory/metaspace/printCLDMetaspaceInfoClosure.cpp new file mode 100644 --- /dev/null +++ b/src/hotspot/share/memory/metaspace/printCLDMetaspaceInfoClosure.cpp @@ -0,0 +1,85 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ +#include "precompiled.hpp" +#include "classfile/classLoaderData.hpp" +#include "memory/metaspace/printCLDMetaspaceInfoClosure.hpp" +#include "memory/resourceArea.hpp" +#include "runtime/safepoint.hpp" +#include "utilities/globalDefinitions.hpp" +#include "utilities/ostream.hpp" + + +namespace metaspace { + +PrintCLDMetaspaceInfoClosure::PrintCLDMetaspaceInfoClosure(outputStream* out, size_t scale, bool do_print, bool break_down_by_chunktype) +: _out(out), _scale(scale), _do_print(do_print), _break_down_by_chunktype(break_down_by_chunktype) +, _num_loaders(0) +{ + memset(_num_loaders_by_spacetype, 0, sizeof(_num_loaders_by_spacetype)); +} + +void PrintCLDMetaspaceInfoClosure::do_cld(ClassLoaderData* cld) { + + assert(SafepointSynchronize::is_at_safepoint(), "Must be at a safepoint"); + + ClassLoaderMetaspace* msp = cld->metaspace_or_null(); + if (msp == NULL) { + return; + } + + // Collect statistics for this class loader metaspace + ClassLoaderMetaspaceStatistics this_cld_stat; + msp->add_to_statistics(&this_cld_stat); + + // And add it to the running totals + _stats_total.add(this_cld_stat); + _num_loaders ++; + _stats_by_spacetype[msp->space_type()].add(this_cld_stat); + _num_loaders_by_spacetype[msp->space_type()] ++; + + // Optionally, print. + if (_do_print) { + + _out->print(UINTX_FORMAT_W(4) ": ", _num_loaders); + + if (cld->is_anonymous()) { + _out->print("ClassLoaderData " PTR_FORMAT " for anonymous class", p2i(cld)); + } else { + ResourceMark rm; + _out->print("ClassLoaderData " PTR_FORMAT " for %s", p2i(cld), cld->loader_name()); + } + + if (cld->is_unloading()) { + _out->print(" (unloading)"); + } + + this_cld_stat.print_on(_out, _scale, _break_down_by_chunktype); + _out->cr(); + + } + +} + +} // namespace metaspace + diff --git a/src/hotspot/share/memory/metaspace/printCLDMetaspaceInfoClosure.hpp b/src/hotspot/share/memory/metaspace/printCLDMetaspaceInfoClosure.hpp new file mode 100644 --- /dev/null +++ b/src/hotspot/share/memory/metaspace/printCLDMetaspaceInfoClosure.hpp @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_MEMORY_METASPACE_PRINTCLDMETASPACEINFOCLOSURE_HPP_ +#define SHARE_MEMORY_METASPACE_PRINTCLDMETASPACEINFOCLOSURE_HPP_ + +#include "memory/iterator.hpp" +#include "memory/metaspace.hpp" +#include "memory/metaspace/metaspaceStatistics.hpp" +#include "utilities/globalDefinitions.hpp" + +class outputStream; + +namespace metaspace { + +class PrintCLDMetaspaceInfoClosure : public CLDClosure { +private: + outputStream* const _out; + const size_t _scale; + const bool _do_print; + const bool _break_down_by_chunktype; + +public: + + uintx _num_loaders; + ClassLoaderMetaspaceStatistics _stats_total; + + uintx _num_loaders_by_spacetype [Metaspace::MetaspaceTypeCount]; + ClassLoaderMetaspaceStatistics _stats_by_spacetype [Metaspace::MetaspaceTypeCount]; + + PrintCLDMetaspaceInfoClosure(outputStream* out, size_t scale, bool do_print, bool break_down_by_chunktype); + void do_cld(ClassLoaderData* cld); + +}; + +} // namespace metaspace + +#endif /* SHARE_MEMORY_METASPACE_PRINTCLDMETASPACEINFOCLOSURE_HPP_ */ + diff --git a/src/hotspot/share/memory/metaspace/smallBlocks.cpp b/src/hotspot/share/memory/metaspace/smallBlocks.cpp new file mode 100644 --- /dev/null +++ b/src/hotspot/share/memory/metaspace/smallBlocks.cpp @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ +#include "precompiled.hpp" + +#include "memory/metaspace/smallBlocks.hpp" +#include "utilities/globalDefinitions.hpp" +#include "utilities/ostream.hpp" + +namespace metaspace { + +void SmallBlocks::print_on(outputStream* st) const { + st->print_cr("SmallBlocks:"); + for (uint i = _small_block_min_size; i < _small_block_max_size; i++) { + uint k = i - _small_block_min_size; + st->print_cr("small_lists size " SIZE_FORMAT " count " SIZE_FORMAT, _small_lists[k].size(), _small_lists[k].count()); + } +} + + +// Returns the total size, in words, of all blocks, across all block sizes. +size_t SmallBlocks::total_size() const { + size_t result = 0; + for (uint i = _small_block_min_size; i < _small_block_max_size; i++) { + uint k = i - _small_block_min_size; + result = result + _small_lists[k].count() * _small_lists[k].size(); + } + return result; +} + +// Returns the total number of all blocks across all block sizes. +uintx SmallBlocks::total_num_blocks() const { + uintx result = 0; + for (uint i = _small_block_min_size; i < _small_block_max_size; i++) { + uint k = i - _small_block_min_size; + result = result + _small_lists[k].count(); + } + return result; +} + +} // namespace metaspace + diff --git a/src/hotspot/share/memory/metaspace/smallBlocks.hpp b/src/hotspot/share/memory/metaspace/smallBlocks.hpp new file mode 100644 --- /dev/null +++ b/src/hotspot/share/memory/metaspace/smallBlocks.hpp @@ -0,0 +1,89 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_MEMORY_METASPACE_SMALLBLOCKS_HPP_ +#define SHARE_MEMORY_METASPACE_SMALLBLOCKS_HPP_ + +#include "memory/allocation.hpp" +#include "memory/binaryTreeDictionary.hpp" +#include "memory/metaspace/metablock.hpp" +#include "utilities/globalDefinitions.hpp" + +class outputStream; + +namespace metaspace { + +class SmallBlocks : public CHeapObj { + + const static uint _small_block_max_size = sizeof(TreeChunk >)/HeapWordSize; + // Note: this corresponds to the imposed miminum allocation size, see SpaceManager::get_allocation_word_size() + const static uint _small_block_min_size = sizeof(Metablock)/HeapWordSize; + +private: + FreeList _small_lists[_small_block_max_size - _small_block_min_size]; + + FreeList& list_at(size_t word_size) { + assert(word_size >= _small_block_min_size, "There are no metaspace objects less than %u words", _small_block_min_size); + return _small_lists[word_size - _small_block_min_size]; + } + +public: + SmallBlocks() { + for (uint i = _small_block_min_size; i < _small_block_max_size; i++) { + uint k = i - _small_block_min_size; + _small_lists[k].set_size(i); + } + } + + // Returns the total size, in words, of all blocks, across all block sizes. + size_t total_size() const; + + // Returns the total number of all blocks across all block sizes. + uintx total_num_blocks() const; + + static uint small_block_max_size() { return _small_block_max_size; } + static uint small_block_min_size() { return _small_block_min_size; } + + MetaWord* get_block(size_t word_size) { + if (list_at(word_size).count() > 0) { + MetaWord* new_block = (MetaWord*) list_at(word_size).get_chunk_at_head(); + return new_block; + } else { + return NULL; + } + } + void return_block(Metablock* free_chunk, size_t word_size) { + list_at(word_size).return_chunk_at_head(free_chunk, false); + assert(list_at(word_size).count() > 0, "Should have a chunk"); + } + + void print_on(outputStream* st) const; + +}; + +} // namespace metaspace + + +#endif /* SHARE_MEMORY_METASPACE_SMALLBLOCKS_HPP_ */ + diff --git a/src/hotspot/share/memory/metaspace/spaceManager.cpp b/src/hotspot/share/memory/metaspace/spaceManager.cpp new file mode 100644 --- /dev/null +++ b/src/hotspot/share/memory/metaspace/spaceManager.cpp @@ -0,0 +1,522 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ +#include "precompiled.hpp" + +#include "logging/log.hpp" +#include "logging/logStream.hpp" +#include "memory/metaspace/chunkManager.hpp" +#include "memory/metaspace/metachunk.hpp" +#include "memory/metaspace/metaDebug.hpp" +#include "memory/metaspace/metaspaceCommon.hpp" +#include "memory/metaspace/spaceManager.hpp" +#include "memory/metaspace/virtualSpaceList.hpp" +#include "runtime/atomic.hpp" +#include "runtime/init.hpp" +#include "services/memoryService.hpp" +#include "utilities/debug.hpp" +#include "utilities/globalDefinitions.hpp" + +namespace metaspace { + +#define assert_counter(expected_value, real_value, msg) \ + assert( (expected_value) == (real_value), \ + "Counter mismatch (%s): expected " SIZE_FORMAT \ + ", but got: " SIZE_FORMAT ".", msg, expected_value, \ + real_value); + +// SpaceManager methods + +size_t SpaceManager::adjust_initial_chunk_size(size_t requested, bool is_class_space) { + size_t chunk_sizes[] = { + specialized_chunk_size(is_class_space), + small_chunk_size(is_class_space), + medium_chunk_size(is_class_space) + }; + + // Adjust up to one of the fixed chunk sizes ... + for (size_t i = 0; i < ARRAY_SIZE(chunk_sizes); i++) { + if (requested <= chunk_sizes[i]) { + return chunk_sizes[i]; + } + } + + // ... or return the size as a humongous chunk. + return requested; +} + +size_t SpaceManager::adjust_initial_chunk_size(size_t requested) const { + return adjust_initial_chunk_size(requested, is_class()); +} + +size_t SpaceManager::get_initial_chunk_size(Metaspace::MetaspaceType type) const { + size_t requested; + + if (is_class()) { + switch (type) { + case Metaspace::BootMetaspaceType: requested = Metaspace::first_class_chunk_word_size(); break; + case Metaspace::AnonymousMetaspaceType: requested = ClassSpecializedChunk; break; + case Metaspace::ReflectionMetaspaceType: requested = ClassSpecializedChunk; break; + default: requested = ClassSmallChunk; break; + } + } else { + switch (type) { + case Metaspace::BootMetaspaceType: requested = Metaspace::first_chunk_word_size(); break; + case Metaspace::AnonymousMetaspaceType: requested = SpecializedChunk; break; + case Metaspace::ReflectionMetaspaceType: requested = SpecializedChunk; break; + default: requested = SmallChunk; break; + } + } + + // Adjust to one of the fixed chunk sizes (unless humongous) + const size_t adjusted = adjust_initial_chunk_size(requested); + + assert(adjusted != 0, "Incorrect initial chunk size. Requested: " + SIZE_FORMAT " adjusted: " SIZE_FORMAT, requested, adjusted); + + return adjusted; +} + +void SpaceManager::locked_print_chunks_in_use_on(outputStream* st) const { + + for (ChunkIndex i = ZeroIndex; i < NumberOfInUseLists; i = next_chunk_index(i)) { + st->print("SpaceManager: " UINTX_FORMAT " %s chunks.", + num_chunks_by_type(i), chunk_size_name(i)); + } + + chunk_manager()->locked_print_free_chunks(st); +} + +size_t SpaceManager::calc_chunk_size(size_t word_size) { + + // Decide between a small chunk and a medium chunk. Up to + // _small_chunk_limit small chunks can be allocated. + // After that a medium chunk is preferred. + size_t chunk_word_size; + + // Special case for anonymous metadata space. + // Anonymous metadata space is usually small, with majority within 1K - 2K range and + // rarely about 4K (64-bits JVM). + // Instead of jumping to SmallChunk after initial chunk exhausted, keeping allocation + // from SpecializeChunk up to _anon_or_delegating_metadata_specialize_chunk_limit (4) + // reduces space waste from 60+% to around 30%. + if ((_space_type == Metaspace::AnonymousMetaspaceType || _space_type == Metaspace::ReflectionMetaspaceType) && + _mdtype == Metaspace::NonClassType && + num_chunks_by_type(SpecializedIndex) < anon_and_delegating_metadata_specialize_chunk_limit && + word_size + Metachunk::overhead() <= SpecializedChunk) { + return SpecializedChunk; + } + + if (num_chunks_by_type(MediumIndex) == 0 && + num_chunks_by_type(SmallIndex) < small_chunk_limit) { + chunk_word_size = (size_t) small_chunk_size(); + if (word_size + Metachunk::overhead() > small_chunk_size()) { + chunk_word_size = medium_chunk_size(); + } + } else { + chunk_word_size = medium_chunk_size(); + } + + // Might still need a humongous chunk. Enforce + // humongous allocations sizes to be aligned up to + // the smallest chunk size. + size_t if_humongous_sized_chunk = + align_up(word_size + Metachunk::overhead(), + smallest_chunk_size()); + chunk_word_size = + MAX2((size_t) chunk_word_size, if_humongous_sized_chunk); + + assert(!SpaceManager::is_humongous(word_size) || + chunk_word_size == if_humongous_sized_chunk, + "Size calculation is wrong, word_size " SIZE_FORMAT + " chunk_word_size " SIZE_FORMAT, + word_size, chunk_word_size); + Log(gc, metaspace, alloc) log; + if (log.is_debug() && SpaceManager::is_humongous(word_size)) { + log.debug("Metadata humongous allocation:"); + log.debug(" word_size " PTR_FORMAT, word_size); + log.debug(" chunk_word_size " PTR_FORMAT, chunk_word_size); + log.debug(" chunk overhead " PTR_FORMAT, Metachunk::overhead()); + } + return chunk_word_size; +} + +void SpaceManager::track_metaspace_memory_usage() { + if (is_init_completed()) { + if (is_class()) { + MemoryService::track_compressed_class_memory_usage(); + } + MemoryService::track_metaspace_memory_usage(); + } +} + +MetaWord* SpaceManager::grow_and_allocate(size_t word_size) { + assert_lock_strong(_lock); + assert(vs_list()->current_virtual_space() != NULL, + "Should have been set"); + assert(current_chunk() == NULL || + current_chunk()->allocate(word_size) == NULL, + "Don't need to expand"); + MutexLockerEx cl(MetaspaceExpand_lock, Mutex::_no_safepoint_check_flag); + + if (log_is_enabled(Trace, gc, metaspace, freelist)) { + size_t words_left = 0; + size_t words_used = 0; + if (current_chunk() != NULL) { + words_left = current_chunk()->free_word_size(); + words_used = current_chunk()->used_word_size(); + } + log_trace(gc, metaspace, freelist)("SpaceManager::grow_and_allocate for " SIZE_FORMAT " words " SIZE_FORMAT " words used " SIZE_FORMAT " words left", + word_size, words_used, words_left); + } + + // Get another chunk + size_t chunk_word_size = calc_chunk_size(word_size); + Metachunk* next = get_new_chunk(chunk_word_size); + + MetaWord* mem = NULL; + + // If a chunk was available, add it to the in-use chunk list + // and do an allocation from it. + if (next != NULL) { + // Add to this manager's list of chunks in use. + // If the new chunk is humongous, it was created to serve a single large allocation. In that + // case it usually makes no sense to make it the current chunk, since the next allocation would + // need to allocate a new chunk anyway, while we would now prematurely retire a perfectly + // good chunk which could be used for more normal allocations. + bool make_current = true; + if (next->get_chunk_type() == HumongousIndex && + current_chunk() != NULL) { + make_current = false; + } + add_chunk(next, make_current); + mem = next->allocate(word_size); + } + + // Track metaspace memory usage statistic. + track_metaspace_memory_usage(); + + return mem; +} + +void SpaceManager::print_on(outputStream* st) const { + SpaceManagerStatistics stat; + add_to_statistics(&stat); // will lock _lock. + stat.print_on(st, 1*K, false); +} + +SpaceManager::SpaceManager(Metaspace::MetadataType mdtype, + Metaspace::MetaspaceType space_type,// + Mutex* lock) : + _mdtype(mdtype), + _space_type(space_type), + _capacity_words(0), + _used_words(0), + _overhead_words(0), + _block_freelists(NULL), + _lock(lock), + _chunk_list(NULL), + _current_chunk(NULL) +{ + Metadebug::init_allocation_fail_alot_count(); + memset(_num_chunks_by_type, 0, sizeof(_num_chunks_by_type)); + log_trace(gc, metaspace, freelist)("SpaceManager(): " PTR_FORMAT, p2i(this)); +} + +void SpaceManager::account_for_new_chunk(const Metachunk* new_chunk) { + + assert_lock_strong(MetaspaceExpand_lock); + + _capacity_words += new_chunk->word_size(); + _overhead_words += Metachunk::overhead(); + DEBUG_ONLY(new_chunk->verify()); + _num_chunks_by_type[new_chunk->get_chunk_type()] ++; + + // Adjust global counters: + MetaspaceUtils::inc_capacity(mdtype(), new_chunk->word_size()); + MetaspaceUtils::inc_overhead(mdtype(), Metachunk::overhead()); +} + +void SpaceManager::account_for_allocation(size_t words) { + // Note: we should be locked with the ClassloaderData-specific metaspace lock. + // We may or may not be locked with the global metaspace expansion lock. + assert_lock_strong(lock()); + + // Add to the per SpaceManager totals. This can be done non-atomically. + _used_words += words; + + // Adjust global counters. This will be done atomically. + MetaspaceUtils::inc_used(mdtype(), words); +} + +void SpaceManager::account_for_spacemanager_death() { + + assert_lock_strong(MetaspaceExpand_lock); + + MetaspaceUtils::dec_capacity(mdtype(), _capacity_words); + MetaspaceUtils::dec_overhead(mdtype(), _overhead_words); + MetaspaceUtils::dec_used(mdtype(), _used_words); +} + +SpaceManager::~SpaceManager() { + + // This call this->_lock which can't be done while holding MetaspaceExpand_lock + DEBUG_ONLY(verify_metrics()); + + MutexLockerEx fcl(MetaspaceExpand_lock, + Mutex::_no_safepoint_check_flag); + + chunk_manager()->slow_locked_verify(); + + account_for_spacemanager_death(); + + Log(gc, metaspace, freelist) log; + if (log.is_trace()) { + log.trace("~SpaceManager(): " PTR_FORMAT, p2i(this)); + ResourceMark rm; + LogStream ls(log.trace()); + locked_print_chunks_in_use_on(&ls); + if (block_freelists() != NULL) { + block_freelists()->print_on(&ls); + } + } + + // Add all the chunks in use by this space manager + // to the global list of free chunks. + + // Follow each list of chunks-in-use and add them to the + // free lists. Each list is NULL terminated. + chunk_manager()->return_chunk_list(chunk_list()); +#ifdef ASSERT + _chunk_list = NULL; + _current_chunk = NULL; +#endif + + chunk_manager()->slow_locked_verify(); + + if (_block_freelists != NULL) { + delete _block_freelists; + } +} + +void SpaceManager::deallocate(MetaWord* p, size_t word_size) { + assert_lock_strong(lock()); + // Allocations and deallocations are in raw_word_size + size_t raw_word_size = get_allocation_word_size(word_size); + // Lazily create a block_freelist + if (block_freelists() == NULL) { + _block_freelists = new BlockFreelist(); + } + block_freelists()->return_block(p, raw_word_size); + DEBUG_ONLY(Atomic::inc(&(g_internal_statistics.num_deallocs))); +} + +// Adds a chunk to the list of chunks in use. +void SpaceManager::add_chunk(Metachunk* new_chunk, bool make_current) { + + assert_lock_strong(_lock); + assert(new_chunk != NULL, "Should not be NULL"); + assert(new_chunk->next() == NULL, "Should not be on a list"); + + new_chunk->reset_empty(); + + // Find the correct list and and set the current + // chunk for that list. + ChunkIndex index = chunk_manager()->list_index(new_chunk->word_size()); + + if (make_current) { + // If we are to make the chunk current, retire the old current chunk and replace + // it with the new chunk. + retire_current_chunk(); + set_current_chunk(new_chunk); + } + + // Add the new chunk at the head of its respective chunk list. + new_chunk->set_next(_chunk_list); + _chunk_list = new_chunk; + + // Adjust counters. + account_for_new_chunk(new_chunk); + + assert(new_chunk->is_empty(), "Not ready for reuse"); + Log(gc, metaspace, freelist) log; + if (log.is_trace()) { + log.trace("SpaceManager::added chunk: "); + ResourceMark rm; + LogStream ls(log.trace()); + new_chunk->print_on(&ls); + chunk_manager()->locked_print_free_chunks(&ls); + } +} + +void SpaceManager::retire_current_chunk() { + if (current_chunk() != NULL) { + size_t remaining_words = current_chunk()->free_word_size(); + if (remaining_words >= SmallBlocks::small_block_min_size()) { + MetaWord* ptr = current_chunk()->allocate(remaining_words); + deallocate(ptr, remaining_words); + account_for_allocation(remaining_words); + } + } +} + +Metachunk* SpaceManager::get_new_chunk(size_t chunk_word_size) { + // Get a chunk from the chunk freelist + Metachunk* next = chunk_manager()->chunk_freelist_allocate(chunk_word_size); + + if (next == NULL) { + next = vs_list()->get_new_chunk(chunk_word_size, + medium_chunk_bunch()); + } + + Log(gc, metaspace, alloc) log; + if (log.is_debug() && next != NULL && + SpaceManager::is_humongous(next->word_size())) { + log.debug(" new humongous chunk word size " PTR_FORMAT, next->word_size()); + } + + return next; +} + +MetaWord* SpaceManager::allocate(size_t word_size) { + MutexLockerEx cl(lock(), Mutex::_no_safepoint_check_flag); + size_t raw_word_size = get_allocation_word_size(word_size); + BlockFreelist* fl = block_freelists(); + MetaWord* p = NULL; + + DEBUG_ONLY(if (VerifyMetaspace) verify_metrics_locked()); + + // Allocation from the dictionary is expensive in the sense that + // the dictionary has to be searched for a size. Don't allocate + // from the dictionary until it starts to get fat. Is this + // a reasonable policy? Maybe an skinny dictionary is fast enough + // for allocations. Do some profiling. JJJ + if (fl != NULL && fl->total_size() > allocation_from_dictionary_limit) { + p = fl->get_block(raw_word_size); + if (p != NULL) { + DEBUG_ONLY(Atomic::inc(&g_internal_statistics.num_allocs_from_deallocated_blocks)); + } + } + if (p == NULL) { + p = allocate_work(raw_word_size); + } + + return p; +} + +// Returns the address of spaced allocated for "word_size". +// This methods does not know about blocks (Metablocks) +MetaWord* SpaceManager::allocate_work(size_t word_size) { + assert_lock_strong(lock()); +#ifdef ASSERT + if (Metadebug::test_metadata_failure()) { + return NULL; + } +#endif + // Is there space in the current chunk? + MetaWord* result = NULL; + + if (current_chunk() != NULL) { + result = current_chunk()->allocate(word_size); + } + + if (result == NULL) { + result = grow_and_allocate(word_size); + } + + if (result != NULL) { + account_for_allocation(word_size); + } + + return result; +} + +void SpaceManager::verify() { + Metachunk* curr = chunk_list(); + while (curr != NULL) { + DEBUG_ONLY(do_verify_chunk(curr);) + assert(curr->is_tagged_free() == false, "Chunk should be tagged as in use."); + curr = curr->next(); + } +} + +void SpaceManager::verify_chunk_size(Metachunk* chunk) { + assert(is_humongous(chunk->word_size()) || + chunk->word_size() == medium_chunk_size() || + chunk->word_size() == small_chunk_size() || + chunk->word_size() == specialized_chunk_size(), + "Chunk size is wrong"); + return; +} + +void SpaceManager::add_to_statistics_locked(SpaceManagerStatistics* out) const { + assert_lock_strong(lock()); + Metachunk* chunk = chunk_list(); + while (chunk != NULL) { + UsedChunksStatistics& chunk_stat = out->chunk_stats(chunk->get_chunk_type()); + chunk_stat.add_num(1); + chunk_stat.add_cap(chunk->word_size()); + chunk_stat.add_overhead(Metachunk::overhead()); + chunk_stat.add_used(chunk->used_word_size() - Metachunk::overhead()); + if (chunk != current_chunk()) { + chunk_stat.add_waste(chunk->free_word_size()); + } else { + chunk_stat.add_free(chunk->free_word_size()); + } + chunk = chunk->next(); + } + if (block_freelists() != NULL) { + out->add_free_blocks_info(block_freelists()->num_blocks(), block_freelists()->total_size()); + } +} + +void SpaceManager::add_to_statistics(SpaceManagerStatistics* out) const { + MutexLockerEx cl(lock(), Mutex::_no_safepoint_check_flag); + add_to_statistics_locked(out); +} + +#ifdef ASSERT +void SpaceManager::verify_metrics_locked() const { + assert_lock_strong(lock()); + + SpaceManagerStatistics stat; + add_to_statistics_locked(&stat); + + UsedChunksStatistics chunk_stats = stat.totals(); + + DEBUG_ONLY(chunk_stats.check_sanity()); + + assert_counter(_capacity_words, chunk_stats.cap(), "SpaceManager::_capacity_words"); + assert_counter(_used_words, chunk_stats.used(), "SpaceManager::_used_words"); + assert_counter(_overhead_words, chunk_stats.overhead(), "SpaceManager::_overhead_words"); +} + +void SpaceManager::verify_metrics() const { + MutexLockerEx cl(lock(), Mutex::_no_safepoint_check_flag); + verify_metrics_locked(); +} +#endif // ASSERT + + +} // namespace metaspace + diff --git a/src/hotspot/share/memory/metaspace/spaceManager.hpp b/src/hotspot/share/memory/metaspace/spaceManager.hpp new file mode 100644 --- /dev/null +++ b/src/hotspot/share/memory/metaspace/spaceManager.hpp @@ -0,0 +1,234 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_MEMORY_METASPACE_SPACEMANAGER_HPP_ +#define SHARE_MEMORY_METASPACE_SPACEMANAGER_HPP_ + +#include "memory/allocation.hpp" +#include "memory/metaspace.hpp" +#include "memory/metaspace/blockFreelist.hpp" +#include "memory/metaspace/metaspaceCommon.hpp" +#include "memory/metaspace/metachunk.hpp" +#include "memory/metaspace/metaspaceStatistics.hpp" +#include "utilities/debug.hpp" +#include "utilities/globalDefinitions.hpp" + +class outputStream; +class Mutex; + +namespace metaspace { + +// SpaceManager - used by Metaspace to handle allocations +class SpaceManager : public CHeapObj { + friend class ::ClassLoaderMetaspace; + friend class Metadebug; + + private: + + // protects allocations + Mutex* const _lock; + + // Type of metadata allocated. + const Metaspace::MetadataType _mdtype; + + // Type of metaspace + const Metaspace::MetaspaceType _space_type; + + // List of chunks in use by this SpaceManager. Allocations + // are done from the current chunk. The list is used for deallocating + // chunks when the SpaceManager is freed. + Metachunk* _chunk_list; + Metachunk* _current_chunk; + + enum { + + // Maximum number of small chunks to allocate to a SpaceManager + small_chunk_limit = 4, + + // Maximum number of specialize chunks to allocate for anonymous and delegating + // metadata space to a SpaceManager + anon_and_delegating_metadata_specialize_chunk_limit = 4, + + allocation_from_dictionary_limit = 4 * K + + }; + + // Some running counters, but lets keep their number small to not add to much to + // the per-classloader footprint. + // Note: capacity = used + free + waste + overhead. We do not keep running counters for + // free and waste. Their sum can be deduced from the three other values. + size_t _overhead_words; + size_t _capacity_words; + size_t _used_words; + uintx _num_chunks_by_type[NumberOfInUseLists]; + + // Free lists of blocks are per SpaceManager since they + // are assumed to be in chunks in use by the SpaceManager + // and all chunks in use by a SpaceManager are freed when + // the class loader using the SpaceManager is collected. + BlockFreelist* _block_freelists; + + private: + // Accessors + Metachunk* chunk_list() const { return _chunk_list; } + + BlockFreelist* block_freelists() const { return _block_freelists; } + + Metaspace::MetadataType mdtype() { return _mdtype; } + + VirtualSpaceList* vs_list() const { return Metaspace::get_space_list(_mdtype); } + ChunkManager* chunk_manager() const { return Metaspace::get_chunk_manager(_mdtype); } + + Metachunk* current_chunk() const { return _current_chunk; } + void set_current_chunk(Metachunk* v) { + _current_chunk = v; + } + + Metachunk* find_current_chunk(size_t word_size); + + // Add chunk to the list of chunks in use + void add_chunk(Metachunk* v, bool make_current); + void retire_current_chunk(); + + Mutex* lock() const { return _lock; } + + // Adds to the given statistic object. Expects to be locked with lock(). + void add_to_statistics_locked(SpaceManagerStatistics* out) const; + + // Verify internal counters against the current state. Expects to be locked with lock(). + DEBUG_ONLY(void verify_metrics_locked() const;) + + public: + SpaceManager(Metaspace::MetadataType mdtype, + Metaspace::MetaspaceType space_type, + Mutex* lock); + ~SpaceManager(); + + enum ChunkMultiples { + MediumChunkMultiple = 4 + }; + + static size_t specialized_chunk_size(bool is_class) { return is_class ? ClassSpecializedChunk : SpecializedChunk; } + static size_t small_chunk_size(bool is_class) { return is_class ? ClassSmallChunk : SmallChunk; } + static size_t medium_chunk_size(bool is_class) { return is_class ? ClassMediumChunk : MediumChunk; } + + static size_t smallest_chunk_size(bool is_class) { return specialized_chunk_size(is_class); } + + // Accessors + bool is_class() const { return _mdtype == Metaspace::ClassType; } + + size_t specialized_chunk_size() const { return specialized_chunk_size(is_class()); } + size_t small_chunk_size() const { return small_chunk_size(is_class()); } + size_t medium_chunk_size() const { return medium_chunk_size(is_class()); } + + size_t smallest_chunk_size() const { return smallest_chunk_size(is_class()); } + + size_t medium_chunk_bunch() const { return medium_chunk_size() * MediumChunkMultiple; } + + bool is_humongous(size_t word_size) { return word_size > medium_chunk_size(); } + + size_t capacity_words() const { return _capacity_words; } + size_t used_words() const { return _used_words; } + size_t overhead_words() const { return _overhead_words; } + + // Adjust local, global counters after a new chunk has been added. + void account_for_new_chunk(const Metachunk* new_chunk); + + // Adjust local, global counters after space has been allocated from the current chunk. + void account_for_allocation(size_t words); + + // Adjust global counters just before the SpaceManager dies, after all its chunks + // have been returned to the freelist. + void account_for_spacemanager_death(); + + // Adjust the initial chunk size to match one of the fixed chunk list sizes, + // or return the unadjusted size if the requested size is humongous. + static size_t adjust_initial_chunk_size(size_t requested, bool is_class_space); + size_t adjust_initial_chunk_size(size_t requested) const; + + // Get the initial chunks size for this metaspace type. + size_t get_initial_chunk_size(Metaspace::MetaspaceType type) const; + + // Todo: remove this once we have counters by chunk type. + uintx num_chunks_by_type(ChunkIndex chunk_type) const { return _num_chunks_by_type[chunk_type]; } + + Metachunk* get_new_chunk(size_t chunk_word_size); + + // Block allocation and deallocation. + // Allocates a block from the current chunk + MetaWord* allocate(size_t word_size); + + // Helper for allocations + MetaWord* allocate_work(size_t word_size); + + // Returns a block to the per manager freelist + void deallocate(MetaWord* p, size_t word_size); + + // Based on the allocation size and a minimum chunk size, + // returned chunk size (for expanding space for chunk allocation). + size_t calc_chunk_size(size_t allocation_word_size); + + // Called when an allocation from the current chunk fails. + // Gets a new chunk (may require getting a new virtual space), + // and allocates from that chunk. + MetaWord* grow_and_allocate(size_t word_size); + + // Notify memory usage to MemoryService. + void track_metaspace_memory_usage(); + + // debugging support. + + void print_on(outputStream* st) const; + void locked_print_chunks_in_use_on(outputStream* st) const; + + void verify(); + void verify_chunk_size(Metachunk* chunk); + + // This adjusts the size given to be greater than the minimum allocation size in + // words for data in metaspace. Esentially the minimum size is currently 3 words. + size_t get_allocation_word_size(size_t word_size) { + size_t byte_size = word_size * BytesPerWord; + + size_t raw_bytes_size = MAX2(byte_size, sizeof(Metablock)); + raw_bytes_size = align_up(raw_bytes_size, Metachunk::object_alignment()); + + size_t raw_word_size = raw_bytes_size / BytesPerWord; + assert(raw_word_size * BytesPerWord == raw_bytes_size, "Size problem"); + + return raw_word_size; + } + + // Adds to the given statistic object. + void add_to_statistics(SpaceManagerStatistics* out) const; + + // Verify internal counters against the current state. + DEBUG_ONLY(void verify_metrics() const;) + +}; + + +} // namespace metaspace + +#endif /* SHARE_MEMORY_METASPACE_SPACEMANAGER_HPP_ */ + diff --git a/src/hotspot/share/memory/metaspace/virtualSpaceList.cpp b/src/hotspot/share/memory/metaspace/virtualSpaceList.cpp new file mode 100644 --- /dev/null +++ b/src/hotspot/share/memory/metaspace/virtualSpaceList.cpp @@ -0,0 +1,387 @@ +/* + * virtualSpaceList.cpp + * + * Created on: May 6, 2018 + * Author: thomas + */ + + +#include "precompiled.hpp" +#include "logging/log.hpp" +#include "logging/logStream.hpp" +#include "memory/metaspace.hpp" +#include "memory/metaspace/chunkManager.hpp" +#include "memory/metaspace/metachunk.hpp" +#include "memory/metaspace/metaspaceCommon.hpp" +#include "memory/metaspace/virtualSpaceList.hpp" +#include "memory/metaspace/virtualSpaceNode.hpp" +#include "runtime/orderAccess.inline.hpp" +#include "runtime/mutexLocker.hpp" +#include "runtime/safepoint.hpp" + +namespace metaspace { + + +VirtualSpaceList::~VirtualSpaceList() { + VirtualSpaceListIterator iter(virtual_space_list()); + while (iter.repeat()) { + VirtualSpaceNode* vsl = iter.get_next(); + delete vsl; + } +} + +void VirtualSpaceList::inc_reserved_words(size_t v) { + assert_lock_strong(MetaspaceExpand_lock); + _reserved_words = _reserved_words + v; +} +void VirtualSpaceList::dec_reserved_words(size_t v) { + assert_lock_strong(MetaspaceExpand_lock); + _reserved_words = _reserved_words - v; +} + +#define assert_committed_below_limit() \ + assert(MetaspaceUtils::committed_bytes() <= MaxMetaspaceSize, \ + "Too much committed memory. Committed: " SIZE_FORMAT \ + " limit (MaxMetaspaceSize): " SIZE_FORMAT, \ + MetaspaceUtils::committed_bytes(), MaxMetaspaceSize); + +void VirtualSpaceList::inc_committed_words(size_t v) { + assert_lock_strong(MetaspaceExpand_lock); + _committed_words = _committed_words + v; + + assert_committed_below_limit(); +} +void VirtualSpaceList::dec_committed_words(size_t v) { + assert_lock_strong(MetaspaceExpand_lock); + _committed_words = _committed_words - v; + + assert_committed_below_limit(); +} + +void VirtualSpaceList::inc_virtual_space_count() { + assert_lock_strong(MetaspaceExpand_lock); + _virtual_space_count++; +} + +void VirtualSpaceList::dec_virtual_space_count() { + assert_lock_strong(MetaspaceExpand_lock); + _virtual_space_count--; +} + +// Walk the list of VirtualSpaceNodes and delete +// nodes with a 0 container_count. Remove Metachunks in +// the node from their respective freelists. +void VirtualSpaceList::purge(ChunkManager* chunk_manager) { + assert(SafepointSynchronize::is_at_safepoint(), "must be called at safepoint for contains to work"); + assert_lock_strong(MetaspaceExpand_lock); + // Don't use a VirtualSpaceListIterator because this + // list is being changed and a straightforward use of an iterator is not safe. + VirtualSpaceNode* purged_vsl = NULL; + VirtualSpaceNode* prev_vsl = virtual_space_list(); + VirtualSpaceNode* next_vsl = prev_vsl; + while (next_vsl != NULL) { + VirtualSpaceNode* vsl = next_vsl; + DEBUG_ONLY(vsl->verify_container_count();) + next_vsl = vsl->next(); + // Don't free the current virtual space since it will likely + // be needed soon. + if (vsl->container_count() == 0 && vsl != current_virtual_space()) { + log_trace(gc, metaspace, freelist)("Purging VirtualSpaceNode " PTR_FORMAT " (capacity: " SIZE_FORMAT + ", used: " SIZE_FORMAT ").", p2i(vsl), vsl->capacity_words_in_vs(), vsl->used_words_in_vs()); + DEBUG_ONLY(Atomic::inc(&g_internal_statistics.num_vsnodes_purged)); + // Unlink it from the list + if (prev_vsl == vsl) { + // This is the case of the current node being the first node. + assert(vsl == virtual_space_list(), "Expected to be the first node"); + set_virtual_space_list(vsl->next()); + } else { + prev_vsl->set_next(vsl->next()); + } + + vsl->purge(chunk_manager); + dec_reserved_words(vsl->reserved_words()); + dec_committed_words(vsl->committed_words()); + dec_virtual_space_count(); + purged_vsl = vsl; + delete vsl; + } else { + prev_vsl = vsl; + } + } +#ifdef ASSERT + if (purged_vsl != NULL) { + // List should be stable enough to use an iterator here. + VirtualSpaceListIterator iter(virtual_space_list()); + while (iter.repeat()) { + VirtualSpaceNode* vsl = iter.get_next(); + assert(vsl != purged_vsl, "Purge of vsl failed"); + } + } +#endif +} + + +// This function looks at the mmap regions in the metaspace without locking. +// The chunks are added with store ordering and not deleted except for at +// unloading time during a safepoint. +bool VirtualSpaceList::contains(const void* ptr) { + // List should be stable enough to use an iterator here because removing virtual + // space nodes is only allowed at a safepoint. + VirtualSpaceListIterator iter(virtual_space_list()); + while (iter.repeat()) { + VirtualSpaceNode* vsn = iter.get_next(); + if (vsn->contains(ptr)) { + return true; + } + } + return false; +} + +void VirtualSpaceList::retire_current_virtual_space() { + assert_lock_strong(MetaspaceExpand_lock); + + VirtualSpaceNode* vsn = current_virtual_space(); + + ChunkManager* cm = is_class() ? Metaspace::chunk_manager_class() : + Metaspace::chunk_manager_metadata(); + + vsn->retire(cm); +} + +VirtualSpaceList::VirtualSpaceList(size_t word_size) : + _is_class(false), + _virtual_space_list(NULL), + _current_virtual_space(NULL), + _reserved_words(0), + _committed_words(0), + _virtual_space_count(0) { + MutexLockerEx cl(MetaspaceExpand_lock, + Mutex::_no_safepoint_check_flag); + create_new_virtual_space(word_size); +} + +VirtualSpaceList::VirtualSpaceList(ReservedSpace rs) : + _is_class(true), + _virtual_space_list(NULL), + _current_virtual_space(NULL), + _reserved_words(0), + _committed_words(0), + _virtual_space_count(0) { + MutexLockerEx cl(MetaspaceExpand_lock, + Mutex::_no_safepoint_check_flag); + VirtualSpaceNode* class_entry = new VirtualSpaceNode(is_class(), rs); + bool succeeded = class_entry->initialize(); + if (succeeded) { + link_vs(class_entry); + } +} + +size_t VirtualSpaceList::free_bytes() { + return current_virtual_space()->free_words_in_vs() * BytesPerWord; +} + +// Allocate another meta virtual space and add it to the list. +bool VirtualSpaceList::create_new_virtual_space(size_t vs_word_size) { + assert_lock_strong(MetaspaceExpand_lock); + + if (is_class()) { + assert(false, "We currently don't support more than one VirtualSpace for" + " the compressed class space. The initialization of the" + " CCS uses another code path and should not hit this path."); + return false; + } + + if (vs_word_size == 0) { + assert(false, "vs_word_size should always be at least _reserve_alignment large."); + return false; + } + + // Reserve the space + size_t vs_byte_size = vs_word_size * BytesPerWord; + assert_is_aligned(vs_byte_size, Metaspace::reserve_alignment()); + + // Allocate the meta virtual space and initialize it. + VirtualSpaceNode* new_entry = new VirtualSpaceNode(is_class(), vs_byte_size); + if (!new_entry->initialize()) { + delete new_entry; + return false; + } else { + assert(new_entry->reserved_words() == vs_word_size, + "Reserved memory size differs from requested memory size"); + // ensure lock-free iteration sees fully initialized node + OrderAccess::storestore(); + link_vs(new_entry); + DEBUG_ONLY(Atomic::inc(&g_internal_statistics.num_vsnodes_created)); + return true; + } +} + +void VirtualSpaceList::link_vs(VirtualSpaceNode* new_entry) { + if (virtual_space_list() == NULL) { + set_virtual_space_list(new_entry); + } else { + current_virtual_space()->set_next(new_entry); + } + set_current_virtual_space(new_entry); + inc_reserved_words(new_entry->reserved_words()); + inc_committed_words(new_entry->committed_words()); + inc_virtual_space_count(); +#ifdef ASSERT + new_entry->mangle(); +#endif + LogTarget(Trace, gc, metaspace) lt; + if (lt.is_enabled()) { + LogStream ls(lt); + VirtualSpaceNode* vsl = current_virtual_space(); + ResourceMark rm; + vsl->print_on(&ls); + } +} + +bool VirtualSpaceList::expand_node_by(VirtualSpaceNode* node, + size_t min_words, + size_t preferred_words) { + size_t before = node->committed_words(); + + bool result = node->expand_by(min_words, preferred_words); + + size_t after = node->committed_words(); + + // after and before can be the same if the memory was pre-committed. + assert(after >= before, "Inconsistency"); + inc_committed_words(after - before); + + return result; +} + +bool VirtualSpaceList::expand_by(size_t min_words, size_t preferred_words) { + assert_is_aligned(min_words, Metaspace::commit_alignment_words()); + assert_is_aligned(preferred_words, Metaspace::commit_alignment_words()); + assert(min_words <= preferred_words, "Invalid arguments"); + + const char* const class_or_not = (is_class() ? "class" : "non-class"); + + if (!MetaspaceGC::can_expand(min_words, this->is_class())) { + log_trace(gc, metaspace, freelist)("Cannot expand %s virtual space list.", + class_or_not); + return false; + } + + size_t allowed_expansion_words = MetaspaceGC::allowed_expansion(); + if (allowed_expansion_words < min_words) { + log_trace(gc, metaspace, freelist)("Cannot expand %s virtual space list (must try gc first).", + class_or_not); + return false; + } + + size_t max_expansion_words = MIN2(preferred_words, allowed_expansion_words); + + // Commit more memory from the the current virtual space. + bool vs_expanded = expand_node_by(current_virtual_space(), + min_words, + max_expansion_words); + if (vs_expanded) { + log_trace(gc, metaspace, freelist)("Expanded %s virtual space list.", + class_or_not); + return true; + } + log_trace(gc, metaspace, freelist)("%s virtual space list: retire current node.", + class_or_not); + retire_current_virtual_space(); + + // Get another virtual space. + size_t grow_vs_words = MAX2((size_t)VirtualSpaceSize, preferred_words); + grow_vs_words = align_up(grow_vs_words, Metaspace::reserve_alignment_words()); + + if (create_new_virtual_space(grow_vs_words)) { + if (current_virtual_space()->is_pre_committed()) { + // The memory was pre-committed, so we are done here. + assert(min_words <= current_virtual_space()->committed_words(), + "The new VirtualSpace was pre-committed, so it" + "should be large enough to fit the alloc request."); + return true; + } + + return expand_node_by(current_virtual_space(), + min_words, + max_expansion_words); + } + + return false; +} + +// Given a chunk, calculate the largest possible padding space which +// could be required when allocating it. +static size_t largest_possible_padding_size_for_chunk(size_t chunk_word_size, bool is_class) { + const ChunkIndex chunk_type = get_chunk_type_by_size(chunk_word_size, is_class); + if (chunk_type != HumongousIndex) { + // Normal, non-humongous chunks are allocated at chunk size + // boundaries, so the largest padding space required would be that + // minus the smallest chunk size. + const size_t smallest_chunk_size = is_class ? ClassSpecializedChunk : SpecializedChunk; + return chunk_word_size - smallest_chunk_size; + } else { + // Humongous chunks are allocated at smallest-chunksize + // boundaries, so there is no padding required. + return 0; + } +} + + +Metachunk* VirtualSpaceList::get_new_chunk(size_t chunk_word_size, size_t suggested_commit_granularity) { + + // Allocate a chunk out of the current virtual space. + Metachunk* next = current_virtual_space()->get_chunk_vs(chunk_word_size); + + if (next != NULL) { + return next; + } + + // The expand amount is currently only determined by the requested sizes + // and not how much committed memory is left in the current virtual space. + + // We must have enough space for the requested size and any + // additional reqired padding chunks. + const size_t size_for_padding = largest_possible_padding_size_for_chunk(chunk_word_size, this->is_class()); + + size_t min_word_size = align_up(chunk_word_size + size_for_padding, Metaspace::commit_alignment_words()); + size_t preferred_word_size = align_up(suggested_commit_granularity, Metaspace::commit_alignment_words()); + if (min_word_size >= preferred_word_size) { + // Can happen when humongous chunks are allocated. + preferred_word_size = min_word_size; + } + + bool expanded = expand_by(min_word_size, preferred_word_size); + if (expanded) { + next = current_virtual_space()->get_chunk_vs(chunk_word_size); + assert(next != NULL, "The allocation was expected to succeed after the expansion"); + } + + return next; +} + +void VirtualSpaceList::print_on(outputStream* st, size_t scale) const { + st->print_cr(SIZE_FORMAT " nodes, current node: " PTR_FORMAT, + _virtual_space_count, p2i(_current_virtual_space)); + VirtualSpaceListIterator iter(virtual_space_list()); + while (iter.repeat()) { + st->cr(); + VirtualSpaceNode* node = iter.get_next(); + node->print_on(st, scale); + } +} + +void VirtualSpaceList::print_map(outputStream* st) const { + VirtualSpaceNode* list = virtual_space_list(); + VirtualSpaceListIterator iter(list); + unsigned i = 0; + while (iter.repeat()) { + st->print_cr("Node %u:", i); + VirtualSpaceNode* node = iter.get_next(); + node->print_map(st, this->is_class()); + i ++; + } +} + +} // namespace metaspace + diff --git a/src/hotspot/share/memory/metaspace/virtualSpaceList.hpp b/src/hotspot/share/memory/metaspace/virtualSpaceList.hpp new file mode 100644 --- /dev/null +++ b/src/hotspot/share/memory/metaspace/virtualSpaceList.hpp @@ -0,0 +1,151 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_MEMORY_METASPACE_VIRTUALSPACELIST_HPP_ +#define SHARE_MEMORY_METASPACE_VIRTUALSPACELIST_HPP_ + +#include "memory/allocation.hpp" +#include "memory/metaspace/virtualSpaceNode.hpp" +#include "utilities/globalDefinitions.hpp" + + +namespace metaspace { + +class Metachunk; +class ChunkManager; + +// List of VirtualSpaces for metadata allocation. +class VirtualSpaceList : public CHeapObj { + friend class VirtualSpaceNode; + + enum VirtualSpaceSizes { + VirtualSpaceSize = 256 * K + }; + + // Head of the list + VirtualSpaceNode* _virtual_space_list; + // virtual space currently being used for allocations + VirtualSpaceNode* _current_virtual_space; + + // Is this VirtualSpaceList used for the compressed class space + bool _is_class; + + // Sum of reserved and committed memory in the virtual spaces + size_t _reserved_words; + size_t _committed_words; + + // Number of virtual spaces + size_t _virtual_space_count; + + ~VirtualSpaceList(); + + VirtualSpaceNode* virtual_space_list() const { return _virtual_space_list; } + + void set_virtual_space_list(VirtualSpaceNode* v) { + _virtual_space_list = v; + } + void set_current_virtual_space(VirtualSpaceNode* v) { + _current_virtual_space = v; + } + + void link_vs(VirtualSpaceNode* new_entry); + + // Get another virtual space and add it to the list. This + // is typically prompted by a failed attempt to allocate a chunk + // and is typically followed by the allocation of a chunk. + bool create_new_virtual_space(size_t vs_word_size); + + // Chunk up the unused committed space in the current + // virtual space and add the chunks to the free list. + void retire_current_virtual_space(); + + public: + VirtualSpaceList(size_t word_size); + VirtualSpaceList(ReservedSpace rs); + + size_t free_bytes(); + + Metachunk* get_new_chunk(size_t chunk_word_size, + size_t suggested_commit_granularity); + + bool expand_node_by(VirtualSpaceNode* node, + size_t min_words, + size_t preferred_words); + + bool expand_by(size_t min_words, + size_t preferred_words); + + VirtualSpaceNode* current_virtual_space() { + return _current_virtual_space; + } + + bool is_class() const { return _is_class; } + + bool initialization_succeeded() { return _virtual_space_list != NULL; } + + size_t reserved_words() { return _reserved_words; } + size_t reserved_bytes() { return reserved_words() * BytesPerWord; } + size_t committed_words() { return _committed_words; } + size_t committed_bytes() { return committed_words() * BytesPerWord; } + + void inc_reserved_words(size_t v); + void dec_reserved_words(size_t v); + void inc_committed_words(size_t v); + void dec_committed_words(size_t v); + void inc_virtual_space_count(); + void dec_virtual_space_count(); + + bool contains(const void* ptr); + + // Unlink empty VirtualSpaceNodes and free it. + void purge(ChunkManager* chunk_manager); + + void print_on(outputStream* st) const { print_on(st, K); } + void print_on(outputStream* st, size_t scale) const; + void print_map(outputStream* st) const; + + class VirtualSpaceListIterator : public StackObj { + VirtualSpaceNode* _virtual_spaces; + public: + VirtualSpaceListIterator(VirtualSpaceNode* virtual_spaces) : + _virtual_spaces(virtual_spaces) {} + + bool repeat() { + return _virtual_spaces != NULL; + } + + VirtualSpaceNode* get_next() { + VirtualSpaceNode* result = _virtual_spaces; + if (_virtual_spaces != NULL) { + _virtual_spaces = _virtual_spaces->next(); + } + return result; + } + }; +}; + +} // namespace metaspace + +#endif /* SHARE_MEMORY_METASPACE_VIRTUALSPACELIST_HPP_ */ + diff --git a/src/hotspot/share/memory/metaspace/virtualSpaceNode.cpp b/src/hotspot/share/memory/metaspace/virtualSpaceNode.cpp new file mode 100644 --- /dev/null +++ b/src/hotspot/share/memory/metaspace/virtualSpaceNode.cpp @@ -0,0 +1,596 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" + +#include "logging/log.hpp" +#include "logging/logStream.hpp" +#include "memory/metaspace/metachunk.hpp" +#include "memory/metaspace.hpp" +#include "memory/metaspace/chunkManager.hpp" +#include "memory/metaspace/metaspaceCommon.hpp" +#include "memory/metaspace/occupancyMap.hpp" +#include "memory/metaspace/virtualSpaceNode.hpp" +#include "memory/virtualspace.hpp" +#include "runtime/os.hpp" +#include "services/memTracker.hpp" +#include "utilities/copy.hpp" +#include "utilities/debug.hpp" +#include "utilities/globalDefinitions.hpp" + +namespace metaspace { + +// Decide if large pages should be committed when the memory is reserved. +static bool should_commit_large_pages_when_reserving(size_t bytes) { + if (UseLargePages && UseLargePagesInMetaspace && !os::can_commit_large_page_memory()) { + size_t words = bytes / BytesPerWord; + bool is_class = false; // We never reserve large pages for the class space. + if (MetaspaceGC::can_expand(words, is_class) && + MetaspaceGC::allowed_expansion() >= words) { + return true; + } + } + + return false; +} + +// byte_size is the size of the associated virtualspace. +VirtualSpaceNode::VirtualSpaceNode(bool is_class, size_t bytes) : + _is_class(is_class), _top(NULL), _next(NULL), _rs(), _container_count(0), _occupancy_map(NULL) { + assert_is_aligned(bytes, Metaspace::reserve_alignment()); + bool large_pages = should_commit_large_pages_when_reserving(bytes); + _rs = ReservedSpace(bytes, Metaspace::reserve_alignment(), large_pages); + + if (_rs.is_reserved()) { + assert(_rs.base() != NULL, "Catch if we get a NULL address"); + assert(_rs.size() != 0, "Catch if we get a 0 size"); + assert_is_aligned(_rs.base(), Metaspace::reserve_alignment()); + assert_is_aligned(_rs.size(), Metaspace::reserve_alignment()); + + MemTracker::record_virtual_memory_type((address)_rs.base(), mtClass); + } +} + +void VirtualSpaceNode::purge(ChunkManager* chunk_manager) { + DEBUG_ONLY(this->verify();) + Metachunk* chunk = first_chunk(); + Metachunk* invalid_chunk = (Metachunk*) top(); + while (chunk < invalid_chunk ) { + assert(chunk->is_tagged_free(), "Should be tagged free"); + MetaWord* next = ((MetaWord*)chunk) + chunk->word_size(); + chunk_manager->remove_chunk(chunk); + chunk->remove_sentinel(); + assert(chunk->next() == NULL && + chunk->prev() == NULL, + "Was not removed from its list"); + chunk = (Metachunk*) next; + } +} + +void VirtualSpaceNode::print_map(outputStream* st, bool is_class) const { + + if (bottom() == top()) { + return; + } + + const size_t spec_chunk_size = is_class ? ClassSpecializedChunk : SpecializedChunk; + const size_t small_chunk_size = is_class ? ClassSmallChunk : SmallChunk; + const size_t med_chunk_size = is_class ? ClassMediumChunk : MediumChunk; + + int line_len = 100; + const size_t section_len = align_up(spec_chunk_size * line_len, med_chunk_size); + line_len = (int)(section_len / spec_chunk_size); + + static const int NUM_LINES = 4; + + char* lines[NUM_LINES]; + for (int i = 0; i < NUM_LINES; i ++) { + lines[i] = (char*)os::malloc(line_len, mtInternal); + } + int pos = 0; + const MetaWord* p = bottom(); + const Metachunk* chunk = (const Metachunk*)p; + const MetaWord* chunk_end = p + chunk->word_size(); + while (p < top()) { + if (pos == line_len) { + pos = 0; + for (int i = 0; i < NUM_LINES; i ++) { + st->fill_to(22); + st->print_raw(lines[i], line_len); + st->cr(); + } + } + if (pos == 0) { + st->print(PTR_FORMAT ":", p2i(p)); + } + if (p == chunk_end) { + chunk = (Metachunk*)p; + chunk_end = p + chunk->word_size(); + } + // line 1: chunk starting points (a dot if that area is a chunk start). + lines[0][pos] = p == (const MetaWord*)chunk ? '.' : ' '; + + // Line 2: chunk type (x=spec, s=small, m=medium, h=humongous), uppercase if + // chunk is in use. + const bool chunk_is_free = ((Metachunk*)chunk)->is_tagged_free(); + if (chunk->word_size() == spec_chunk_size) { + lines[1][pos] = chunk_is_free ? 'x' : 'X'; + } else if (chunk->word_size() == small_chunk_size) { + lines[1][pos] = chunk_is_free ? 's' : 'S'; + } else if (chunk->word_size() == med_chunk_size) { + lines[1][pos] = chunk_is_free ? 'm' : 'M'; + } else if (chunk->word_size() > med_chunk_size) { + lines[1][pos] = chunk_is_free ? 'h' : 'H'; + } else { + ShouldNotReachHere(); + } + + // Line 3: chunk origin + const ChunkOrigin origin = chunk->get_origin(); + lines[2][pos] = origin == origin_normal ? ' ' : '0' + (int) origin; + + // Line 4: Virgin chunk? Virgin chunks are chunks created as a byproduct of padding or splitting, + // but were never used. + lines[3][pos] = chunk->get_use_count() > 0 ? ' ' : 'v'; + + p += spec_chunk_size; + pos ++; + } + if (pos > 0) { + for (int i = 0; i < NUM_LINES; i ++) { + st->fill_to(22); + st->print_raw(lines[i], line_len); + st->cr(); + } + } + for (int i = 0; i < NUM_LINES; i ++) { + os::free(lines[i]); + } +} + + +#ifdef ASSERT +uintx VirtualSpaceNode::container_count_slow() { + uintx count = 0; + Metachunk* chunk = first_chunk(); + Metachunk* invalid_chunk = (Metachunk*) top(); + while (chunk < invalid_chunk ) { + MetaWord* next = ((MetaWord*)chunk) + chunk->word_size(); + do_verify_chunk(chunk); + // Don't count the chunks on the free lists. Those are + // still part of the VirtualSpaceNode but not currently + // counted. + if (!chunk->is_tagged_free()) { + count++; + } + chunk = (Metachunk*) next; + } + return count; +} +#endif + +#ifdef ASSERT +// Verify counters, all chunks in this list node and the occupancy map. +void VirtualSpaceNode::verify() { + uintx num_in_use_chunks = 0; + Metachunk* chunk = first_chunk(); + Metachunk* invalid_chunk = (Metachunk*) top(); + + // Iterate the chunks in this node and verify each chunk. + while (chunk < invalid_chunk ) { + DEBUG_ONLY(do_verify_chunk(chunk);) + if (!chunk->is_tagged_free()) { + num_in_use_chunks ++; + } + MetaWord* next = ((MetaWord*)chunk) + chunk->word_size(); + chunk = (Metachunk*) next; + } + assert(_container_count == num_in_use_chunks, "Container count mismatch (real: " UINTX_FORMAT + ", counter: " UINTX_FORMAT ".", num_in_use_chunks, _container_count); + // Also verify the occupancy map. + occupancy_map()->verify(this->bottom(), this->top()); +} +#endif // ASSERT + +#ifdef ASSERT +// Verify that all free chunks in this node are ideally merged +// (there not should be multiple small chunks where a large chunk could exist.) +void VirtualSpaceNode::verify_free_chunks_are_ideally_merged() { + Metachunk* chunk = first_chunk(); + Metachunk* invalid_chunk = (Metachunk*) top(); + // Shorthands. + const size_t size_med = (is_class() ? ClassMediumChunk : MediumChunk) * BytesPerWord; + const size_t size_small = (is_class() ? ClassSmallChunk : SmallChunk) * BytesPerWord; + int num_free_chunks_since_last_med_boundary = -1; + int num_free_chunks_since_last_small_boundary = -1; + while (chunk < invalid_chunk ) { + // Test for missed chunk merge opportunities: count number of free chunks since last chunk boundary. + // Reset the counter when encountering a non-free chunk. + if (chunk->get_chunk_type() != HumongousIndex) { + if (chunk->is_tagged_free()) { + // Count successive free, non-humongous chunks. + if (is_aligned(chunk, size_small)) { + assert(num_free_chunks_since_last_small_boundary <= 1, + "Missed chunk merge opportunity at " PTR_FORMAT " for chunk size " SIZE_FORMAT_HEX ".", p2i(chunk) - size_small, size_small); + num_free_chunks_since_last_small_boundary = 0; + } else if (num_free_chunks_since_last_small_boundary != -1) { + num_free_chunks_since_last_small_boundary ++; + } + if (is_aligned(chunk, size_med)) { + assert(num_free_chunks_since_last_med_boundary <= 1, + "Missed chunk merge opportunity at " PTR_FORMAT " for chunk size " SIZE_FORMAT_HEX ".", p2i(chunk) - size_med, size_med); + num_free_chunks_since_last_med_boundary = 0; + } else if (num_free_chunks_since_last_med_boundary != -1) { + num_free_chunks_since_last_med_boundary ++; + } + } else { + // Encountering a non-free chunk, reset counters. + num_free_chunks_since_last_med_boundary = -1; + num_free_chunks_since_last_small_boundary = -1; + } + } else { + // One cannot merge areas with a humongous chunk in the middle. Reset counters. + num_free_chunks_since_last_med_boundary = -1; + num_free_chunks_since_last_small_boundary = -1; + } + + MetaWord* next = ((MetaWord*)chunk) + chunk->word_size(); + chunk = (Metachunk*) next; + } +} +#endif // ASSERT + +void VirtualSpaceNode::inc_container_count() { + assert_lock_strong(MetaspaceExpand_lock); + _container_count++; +} + +void VirtualSpaceNode::dec_container_count() { + assert_lock_strong(MetaspaceExpand_lock); + _container_count--; +} + +#ifdef ASSERT +void VirtualSpaceNode::verify_container_count() { + assert(_container_count == container_count_slow(), + "Inconsistency in container_count _container_count " UINTX_FORMAT + " container_count_slow() " UINTX_FORMAT, _container_count, container_count_slow()); +} +#endif + +VirtualSpaceNode::~VirtualSpaceNode() { + _rs.release(); + if (_occupancy_map != NULL) { + delete _occupancy_map; + } +#ifdef ASSERT + size_t word_size = sizeof(*this) / BytesPerWord; + Copy::fill_to_words((HeapWord*) this, word_size, 0xf1f1f1f1); +#endif +} + +size_t VirtualSpaceNode::used_words_in_vs() const { + return pointer_delta(top(), bottom(), sizeof(MetaWord)); +} + +// Space committed in the VirtualSpace +size_t VirtualSpaceNode::capacity_words_in_vs() const { + return pointer_delta(end(), bottom(), sizeof(MetaWord)); +} + +size_t VirtualSpaceNode::free_words_in_vs() const { + return pointer_delta(end(), top(), sizeof(MetaWord)); +} + +// Given an address larger than top(), allocate padding chunks until top is at the given address. +void VirtualSpaceNode::allocate_padding_chunks_until_top_is_at(MetaWord* target_top) { + + assert(target_top > top(), "Sanity"); + + // Padding chunks are added to the freelist. + ChunkManager* const chunk_manager = Metaspace::get_chunk_manager(this->is_class()); + + // shorthands + const size_t spec_word_size = chunk_manager->specialized_chunk_word_size(); + const size_t small_word_size = chunk_manager->small_chunk_word_size(); + const size_t med_word_size = chunk_manager->medium_chunk_word_size(); + + while (top() < target_top) { + + // We could make this coding more generic, but right now we only deal with two possible chunk sizes + // for padding chunks, so it is not worth it. + size_t padding_chunk_word_size = small_word_size; + if (is_aligned(top(), small_word_size * sizeof(MetaWord)) == false) { + assert_is_aligned(top(), spec_word_size * sizeof(MetaWord)); // Should always hold true. + padding_chunk_word_size = spec_word_size; + } + MetaWord* here = top(); + assert_is_aligned(here, padding_chunk_word_size * sizeof(MetaWord)); + inc_top(padding_chunk_word_size); + + // Create new padding chunk. + ChunkIndex padding_chunk_type = get_chunk_type_by_size(padding_chunk_word_size, is_class()); + assert(padding_chunk_type == SpecializedIndex || padding_chunk_type == SmallIndex, "sanity"); + + Metachunk* const padding_chunk = + ::new (here) Metachunk(padding_chunk_type, is_class(), padding_chunk_word_size, this); + assert(padding_chunk == (Metachunk*)here, "Sanity"); + DEBUG_ONLY(padding_chunk->set_origin(origin_pad);) + log_trace(gc, metaspace, freelist)("Created padding chunk in %s at " + PTR_FORMAT ", size " SIZE_FORMAT_HEX ".", + (is_class() ? "class space " : "metaspace"), + p2i(padding_chunk), padding_chunk->word_size() * sizeof(MetaWord)); + + // Mark chunk start in occupancy map. + occupancy_map()->set_chunk_starts_at_address((MetaWord*)padding_chunk, true); + + // Chunks are born as in-use (see MetaChunk ctor). So, before returning + // the padding chunk to its chunk manager, mark it as in use (ChunkManager + // will assert that). + do_update_in_use_info_for_chunk(padding_chunk, true); + + // Return Chunk to freelist. + inc_container_count(); + chunk_manager->return_single_chunk(padding_chunk); + // Please note: at this point, ChunkManager::return_single_chunk() + // may already have merged the padding chunk with neighboring chunks, so + // it may have vanished at this point. Do not reference the padding + // chunk beyond this point. + } + + assert(top() == target_top, "Sanity"); + +} // allocate_padding_chunks_until_top_is_at() + +// Allocates the chunk from the virtual space only. +// This interface is also used internally for debugging. Not all +// chunks removed here are necessarily used for allocation. +Metachunk* VirtualSpaceNode::take_from_committed(size_t chunk_word_size) { + // Non-humongous chunks are to be allocated aligned to their chunk + // size. So, start addresses of medium chunks are aligned to medium + // chunk size, those of small chunks to small chunk size and so + // forth. This facilitates merging of free chunks and reduces + // fragmentation. Chunk sizes are spec < small < medium, with each + // larger chunk size being a multiple of the next smaller chunk + // size. + // Because of this alignment, me may need to create a number of padding + // chunks. These chunks are created and added to the freelist. + + // The chunk manager to which we will give our padding chunks. + ChunkManager* const chunk_manager = Metaspace::get_chunk_manager(this->is_class()); + + // shorthands + const size_t spec_word_size = chunk_manager->specialized_chunk_word_size(); + const size_t small_word_size = chunk_manager->small_chunk_word_size(); + const size_t med_word_size = chunk_manager->medium_chunk_word_size(); + + assert(chunk_word_size == spec_word_size || chunk_word_size == small_word_size || + chunk_word_size >= med_word_size, "Invalid chunk size requested."); + + // Chunk alignment (in bytes) == chunk size unless humongous. + // Humongous chunks are aligned to the smallest chunk size (spec). + const size_t required_chunk_alignment = (chunk_word_size > med_word_size ? + spec_word_size : chunk_word_size) * sizeof(MetaWord); + + // Do we have enough space to create the requested chunk plus + // any padding chunks needed? + MetaWord* const next_aligned = + static_cast(align_up(top(), required_chunk_alignment)); + if (!is_available((next_aligned - top()) + chunk_word_size)) { + return NULL; + } + + // Before allocating the requested chunk, allocate padding chunks if necessary. + // We only need to do this for small or medium chunks: specialized chunks are the + // smallest size, hence always aligned. Homungous chunks are allocated unaligned + // (implicitly, also aligned to smallest chunk size). + if ((chunk_word_size == med_word_size || chunk_word_size == small_word_size) && next_aligned > top()) { + log_trace(gc, metaspace, freelist)("Creating padding chunks in %s between %p and %p...", + (is_class() ? "class space " : "metaspace"), + top(), next_aligned); + allocate_padding_chunks_until_top_is_at(next_aligned); + // Now, top should be aligned correctly. + assert_is_aligned(top(), required_chunk_alignment); + } + + // Now, top should be aligned correctly. + assert_is_aligned(top(), required_chunk_alignment); + + // Bottom of the new chunk + MetaWord* chunk_limit = top(); + assert(chunk_limit != NULL, "Not safe to call this method"); + + // The virtual spaces are always expanded by the + // commit granularity to enforce the following condition. + // Without this the is_available check will not work correctly. + assert(_virtual_space.committed_size() == _virtual_space.actual_committed_size(), + "The committed memory doesn't match the expanded memory."); + + if (!is_available(chunk_word_size)) { + LogTarget(Debug, gc, metaspace, freelist) lt; + if (lt.is_enabled()) { + LogStream ls(lt); + ls.print("VirtualSpaceNode::take_from_committed() not available " SIZE_FORMAT " words ", chunk_word_size); + // Dump some information about the virtual space that is nearly full + print_on(&ls); + } + return NULL; + } + + // Take the space (bump top on the current virtual space). + inc_top(chunk_word_size); + + // Initialize the chunk + ChunkIndex chunk_type = get_chunk_type_by_size(chunk_word_size, is_class()); + Metachunk* result = ::new (chunk_limit) Metachunk(chunk_type, is_class(), chunk_word_size, this); + assert(result == (Metachunk*)chunk_limit, "Sanity"); + occupancy_map()->set_chunk_starts_at_address((MetaWord*)result, true); + do_update_in_use_info_for_chunk(result, true); + + inc_container_count(); + + if (VerifyMetaspace) { + DEBUG_ONLY(chunk_manager->locked_verify()); + DEBUG_ONLY(this->verify()); + } + + DEBUG_ONLY(do_verify_chunk(result)); + + result->inc_use_count(); + + return result; +} + + +// Expand the virtual space (commit more of the reserved space) +bool VirtualSpaceNode::expand_by(size_t min_words, size_t preferred_words) { + size_t min_bytes = min_words * BytesPerWord; + size_t preferred_bytes = preferred_words * BytesPerWord; + + size_t uncommitted = virtual_space()->reserved_size() - virtual_space()->actual_committed_size(); + + if (uncommitted < min_bytes) { + return false; + } + + size_t commit = MIN2(preferred_bytes, uncommitted); + bool result = virtual_space()->expand_by(commit, false); + + if (result) { + log_trace(gc, metaspace, freelist)("Expanded %s virtual space list node by " SIZE_FORMAT " words.", + (is_class() ? "class" : "non-class"), commit); + DEBUG_ONLY(Atomic::inc(&g_internal_statistics.num_committed_space_expanded)); + } else { + log_trace(gc, metaspace, freelist)("Failed to expand %s virtual space list node by " SIZE_FORMAT " words.", + (is_class() ? "class" : "non-class"), commit); + } + + assert(result, "Failed to commit memory"); + + return result; +} + +Metachunk* VirtualSpaceNode::get_chunk_vs(size_t chunk_word_size) { + assert_lock_strong(MetaspaceExpand_lock); + Metachunk* result = take_from_committed(chunk_word_size); + return result; +} + +bool VirtualSpaceNode::initialize() { + + if (!_rs.is_reserved()) { + return false; + } + + // These are necessary restriction to make sure that the virtual space always + // grows in steps of Metaspace::commit_alignment(). If both base and size are + // aligned only the middle alignment of the VirtualSpace is used. + assert_is_aligned(_rs.base(), Metaspace::commit_alignment()); + assert_is_aligned(_rs.size(), Metaspace::commit_alignment()); + + // ReservedSpaces marked as special will have the entire memory + // pre-committed. Setting a committed size will make sure that + // committed_size and actual_committed_size agrees. + size_t pre_committed_size = _rs.special() ? _rs.size() : 0; + + bool result = virtual_space()->initialize_with_granularity(_rs, pre_committed_size, + Metaspace::commit_alignment()); + if (result) { + assert(virtual_space()->committed_size() == virtual_space()->actual_committed_size(), + "Checking that the pre-committed memory was registered by the VirtualSpace"); + + set_top((MetaWord*)virtual_space()->low()); + set_reserved(MemRegion((HeapWord*)_rs.base(), + (HeapWord*)(_rs.base() + _rs.size()))); + + assert(reserved()->start() == (HeapWord*) _rs.base(), + "Reserved start was not set properly " PTR_FORMAT + " != " PTR_FORMAT, p2i(reserved()->start()), p2i(_rs.base())); + assert(reserved()->word_size() == _rs.size() / BytesPerWord, + "Reserved size was not set properly " SIZE_FORMAT + " != " SIZE_FORMAT, reserved()->word_size(), + _rs.size() / BytesPerWord); + } + + // Initialize Occupancy Map. + const size_t smallest_chunk_size = is_class() ? ClassSpecializedChunk : SpecializedChunk; + _occupancy_map = new OccupancyMap(bottom(), reserved_words(), smallest_chunk_size); + + return result; +} + +void VirtualSpaceNode::print_on(outputStream* st, size_t scale) const { + size_t used_words = used_words_in_vs(); + size_t commit_words = committed_words(); + size_t res_words = reserved_words(); + VirtualSpace* vs = virtual_space(); + + st->print("node @" PTR_FORMAT ": ", p2i(this)); + st->print("reserved="); + print_scaled_words(st, res_words, scale); + st->print(", committed="); + print_scaled_words_and_percentage(st, commit_words, res_words, scale); + st->print(", used="); + print_scaled_words_and_percentage(st, used_words, res_words, scale); + st->cr(); + st->print(" [" PTR_FORMAT ", " PTR_FORMAT ", " + PTR_FORMAT ", " PTR_FORMAT ")", + p2i(bottom()), p2i(top()), p2i(end()), + p2i(vs->high_boundary())); +} + +#ifdef ASSERT +void VirtualSpaceNode::mangle() { + size_t word_size = capacity_words_in_vs(); + Copy::fill_to_words((HeapWord*) low(), word_size, 0xf1f1f1f1); +} +#endif // ASSERT + +void VirtualSpaceNode::retire(ChunkManager* chunk_manager) { + DEBUG_ONLY(verify_container_count();) + assert(this->is_class() == chunk_manager->is_class(), "Wrong ChunkManager?"); + for (int i = (int)MediumIndex; i >= (int)ZeroIndex; --i) { + ChunkIndex index = (ChunkIndex)i; + size_t chunk_size = chunk_manager->size_by_index(index); + + while (free_words_in_vs() >= chunk_size) { + Metachunk* chunk = get_chunk_vs(chunk_size); + // Chunk will be allocated aligned, so allocation may require + // additional padding chunks. That may cause above allocation to + // fail. Just ignore the failed allocation and continue with the + // next smaller chunk size. As the VirtualSpaceNode comitted + // size should be a multiple of the smallest chunk size, we + // should always be able to fill the VirtualSpace completely. + if (chunk == NULL) { + break; + } + chunk_manager->return_single_chunk(chunk); + } + DEBUG_ONLY(verify_container_count();) + } + assert(free_words_in_vs() == 0, "should be empty now"); +} + +} // namespace metaspace + diff --git a/src/hotspot/share/memory/metaspace/virtualSpaceNode.hpp b/src/hotspot/share/memory/metaspace/virtualSpaceNode.hpp new file mode 100644 --- /dev/null +++ b/src/hotspot/share/memory/metaspace/virtualSpaceNode.hpp @@ -0,0 +1,169 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_MEMORY_METASPACE_VIRTUALSPACENODE_HPP_ +#define SHARE_MEMORY_METASPACE_VIRTUALSPACENODE_HPP_ + +#include "memory/virtualspace.hpp" +#include "memory/memRegion.hpp" +#include "utilities/debug.hpp" +#include "utilities/globalDefinitions.hpp" + +class outputStream; + +namespace metaspace { + +class Metachunk; +class ChunkManager; +class OccupancyMap; + +// A VirtualSpaceList node. +class VirtualSpaceNode : public CHeapObj { + friend class VirtualSpaceList; + + // Link to next VirtualSpaceNode + VirtualSpaceNode* _next; + + // Whether this node is contained in class or metaspace. + const bool _is_class; + + // total in the VirtualSpace + MemRegion _reserved; + ReservedSpace _rs; + VirtualSpace _virtual_space; + MetaWord* _top; + // count of chunks contained in this VirtualSpace + uintx _container_count; + + OccupancyMap* _occupancy_map; + + // Convenience functions to access the _virtual_space + char* low() const { return virtual_space()->low(); } + char* high() const { return virtual_space()->high(); } + + // The first Metachunk will be allocated at the bottom of the + // VirtualSpace + Metachunk* first_chunk() { return (Metachunk*) bottom(); } + + // Committed but unused space in the virtual space + size_t free_words_in_vs() const; + + // True if this node belongs to class metaspace. + bool is_class() const { return _is_class; } + + // Helper function for take_from_committed: allocate padding chunks + // until top is at the given address. + void allocate_padding_chunks_until_top_is_at(MetaWord* target_top); + + public: + + VirtualSpaceNode(bool is_class, size_t byte_size); + VirtualSpaceNode(bool is_class, ReservedSpace rs) : + _is_class(is_class), _top(NULL), _next(NULL), _rs(rs), _container_count(0), _occupancy_map(NULL) {} + ~VirtualSpaceNode(); + + // Convenience functions for logical bottom and end + MetaWord* bottom() const { return (MetaWord*) _virtual_space.low(); } + MetaWord* end() const { return (MetaWord*) _virtual_space.high(); } + + const OccupancyMap* occupancy_map() const { return _occupancy_map; } + OccupancyMap* occupancy_map() { return _occupancy_map; } + + bool contains(const void* ptr) { return ptr >= low() && ptr < high(); } + + size_t reserved_words() const { return _virtual_space.reserved_size() / BytesPerWord; } + size_t committed_words() const { return _virtual_space.actual_committed_size() / BytesPerWord; } + + bool is_pre_committed() const { return _virtual_space.special(); } + + // address of next available space in _virtual_space; + // Accessors + VirtualSpaceNode* next() { return _next; } + void set_next(VirtualSpaceNode* v) { _next = v; } + + void set_reserved(MemRegion const v) { _reserved = v; } + void set_top(MetaWord* v) { _top = v; } + + // Accessors + MemRegion* reserved() { return &_reserved; } + VirtualSpace* virtual_space() const { return (VirtualSpace*) &_virtual_space; } + + // Returns true if "word_size" is available in the VirtualSpace + bool is_available(size_t word_size) { return word_size <= pointer_delta(end(), _top, sizeof(MetaWord)); } + + MetaWord* top() const { return _top; } + void inc_top(size_t word_size) { _top += word_size; } + + uintx container_count() { return _container_count; } + void inc_container_count(); + void dec_container_count(); +#ifdef ASSERT + uintx container_count_slow(); + void verify_container_count(); +#endif + + // used and capacity in this single entry in the list + size_t used_words_in_vs() const; + size_t capacity_words_in_vs() const; + + bool initialize(); + + // get space from the virtual space + Metachunk* take_from_committed(size_t chunk_word_size); + + // Allocate a chunk from the virtual space and return it. + Metachunk* get_chunk_vs(size_t chunk_word_size); + + // Expands/shrinks the committed space in a virtual space. Delegates + // to Virtualspace + bool expand_by(size_t min_words, size_t preferred_words); + + // In preparation for deleting this node, remove all the chunks + // in the node from any freelist. + void purge(ChunkManager* chunk_manager); + + // If an allocation doesn't fit in the current node a new node is created. + // Allocate chunks out of the remaining committed space in this node + // to avoid wasting that memory. + // This always adds up because all the chunk sizes are multiples of + // the smallest chunk size. + void retire(ChunkManager* chunk_manager); + + void print_on(outputStream* st) const { print_on(st, K); } + void print_on(outputStream* st, size_t scale) const; + void print_map(outputStream* st, bool is_class) const; + + // Debug support + DEBUG_ONLY(void mangle();) + // Verify counters, all chunks in this list node and the occupancy map. + DEBUG_ONLY(void verify();) + // Verify that all free chunks in this node are ideally merged + // (there not should be multiple small chunks where a large chunk could exist.) + DEBUG_ONLY(void verify_free_chunks_are_ideally_merged();) + +}; + +} // namespace metaspace + +#endif /* SHARE_MEMORY_METASPACE_VIRTUALSPACENODE_HPP_ */ diff --git a/src/hotspot/share/runtime/vmStructs.cpp b/src/hotspot/share/runtime/vmStructs.cpp --- a/src/hotspot/share/runtime/vmStructs.cpp +++ b/src/hotspot/share/runtime/vmStructs.cpp @@ -51,7 +51,7 @@ #include "memory/allocation.hpp" #include "memory/allocation.inline.hpp" #include "memory/heap.hpp" -#include "memory/metachunk.hpp" +#include "memory/metaspace/metablock.hpp" #include "memory/padded.hpp" #include "memory/referenceType.hpp" #include "memory/universe.hpp" diff --git a/test/hotspot/gtest/memory/test_chunkManager.cpp b/test/hotspot/gtest/memory/test_chunkManager.cpp --- a/test/hotspot/gtest/memory/test_chunkManager.cpp +++ b/test/hotspot/gtest/memory/test_chunkManager.cpp @@ -22,18 +22,47 @@ */ #include "precompiled.hpp" +#include "memory/metaspace/chunkManager.hpp" +#include "memory/metaspace/metaspaceCommon.hpp" // The test function is only available in debug builds #ifdef ASSERT #include "unittest.hpp" -void ChunkManager_test_list_index(); +using namespace metaspace; TEST(ChunkManager, list_index) { - // The ChunkManager is only available in metaspace.cpp, - // so the test code is located in that file. - ChunkManager_test_list_index(); + + // Test previous bug where a query for a humongous class metachunk, + // incorrectly matched the non-class medium metachunk size. + { + ChunkManager manager(true); + + ASSERT_TRUE(MediumChunk > ClassMediumChunk) << "Precondition for test"; + + ChunkIndex index = manager.list_index(MediumChunk); + + ASSERT_TRUE(index == HumongousIndex) << + "Requested size is larger than ClassMediumChunk," + " so should return HumongousIndex. Got index: " << index; + } + + // Check the specified sizes as well. + { + ChunkManager manager(true); + ASSERT_TRUE(manager.list_index(ClassSpecializedChunk) == SpecializedIndex); + ASSERT_TRUE(manager.list_index(ClassSmallChunk) == SmallIndex); + ASSERT_TRUE(manager.list_index(ClassMediumChunk) == MediumIndex); + ASSERT_TRUE(manager.list_index(ClassMediumChunk + ClassSpecializedChunk) == HumongousIndex); + } + { + ChunkManager manager(false); + ASSERT_TRUE(manager.list_index(SpecializedChunk) == SpecializedIndex); + ASSERT_TRUE(manager.list_index(SmallChunk) == SmallIndex); + ASSERT_TRUE(manager.list_index(MediumChunk) == MediumIndex); + ASSERT_TRUE(manager.list_index(MediumChunk + SpecializedChunk) == HumongousIndex); + } } diff --git a/test/hotspot/gtest/memory/test_metachunk.cpp b/test/hotspot/gtest/memory/test_metachunk.cpp --- a/test/hotspot/gtest/memory/test_metachunk.cpp +++ b/test/hotspot/gtest/memory/test_metachunk.cpp @@ -23,12 +23,14 @@ #include "precompiled.hpp" #include "memory/allocation.hpp" -#include "memory/metachunk.hpp" +#include "memory/metaspace/metachunk.hpp" #include "unittest.hpp" #include "utilities/align.hpp" #include "utilities/copy.hpp" #include "utilities/debug.hpp" +using namespace metaspace; + class MetachunkTest { public: static MetaWord* initial_top(Metachunk* metachunk) { diff --git a/test/hotspot/gtest/memory/test_spaceManager.cpp b/test/hotspot/gtest/memory/test_spaceManager.cpp --- a/test/hotspot/gtest/memory/test_spaceManager.cpp +++ b/test/hotspot/gtest/memory/test_spaceManager.cpp @@ -22,18 +22,53 @@ */ #include "precompiled.hpp" +#include "memory/metaspace/spaceManager.hpp" + +using metaspace::SpaceManager; // The test function is only available in debug builds #ifdef ASSERT #include "unittest.hpp" -void SpaceManager_test_adjust_initial_chunk_size(); + +static void test_adjust_initial_chunk_size(bool is_class) { + const size_t smallest = SpaceManager::smallest_chunk_size(is_class); + const size_t normal = SpaceManager::small_chunk_size(is_class); + const size_t medium = SpaceManager::medium_chunk_size(is_class); + +#define do_test(value, expected, is_class_value) \ + do { \ + size_t v = value; \ + size_t e = expected; \ + assert(SpaceManager::adjust_initial_chunk_size(v, (is_class_value)) == e, \ + "Expected: " SIZE_FORMAT " got: " SIZE_FORMAT, e, v); \ + } while (0) + + // Smallest (specialized) + do_test(1, smallest, is_class); + do_test(smallest - 1, smallest, is_class); + do_test(smallest, smallest, is_class); + + // Small + do_test(smallest + 1, normal, is_class); + do_test(normal - 1, normal, is_class); + do_test(normal, normal, is_class); + + // Medium + do_test(normal + 1, medium, is_class); + do_test(medium - 1, medium, is_class); + do_test(medium, medium, is_class); + + // Humongous + do_test(medium + 1, medium + 1, is_class); + +#undef test_adjust_initial_chunk_size +} TEST(SpaceManager, adjust_initial_chunk_size) { - // The SpaceManager is only available in metaspace.cpp, - // so the test code is located in that file. - SpaceManager_test_adjust_initial_chunk_size(); + test_adjust_initial_chunk_size(true); + test_adjust_initial_chunk_size(false); } #endif // ASSERT