# HG changeset patch # User stuefe # Date 1558541157 -7200 # Wed May 22 18:05:57 2019 +0200 # Branch metaspace-improved # Node ID 902274e61ebb8051aeadedde1ed790bd68912d3c # Parent 0927d8c7296ffa7b7715dc7af6ac361b6a9a6d9d [mq]: metaspace-prototype diff -r 0927d8c7296f -r 902274e61ebb src/hotspot/share/memory/metaspace.cpp --- a/src/hotspot/share/memory/metaspace.cpp Wed May 22 07:10:54 2019 +0200 +++ b/src/hotspot/share/memory/metaspace.cpp Wed May 22 18:05:57 2019 +0200 @@ -1450,7 +1450,7 @@ size_t chunk_word_size = get_space_manager(mdtype)->get_initial_chunk_size(type); // Get a chunk from the chunk freelist - Metachunk* chunk = Metaspace::get_chunk_manager(mdtype)->chunk_freelist_allocate(chunk_word_size); + Metachunk* chunk = Metaspace::get_chunk_manager(mdtype)->get_chunk(chunk_word_size); if (chunk == NULL) { chunk = Metaspace::get_space_list(mdtype)->get_new_chunk(chunk_word_size, diff -r 0927d8c7296f -r 902274e61ebb src/hotspot/share/memory/metaspace/chunkManager.cpp --- a/src/hotspot/share/memory/metaspace/chunkManager.cpp Wed May 22 07:10:54 2019 +0200 +++ b/src/hotspot/share/memory/metaspace/chunkManager.cpp Wed May 22 18:05:57 2019 +0200 @@ -41,26 +41,161 @@ namespace metaspace { -ChunkManager::ChunkManager(bool is_class) - : _is_class(is_class), _free_chunks_total(0), _free_chunks_count(0) { - _free_chunks[SpecializedIndex].set_size(get_size_for_nonhumongous_chunktype(SpecializedIndex, is_class)); - _free_chunks[SmallIndex].set_size(get_size_for_nonhumongous_chunktype(SmallIndex, is_class)); - _free_chunks[MediumIndex].set_size(get_size_for_nonhumongous_chunktype(MediumIndex, is_class)); +ChunkManager::ChunkManager(const char* name) + : _name(name), + _free_chunks_total(0), + _free_chunks_count(0) +{ + for (int lvl = 0; lvl < NUM_CHUNK_LEVELS; lvl ++) { + _free_chunks[lvl] = NULL; + _num_free_chunks[lvl] = 0; + } } -void ChunkManager::remove_chunk(Metachunk* chunk) { - size_t word_size = chunk->word_size(); - ChunkIndex index = list_index(word_size); - if (index != HumongousIndex) { - free_chunks(index)->remove_chunk(chunk); - } else { - humongous_dictionary()->remove_chunk(chunk); +#ifdef ASSERT +// Returns true if this manager contains the given chunk. Slow (walks free list) and +// only needed for verifications. +bool ChunkManager::contains_chunk(Metachunk* c) const { + assert(c != NULL, "sanity"); + const int lvl = c->level(); + assert(is_valid_level(lvl), "sanity"); + Metachunk* c2 = _free_chunks[lvl]; + while (c2 != NULL && c2 != c) { + c2 = c2->next(); + } + return c2 == c; +} +#endif + +void ChunkManager::remove_chunk(Metachunk* c) { +#ifdef ASSERT + EVERY_NTH(VerifyMetaspaceInterval) + assert(contains_chunk(c), "chunk is not managed by this chunk manager"); + c->verify(); + END_EVERY_NTH +#endif + + c->remove_from_list(); + account_for_removed_chunk(c); +} + +// Add a chunk to the free list (no coalescation, just an add) +// and adjust accounting. +void ChunkManager::add_chunk(Metachunk* c) { + int lvl = c->level(); + DEBUG_ONLY(check_valid_level(lvl)); + if (_free_chunks[lvl] != NULL) { + _free_chunks[lvl]->set_prev(c); + } + c->set_next(_free_chunks[lvl]); + _free_chunks[lvl] = c; + c->set_prev(NULL); + account_for_added_chunk(c); +} + +// Remove a chunk of the given level from its free list and return it; returns NULL +// if that free list is empty. The returned chunk will be marked as "in use". +// Adjusts internal accounting. +Metachunk* ChunkManager::get_chunk(int lvl) { + assert(is_valid_level(lvl), "sanity"); + if (_free_chunks[lvl] != NULL) { + Metachunk* const res = _free_chunks[lvl]; + remove_chunk(res); + res->set_in_use(); + return res; + } + return NULL; +} + +// Helper function for chunk coalescation. Given a chunk c, return its buddy. +// A chunk has no buddy if: +// - chunk is of max. size ("root chunk"), currently 4MB +// - chunk lives right at the border of its node +// In those cases, NULL is returned. +static Metachunk* get_buddy(const Metachunk* c) { + + if (c->is_root_chunk()) { + return NULL; } - // Chunk has been removed from the chunks free list, update counters. - account_for_removed_chunk(chunk); + MetaWord* buddy_address = NULL; + + // Every chunk is aligned to its size... + assert(is_aligned((address)c, c->size() * sizeof(MetaWord)), "sanity"); + // ... but chunks are allocated in pairs (buddies) unless they are root chunks, + // and the leader chunk is aligned to the next bigger size too. + bool chunk_is_leader = + is_aligned((address)c, c->size() * 2 * sizeof(MetaWord)); + if (chunk_is_leader) { + // buddy is located after this chunk + buddy_address = c->base() + c->word_size(); + } else { + // buddy is located before this chunk + buddy_address = c->base() - c->word_size(); + } + + // buddy may be located outside node boundaries. + if (buddy_address >= c->container()->bottom() && + buddy_address < c->container()->end()) + { + return NULL; + } + + // We have found the buddy. + Metachunk* buddy = (Metachunk*) buddy_address; + DEBUG_ONLY(buddy->verify()); + // They should both be located inside the same container. + assert(c->container() == buddy->container(), "sanity"); + + return buddy; + } +// Return a chunk to the ChunkManager. +void ChunkManager::return_chunk(Metachunk* c) { + + assert(c->is_free() == false, "Chunk should be in use."); + + // Here, we will attempt to coalesce free chunks. + Metachunk* buddy = get_buddy(c); + + while (buddy != NULL && buddy->is_free()) { + + // Merge both chunks. That means removing the follower chunk and increasing + // the level of the leader chunk -> follower chunk melts into leader chunk. + Metachunk* leader = c < buddy ? c : buddy; + Metachunk* follower = c > buddy ? c : buddy; + assert(leader->base() + leader->size() == follower->base(), "sanity"); + + remove_chunk(follower); + DEBUG_ONLY(follower->remove_sentinel()); + + int new_level = leader->level() + 1; + assert(is_valid_level(new_level), "sanity"); + leader->set_level(new_level); + + // and continue searching... + c = leader; + buddy = get_buddy(c); + + } + + // Now we should have arrived at a new chunk. Add it to the freelist. + add_chunk(c); + +#ifdef ASSERT + EVERY_NTH(VerifyMetaspaceInterval) + locked_verify(true); + END_EVERY_NTH +#endif + +} + + + + +/* + bool ChunkManager::attempt_to_coalesce_around_chunk(Metachunk* chunk, ChunkIndex target_chunk_type) { assert_lock_strong(MetaspaceExpand_lock); assert(chunk != NULL, "invalid chunk pointer"); @@ -195,6 +330,7 @@ } return num_removed; } +*/ // Update internal accounting after a chunk was added void ChunkManager::account_for_added_chunk(const Metachunk* c) { @@ -215,16 +351,6 @@ _free_chunks_total -= c->word_size(); } -ChunkIndex ChunkManager::list_index(size_t size) { - return get_chunk_type_by_size(size, is_class()); -} - -size_t ChunkManager::size_by_index(ChunkIndex index) const { - index_bounds_check(index); - assert(index != HumongousIndex, "Do not call for humongous chunks."); - return get_size_for_nonhumongous_chunktype(index, is_class()); -} - #ifdef ASSERT void ChunkManager::verify(bool slow) const { MutexLocker cl(MetaspaceExpand_lock, @@ -234,31 +360,34 @@ void ChunkManager::locked_verify(bool slow) const { log_trace(gc, metaspace, freelist)("verifying %s chunkmanager (%s).", - (is_class() ? "class space" : "metaspace"), (slow ? "slow" : "quick")); + _name, slow ? "slow" : "quick"); assert_lock_strong(MetaspaceExpand_lock); - size_t chunks_counted = 0; + int chunks_counted = 0; size_t wordsize_chunks_counted = 0; - for (ChunkIndex i = ZeroIndex; i < NumberOfFreeLists; i = next_chunk_index(i)) { - const ChunkList* list = _free_chunks + i; - if (list != NULL) { - Metachunk* chunk = list->head(); - while (chunk) { - if (slow) { - do_verify_chunk(chunk); - } - assert(chunk->is_tagged_free(), "Chunk should be tagged as free."); - chunks_counted ++; - wordsize_chunks_counted += chunk->size(); - chunk = chunk->next(); + for (int lvl = 0; lvl < NUM_CHUNK_LEVELS; lvl ++) { + int chunks_in_this_level_counted = 0; + Metachunk* c = _free_chunks[lvl]; + assert(c->prev() == NULL, "sanity"); + while (c != NULL) { + if (slow) { + c->verify(); } + assert(c->is_free(), "Chunk should be tagged as free."); + chunks_counted ++; + wordsize_chunks_counted += c->size(); + + if (c->next() != NULL) { + assert(c->next()->prev() == c, "sanity"); + } + c = c->next(); } + assert(_num_free_chunks[lvl] == chunks_in_this_level_counted, + "Chunk count mismatch for level %d, expected %d, got %d", + lvl, _num_free_chunks[lvl], chunks_in_this_level_counted); } - chunks_counted += humongous_dictionary()->total_free_blocks(); - wordsize_chunks_counted += humongous_dictionary()->total_size(); - assert(chunks_counted == _free_chunks_count && wordsize_chunks_counted == _free_chunks_total, "freelist accounting mismatch: " "we think: " SIZE_FORMAT " chunks, total " SIZE_FORMAT " words, " @@ -274,18 +403,7 @@ _free_chunks_total, _free_chunks_count); } -ChunkList* ChunkManager::free_chunks(ChunkIndex index) { - assert(index == SpecializedIndex || index == SmallIndex || index == MediumIndex, - "Bad index: %d", (int)index); - return &_free_chunks[index]; -} - -ChunkList* ChunkManager::find_free_chunks_list(size_t word_size) { - ChunkIndex index = list_index(word_size); - assert(index < HumongousIndex, "No humongous list"); - return free_chunks(index); -} - +/* // Helper for chunk splitting: given a target chunk size and a larger free chunk, // split up the larger chunk into n smaller chunks, at least one of which should be // the target chunk of target chunk size. The smaller chunks, including the target @@ -388,6 +506,8 @@ return target_chunk; } +*/ + Metachunk* ChunkManager::free_chunks_get(size_t word_size) { assert_lock_strong(MetaspaceExpand_lock); @@ -498,7 +618,7 @@ return chunk; } -Metachunk* ChunkManager::chunk_freelist_allocate(size_t word_size) { +Metachunk* ChunkManager::find_chunk_for_level(size_t word_size) { assert_lock_strong(MetaspaceExpand_lock); // Take from the beginning of the list diff -r 0927d8c7296f -r 902274e61ebb src/hotspot/share/memory/metaspace/chunkManager.hpp --- a/src/hotspot/share/memory/metaspace/chunkManager.hpp Wed May 22 07:10:54 2019 +0200 +++ b/src/hotspot/share/memory/metaspace/chunkManager.hpp Wed May 22 18:05:57 2019 +0200 @@ -26,8 +26,6 @@ #define SHARE_MEMORY_METASPACE_CHUNKMANAGER_HPP #include "memory/allocation.hpp" -#include "memory/binaryTreeDictionary.hpp" -#include "memory/freeList.hpp" #include "memory/metaspace/metachunk.hpp" #include "memory/metaspace/metaspaceStatistics.hpp" #include "memory/metaspaceChunkFreeListSummary.hpp" @@ -37,9 +35,6 @@ namespace metaspace { -typedef class FreeList ChunkList; -typedef BinaryTreeDictionary > ChunkTreeDictionary; - // Manages the global free lists of chunks. class ChunkManager : public CHeapObj { friend class ::ChunkManagerTestAccessor; @@ -48,33 +43,24 @@ // SpecializedChunk // SmallChunk // MediumChunk - ChunkList _free_chunks[NumberOfFreeLists]; - - // Whether or not this is the class chunkmanager. - const bool _is_class; - - // Return non-humongous chunk list by its index. - ChunkList* free_chunks(ChunkIndex index); - - // Returns non-humongous chunk list for the given chunk word size. - ChunkList* find_free_chunks_list(size_t word_size); - - // HumongousChunk - ChunkTreeDictionary _humongous_dictionary; - - // Returns the humongous chunk dictionary. - ChunkTreeDictionary* humongous_dictionary() { return &_humongous_dictionary; } - const ChunkTreeDictionary* humongous_dictionary() const { return &_humongous_dictionary; } + Metachunk* _free_chunks [NUM_CHUNK_LEVELS]; + int _num_free_chunks [NUM_CHUNK_LEVELS]; // Size, in metaspace words, of all chunks managed by this ChunkManager size_t _free_chunks_total; + // Number of chunks in this ChunkManager - size_t _free_chunks_count; + // Todo: do we need this? Seems like a totally pointless information. + int _free_chunks_count; + + // Convenience name is convenient. + const char* const _name; // Update counters after a chunk was added or removed removed. void account_for_added_chunk(const Metachunk* c); void account_for_removed_chunk(const Metachunk* c); +/* // Given a pointer to a chunk, attempts to merge it with neighboring // free chunks to form a bigger chunk. Returns true if successful. bool attempt_to_coalesce_around_chunk(Metachunk* chunk, ChunkIndex target_chunk_type); @@ -95,39 +81,42 @@ // chunk, are returned to the freelist. The pointer to the target chunk is returned. // Note that this chunk is supposed to be removed from the freelist right away. Metachunk* split_chunk(size_t target_chunk_word_size, Metachunk* chunk); +*/ + + // Returns true if this manager contains the given chunk. Slow (walks free list) and + // only needed for verifications. + DEBUG_ONLY(bool contains_chunk(Metachunk* metachunk) const;) + + // Add a chunk to the free list (no coalescation, just an add) + // and adjust accounting. + void add_chunk(Metachunk* chunk); public: - ChunkManager(bool is_class); + ChunkManager(const char* name); - // Add or delete (return) a chunk to the global freelist. - Metachunk* chunk_freelist_allocate(size_t word_size); + // Removing chunks from freelist: - // Map a size to a list index assuming that there are lists - // for special, small, medium, and humongous chunks. - ChunkIndex list_index(size_t size); + // Remove a chunk of the given level from its free list and return it; + // If there is no chunk of that level available, returns NULL. + // The returned chunk will be marked as "in use". + // Adjusts internal accounting. + Metachunk* get_chunk(int level); - // Map a given index to the chunk size. - size_t size_by_index(ChunkIndex index) const; - - bool is_class() const { return _is_class; } - - // Convenience accessors. - size_t medium_chunk_word_size() const { return size_by_index(MediumIndex); } - size_t small_chunk_word_size() const { return size_by_index(SmallIndex); } - size_t specialized_chunk_word_size() const { return size_by_index(SpecializedIndex); } - - // Take a chunk from the ChunkManager. The chunk is expected to be in - // the chunk manager (the freelist if non-humongous, the dictionary if - // humongous). + // Remove the given chunk from its free list and adjust accounting. + // (Called during node purging, so the chunk will shortly cease existing.) void remove_chunk(Metachunk* chunk); - // Return a single chunk of type index to the ChunkManager. - void return_single_chunk(Metachunk* chunk); - // Add the simple linked list of chunks to the freelist of chunks - // of type index. - void return_chunk_list(Metachunk* chunk); + Metachunk* get_chunk_smart(int level); + + // Adding chunks to freelist: + + // Return a chunk to the ChunkManager. + // Will coalesce with buddies if needed, so do not access the chunk after it has been + // returned! It may have become invalid. + // Will adjust accounting. + void return_chunk(Metachunk* chunk); // Total of the space in the free chunks list size_t free_chunks_total_words() const { return _free_chunks_total; } @@ -136,47 +125,12 @@ // Number of chunks in the free chunks list size_t free_chunks_count() const { return _free_chunks_count; } - // Remove from a list by size. Selects list based on size of chunk. - Metachunk* free_chunks_get(size_t chunk_word_size); - -#define index_bounds_check(index) \ - assert(is_valid_chunktype(index), "Bad index: %d", (int) index) - - size_t num_free_chunks(ChunkIndex index) const { - index_bounds_check(index); - - if (index == HumongousIndex) { - return _humongous_dictionary.total_free_blocks(); - } - - ssize_t count = _free_chunks[index].count(); - return count == -1 ? 0 : (size_t) count; + int num_free_chunks(int level) const { + assert(is_valid_level(level), "sanity"); + return _num_free_chunks[level]; } - size_t size_free_chunks_in_bytes(ChunkIndex index) const { - index_bounds_check(index); - - size_t word_size = 0; - if (index == HumongousIndex) { - word_size = _humongous_dictionary.total_size(); - } else { - const size_t size_per_chunk_in_words = _free_chunks[index].size(); - word_size = size_per_chunk_in_words * num_free_chunks(index); - } - - return word_size * BytesPerWord; - } - - MetaspaceChunkFreeListSummary chunk_free_list_summary() const { - return MetaspaceChunkFreeListSummary(num_free_chunks(SpecializedIndex), - num_free_chunks(SmallIndex), - num_free_chunks(MediumIndex), - num_free_chunks(HumongousIndex), - size_free_chunks_in_bytes(SpecializedIndex), - size_free_chunks_in_bytes(SmallIndex), - size_free_chunks_in_bytes(MediumIndex), - size_free_chunks_in_bytes(HumongousIndex)); - } + MetaspaceChunkFreeListSummary chunk_free_list_summary() const; #ifdef ASSERT // Debug support @@ -190,9 +144,10 @@ // Fill in current statistic values to the given statistics object. void collect_statistics(ChunkManagerStatistics* out) const; + const char* name() const { return _name; } + }; } // namespace metaspace - #endif // SHARE_MEMORY_METASPACE_CHUNKMANAGER_HPP diff -r 0927d8c7296f -r 902274e61ebb src/hotspot/share/memory/metaspace/metachunk.cpp --- a/src/hotspot/share/memory/metaspace/metachunk.cpp Wed May 22 07:10:54 2019 +0200 +++ b/src/hotspot/share/memory/metaspace/metachunk.cpp Wed May 22 18:05:57 2019 +0200 @@ -50,50 +50,110 @@ // Metachunk methods -Metachunk::Metachunk(ChunkIndex chunktype, bool is_class, size_t word_size, - VirtualSpaceNode* container) - : Metabase(word_size), - _container(container), - _top(NULL), - _sentinel(CHUNK_SENTINEL), - _chunk_type(chunktype), - _is_class(is_class), - _origin(origin_normal), - _use_count(0) +void Metachunk::remove_from_list() { + if (_prev != NULL) { + _prev->set_next(_next); + } + if (_next != NULL) { + _next->set_prev(_prev); + } + _prev = _next = NULL; +} + + +Metachunk::Metachunk(int level, VirtualSpaceNode* container, bool fully_committed) + : _prev(NULL), _next(NULL) + , _level(level) + , _is_free(true) + , _words_allocated(0) + , _words_committed(0) + , _container(container) + , _sentinel(CHUNK_SENTINEL) { - _top = initial_top(); - set_is_tagged_free(false); + _words_allocated = overhead(); + if (fully_committed) { + _words_committed = size(); + } else { + // We do not know the commit status of the underlying memory, only that the first + // page - containing the header - must be committed. So assume that the rest is not. + // (only affects chunks spanning multiple pages). + const size_t page_size = os::vm_page_size(); + if (size() > page_size) { + _words_committed = page_size; + } else { + _words_committed = size(); + } + } #ifdef ASSERT mangle(uninitMetaWordVal); verify(); #endif } -MetaWord* Metachunk::allocate(size_t word_size) { +// Whether this chunk can uncommit parts of it. +bool Metachunk::can_uncommit() const { + return size() > os::vm_page_size(); +} + +// Make sure we can place wordsize bytes without hitting the commit +// boundary (assuming wordsize is small enough to fit into the chunk). +void Metachunk::ensure_committed(size_t requested_word_size) { + assert((_words_allocated + requested_word_size) < word_size(), "would not fit."); + assert(can_uncommit(), "this chunk is too small to contain uncommitted parts."); + + // For now, commit page wise and lets see how this goes. This may very well be too + // fine granular. + const size_t words_committed_needed = + align_up(_words_allocated + requested_word_size, os::vm_page_size()); + if (words_committed_needed > _words_committed) { + const size_t to_commit = words_committed_needed - _words_committed; + assert(is_aligned(to_commit, (size_t)os::vm_page_size()), "sanity"); + MetaWord* const commit_top = commit_top(); + if (os::commit_memory((char*)commit_top, to_commit, false)) { + _words_committed = words_committed_needed; + } else { + vm_exit_out_of_memory(to_commit, OOM_MMAP_ERROR, "Failed to mmap " SIZE_FORMAT "bytes for metachunk.", to_commit); + } + } +} + +MetaWord* Metachunk::allocate(size_t requested_word_size) { + assert(is_aligned(requested_word_size, sizeof(MetaWord)), "unaligned alloc"); + MetaWord* result = NULL; + + // Would this fit? + const size_t remaining_words = word_size() - _words_allocated; + if (remaining_words < requested_word_size) { + return NULL; + } + + // Maybe we need to commit? + if (can_uncommit()) { + ensure_committed(requested_word_size); + } + // If available, bump the pointer to allocate. - if (free_word_size() >= word_size) { - result = _top; - _top = _top + word_size; - } + result = top(); + _words_allocated += requested_word_size; + return result; } // _bottom points to the start of the chunk including the overhead. size_t Metachunk::used_word_size() const { - return pointer_delta(_top, bottom(), sizeof(MetaWord)); + return _words_allocated - overhead(); } size_t Metachunk::free_word_size() const { - return pointer_delta(end(), _top, sizeof(MetaWord)); + return size() - _words_allocated; } void Metachunk::print_on(outputStream* st) const { st->print_cr("Metachunk:" " bottom " PTR_FORMAT " top " PTR_FORMAT - " end " PTR_FORMAT " size " SIZE_FORMAT " (%s)", - p2i(bottom()), p2i(_top), p2i(end()), word_size(), - chunk_size_name(get_chunk_type())); + " end " PTR_FORMAT " size " SIZE_FORMAT, + p2i(base()), p2i(top()), p2i(end()), word_size()); if (Verbose) { st->print_cr(" used " SIZE_FORMAT " free " SIZE_FORMAT, used_word_size(), free_word_size()); @@ -104,69 +164,31 @@ void Metachunk::mangle(juint word_value) { // Overwrite the payload of the chunk and not the links that // maintain list of chunks. - HeapWord* start = (HeapWord*)initial_top(); - size_t size = word_size() - overhead(); - Copy::fill_to_words(start, size, word_value); + MetaWord* start = base() + overhead(); + assert(_words_committed >= overhead, "sanity"); + size_t mangle_size = _words_committed - overhead(); + Copy::fill_to_words((HeapWord*)start, size, word_value); } void Metachunk::verify() const { assert(is_valid_sentinel(), "Chunk " PTR_FORMAT ": sentinel invalid", p2i(this)); - const ChunkIndex chunk_type = get_chunk_type(); - assert(is_valid_chunktype(chunk_type), "Chunk " PTR_FORMAT ": Invalid chunk type.", p2i(this)); - if (chunk_type != HumongousIndex) { - assert(word_size() == get_size_for_nonhumongous_chunktype(chunk_type, is_class()), - "Chunk " PTR_FORMAT ": wordsize " SIZE_FORMAT " does not fit chunk type %s.", - p2i(this), word_size(), chunk_size_name(chunk_type)); - } - assert(is_valid_chunkorigin(get_origin()), "Chunk " PTR_FORMAT ": Invalid chunk origin.", p2i(this)); - assert(bottom() <= _top && _top <= (MetaWord*)end(), - "Chunk " PTR_FORMAT ": Chunk top out of chunk bounds.", p2i(this)); + assert(is_valid_level(_level), "Invalid level (%d)", _level); - // For non-humongous chunks, starting address shall be aligned - // to its chunk size. Humongous chunks start address is - // aligned to specialized chunk size. - const size_t required_alignment = - (chunk_type != HumongousIndex ? word_size() : get_size_for_nonhumongous_chunktype(SpecializedIndex, is_class())) * sizeof(MetaWord); + assert(base() == (MetaWord*) this, "sanity"); + assert(end() == base() + size(), "sanity"); + assert(top() >= base() + overhead() && top() <= end(), "sanity"); + assert(commit_top() >= top() && commit_top() <= end(), "sanity"); + + assert(_container != NULL, "sanity"); + + // Starting address shall be aligned to chunk size. + const size_t required_alignment = word_size() * sizeof(MetaWord); assert(is_aligned((address)this, required_alignment), - "Chunk " PTR_FORMAT ": (size " SIZE_FORMAT ") not aligned to " SIZE_FORMAT ".", + "Chunk " PTR_FORMAT ": (size " SIZE_FORMAT ") not aligned correctly to " SIZE_FORMAT ".", p2i(this), word_size() * sizeof(MetaWord), required_alignment); } #endif // ASSERT -// Helper, returns a descriptive name for the given index. -const char* chunk_size_name(ChunkIndex index) { - switch (index) { - case SpecializedIndex: - return "specialized"; - case SmallIndex: - return "small"; - case MediumIndex: - return "medium"; - case HumongousIndex: - return "humongous"; - default: - return "Invalid index"; - } -} - -#ifdef ASSERT -void do_verify_chunk(Metachunk* chunk) { - guarantee(chunk != NULL, "Sanity"); - // Verify chunk itself; then verify that it is consistent with the - // occupany map of its containing node. - chunk->verify(); - VirtualSpaceNode* const vsn = chunk->container(); - OccupancyMap* const ocmap = vsn->occupancy_map(); - ocmap->verify_for_chunk(chunk); -} -#endif - -void do_update_in_use_info_for_chunk(Metachunk* chunk, bool inuse) { - chunk->set_is_tagged_free(!inuse); - OccupancyMap* const ocmap = chunk->container()->occupancy_map(); - ocmap->set_region_in_use((MetaWord*)chunk, chunk->word_size(), inuse); -} - } // namespace metaspace diff -r 0927d8c7296f -r 902274e61ebb src/hotspot/share/memory/metaspace/metachunk.hpp --- a/src/hotspot/share/memory/metaspace/metachunk.hpp Wed May 22 07:10:54 2019 +0200 +++ b/src/hotspot/share/memory/metaspace/metachunk.hpp Wed May 22 18:05:57 2019 +0200 @@ -39,73 +39,64 @@ // Metachunks are reused (when freed are put on a global freelist) and // have no permanent association to a SpaceManager. -// +--------------+ <- end --+ --+ -// | | | | -// | | | free | -// | | | | -// | | | | size | capacity -// | | | | -// | | <- top -- + | -// | | | | -// | | | used | -// | | | | -// | | | | -// +--------------+ <- bottom --+ --+ +// +--------------+ <- end ----+ --+ +// | | | | +// | | | free | +// | ----------- | <- commit_top | | +// | | | | size | capacity +// | | | | +// | ----------- | <- top -- + | +// | | | | +// | | | used | +// +--------------+ -- + | +// | header | | overhead | +// +--------------+ <- base ----+ --+ -enum ChunkOrigin { - // Chunk normally born (via take_from_committed) - origin_normal = 1, - // Chunk was born as padding chunk - origin_pad = 2, - // Chunk was born as leftover chunk in VirtualSpaceNode::retire - origin_leftover = 3, - // Chunk was born as result of a merge of smaller chunks - origin_merge = 4, - // Chunk was born as result of a split of a larger chunk - origin_split = 5, - origin_minimum = origin_normal, - origin_maximum = origin_split, - origins_count = origin_maximum + 1 -}; - -inline bool is_valid_chunkorigin(ChunkOrigin origin) { - return origin == origin_normal || - origin == origin_pad || - origin == origin_leftover || - origin == origin_merge || - origin == origin_split; -} - -class Metachunk : public Metabase { +class Metachunk { friend class ::MetachunkTest; + // Note: lots of possibilities to shrink that header! but for now I concentrate on + // easy readability and dev speed. + + Metachunk* _prev; + Metachunk* _next; + + int _level; + + bool _is_free; + + // Current allocated words, including header. + size_t _words_allocated; + + // Current committed words, including header. + size_t _words_committed; + // The VirtualSpaceNode containing this chunk. VirtualSpaceNode* const _container; - // Current allocation top. - MetaWord* _top; +#ifdef ASSERT + // A 32bit sentinel for debugging purposes. + static const uint32_t CHUNK_SENTINEL = 0x4d4554EF; // "MET" + static const uint32_t CHUNK_SENTINEL_INVALID = 0xFEEEEEEF; + uint32_t _sentinel; +#endif - // A 32bit sentinel for debugging purposes. - enum { CHUNK_SENTINEL = 0x4d4554EF, // "MET" - CHUNK_SENTINEL_INVALID = 0xFEEEEEEF - }; + MetaWord* base() const { return (MetaWord*)this; } + MetaWord* end() const { return (MetaWord*)this + word_size(); } + MetaWord* top() const { return base() + _words_allocated; } + MetaWord* commit_top() const { return base() + _words_committed; } - uint32_t _sentinel; + // Make sure we can place wordsize bytes without hitting the commit + // boundary (assuming wordsize is small enough to fit into the chunk). + void ensure_committed(size_t wordsize); - const ChunkIndex _chunk_type; - const bool _is_class; - // Whether the chunk is free (in freelist) or in use by some class loader. - bool _is_tagged_free; - - ChunkOrigin _origin; - int _use_count; - - MetaWord* initial_top() const { return (MetaWord*)this + overhead(); } - MetaWord* top() const { return _top; } + // Whether this chunk can uncommit parts of it. + bool can_uncommit() const; public: + // Metachunks are allocated out of a MetadataVirtualSpace and // and use some of its space to describe itself (plus alignment // considerations). Metadata is allocated in the rest of the chunk. @@ -118,56 +109,62 @@ // Size of the Metachunk header, in words, including alignment. static size_t overhead(); - Metachunk(ChunkIndex chunktype, bool is_class, size_t word_size, VirtualSpaceNode* container); + int level() const { return _level; } + + // Only needed during coalescation + void set_level(int lvl) { _level = lvl; } + + // A "root chunk" is a chunk of the highest level. It cannot coalesce further + // and has no buddy. + bool is_root_chunk() const { return _level == NUM_CHUNK_LEVELS - 1; } + + // Returns size, in bytes, of this chunk including the header. + size_t size() const { + return MIN_CHUNK_BYTE_SIZE << _level; + } + + Metachunk(int level, VirtualSpaceNode* container, bool fully_committed = true); MetaWord* allocate(size_t word_size); VirtualSpaceNode* container() const { return _container; } - MetaWord* bottom() const { return (MetaWord*) this; } - // Reset top to bottom so chunk can be reused. - void reset_empty() { _top = initial_top(); clear_next(); clear_prev(); } - bool is_empty() { return _top == initial_top(); } + void reset() { _words_allocated = 0; } + bool is_empty() { return _words_allocated == 0; } // used (has been allocated) // free (available for future allocations) - size_t word_size() const { return size(); } + size_t word_size() const { return size() / sizeof(MetaWord); } size_t used_word_size() const; size_t free_word_size() const; - bool is_tagged_free() { return _is_tagged_free; } - void set_is_tagged_free(bool v) { _is_tagged_free = v; } + bool is_free() const { return _is_free; } + void set_free() { _is_free = true; } + void set_in_use() { _is_free = false; } - bool contains(const void* ptr) { return bottom() <= ptr && ptr < _top; } + void set_prev(Metachunk* c) { _prev = c; } + void set_next(Metachunk* c) { _next = c; } + Metachunk* prev() const { return _prev; } + Metachunk* next() const { return _next; } + void remove_from_list(); + + bool contains(const void* ptr) const { + return base() <= ptr && ptr < top(); + } void print_on(outputStream* st) const; +#ifdef ASSERT bool is_valid_sentinel() const { return _sentinel == CHUNK_SENTINEL; } void remove_sentinel() { _sentinel = CHUNK_SENTINEL_INVALID; } - - int get_use_count() const { return _use_count; } - void inc_use_count() { _use_count ++; } - - ChunkOrigin get_origin() const { return _origin; } - void set_origin(ChunkOrigin orig) { _origin = orig; } - - ChunkIndex get_chunk_type() const { return _chunk_type; } - bool is_class() const { return _is_class; } +#endif DEBUG_ONLY(void mangle(juint word_value);) DEBUG_ONLY(void verify() const;) }; - -// Helper function that does a bunch of checks for a chunk. -DEBUG_ONLY(void do_verify_chunk(Metachunk* chunk);) - -// Given a Metachunk, update its in-use information (both in the -// chunk and the occupancy map). -void do_update_in_use_info_for_chunk(Metachunk* chunk, bool inuse); - } // namespace metaspace #endif // SHARE_MEMORY_METASPACE_METACHUNK_HPP diff -r 0927d8c7296f -r 902274e61ebb src/hotspot/share/memory/metaspace/metaspaceCommon.cpp --- a/src/hotspot/share/memory/metaspace/metaspaceCommon.cpp Wed May 22 07:10:54 2019 +0200 +++ b/src/hotspot/share/memory/metaspace/metaspaceCommon.cpp Wed May 22 18:05:57 2019 +0200 @@ -130,70 +130,6 @@ } } -// Returns size of this chunk type. -size_t get_size_for_nonhumongous_chunktype(ChunkIndex chunktype, bool is_class) { - assert(is_valid_nonhumongous_chunktype(chunktype), "invalid chunk type."); - size_t size = 0; - if (is_class) { - switch(chunktype) { - case SpecializedIndex: size = ClassSpecializedChunk; break; - case SmallIndex: size = ClassSmallChunk; break; - case MediumIndex: size = ClassMediumChunk; break; - default: - ShouldNotReachHere(); - } - } else { - switch(chunktype) { - case SpecializedIndex: size = SpecializedChunk; break; - case SmallIndex: size = SmallChunk; break; - case MediumIndex: size = MediumChunk; break; - default: - ShouldNotReachHere(); - } - } - return size; -} - -ChunkIndex get_chunk_type_by_size(size_t size, bool is_class) { - if (is_class) { - if (size == ClassSpecializedChunk) { - return SpecializedIndex; - } else if (size == ClassSmallChunk) { - return SmallIndex; - } else if (size == ClassMediumChunk) { - return MediumIndex; - } else if (size > ClassMediumChunk) { - // A valid humongous chunk size is a multiple of the smallest chunk size. - assert(is_aligned(size, ClassSpecializedChunk), "Invalid chunk size"); - return HumongousIndex; - } - } else { - if (size == SpecializedChunk) { - return SpecializedIndex; - } else if (size == SmallChunk) { - return SmallIndex; - } else if (size == MediumChunk) { - return MediumIndex; - } else if (size > MediumChunk) { - // A valid humongous chunk size is a multiple of the smallest chunk size. - assert(is_aligned(size, SpecializedChunk), "Invalid chunk size"); - return HumongousIndex; - } - } - ShouldNotReachHere(); - return (ChunkIndex)-1; -} - -ChunkIndex next_chunk_index(ChunkIndex i) { - assert(i < NumberOfInUseLists, "Out of bound"); - return (ChunkIndex) (i+1); -} - -ChunkIndex prev_chunk_index(ChunkIndex i) { - assert(i > ZeroIndex, "Out of bound"); - return (ChunkIndex) (i-1); -} - const char* loaders_plural(uintx num) { return num == 1 ? "loader" : "loaders"; } diff -r 0927d8c7296f -r 902274e61ebb src/hotspot/share/memory/metaspace/metaspaceCommon.hpp --- a/src/hotspot/share/memory/metaspace/metaspaceCommon.hpp Wed May 22 07:10:54 2019 +0200 +++ b/src/hotspot/share/memory/metaspace/metaspaceCommon.hpp Wed May 22 18:05:57 2019 +0200 @@ -33,14 +33,26 @@ namespace metaspace { -enum ChunkSizes { // in words. - ClassSpecializedChunk = 128, - SpecializedChunk = 128, - ClassSmallChunk = 256, - SmallChunk = 512, - ClassMediumChunk = 4 * K, - MediumChunk = 8 * K -}; +// Large enough to hold 99% of InstanceKlass. +static const size_t MIN_CHUNK_BYTE_SIZE = 1 * K; + +// Let MAX_CHUNK_SIZE be large enough to hold the largest possible InstanceKlass. +static const size_t NUM_CHUNK_LEVELS = 12; +static const size_t MAX_CHUNK_BYTE_SIZE = MIN_CHUNK_BYTE_SIZE << NUM_CHUNK_LEVELS; + +static const size_t MIN_CHUNK_WORD_SIZE = MIN_CHUNK_BYTE_SIZE / sizeof(MetaWord); +static const size_t MAX_CHUNK_WORD_SIZE = MAX_CHUNK_BYTE_SIZE / sizeof(MetaWord); + +inline bool is_valid_level(int level) { + return level >= 0 && level < NUM_CHUNK_LEVELS; +} + +DEBUG_ONLY(inline void check_valid_level(int lvl) { assert(is_valid_level(lvl), "invalid level (%d)", lvl); }) + +inline size_t size_for_level(int level) { + assert(is_valid_level(level), "invalid chunk level (%d)", level); + return MIN_CHUNK_BYTE_SIZE << level; +} // Print a size, in words, scaled. void print_scaled_words(outputStream* st, size_t word_size, size_t scale = 0, int width = -1); @@ -98,48 +110,6 @@ extern internal_statistics_t g_internal_statistics; #endif -// ChunkIndex defines the type of chunk. -// Chunk types differ by size: specialized < small < medium, chunks -// larger than medium are humongous chunks of varying size. -enum ChunkIndex { - ZeroIndex = 0, - SpecializedIndex = ZeroIndex, - SmallIndex = SpecializedIndex + 1, - MediumIndex = SmallIndex + 1, - HumongousIndex = MediumIndex + 1, - NumberOfFreeLists = 3, - NumberOfInUseLists = 4 -}; - -// Utility functions. -size_t get_size_for_nonhumongous_chunktype(ChunkIndex chunk_type, bool is_class); -ChunkIndex get_chunk_type_by_size(size_t size, bool is_class); - -ChunkIndex next_chunk_index(ChunkIndex i); -ChunkIndex prev_chunk_index(ChunkIndex i); -// Returns a descriptive name for a chunk type. -const char* chunk_size_name(ChunkIndex index); - -// Verify chunk sizes. -inline bool is_valid_chunksize(bool is_class, size_t size) { - const size_t reasonable_maximum_humongous_chunk_size = 1 * G; - return is_aligned(size, sizeof(MetaWord)) && - size < reasonable_maximum_humongous_chunk_size && - is_class ? - (size == ClassSpecializedChunk || size == ClassSmallChunk || size >= ClassMediumChunk) : - (size == SpecializedChunk || size == SmallChunk || size >= MediumChunk); -} - -// Verify chunk type. -inline bool is_valid_chunktype(ChunkIndex index) { - return index == SpecializedIndex || index == SmallIndex || - index == MediumIndex || index == HumongousIndex; -} - -inline bool is_valid_nonhumongous_chunktype(ChunkIndex index) { - return is_valid_chunktype(index) && index != HumongousIndex; -} - // Pretty printing helpers const char* classes_plural(uintx num); const char* loaders_plural(uintx num); diff -r 0927d8c7296f -r 902274e61ebb src/hotspot/share/memory/metaspace/spaceManager.cpp --- a/src/hotspot/share/memory/metaspace/spaceManager.cpp Wed May 22 07:10:54 2019 +0200 +++ b/src/hotspot/share/memory/metaspace/spaceManager.cpp Wed May 22 18:05:57 2019 +0200 @@ -384,7 +384,7 @@ Metachunk* SpaceManager::get_new_chunk(size_t chunk_word_size) { // Get a chunk from the chunk freelist - Metachunk* next = chunk_manager()->chunk_freelist_allocate(chunk_word_size); + Metachunk* next = chunk_manager()->get_chunk(chunk_word_size); if (next == NULL) { next = vs_list()->get_new_chunk(chunk_word_size,