diff -r 45843600f2de -r 412ce076c0dc src/hotspot/share/memory/metaspace.cpp --- a/src/hotspot/share/memory/metaspace.cpp Thu Jun 06 08:58:26 2019 +0200 +++ b/src/hotspot/share/memory/metaspace.cpp Wed Jun 05 08:59:17 2019 +0200 @@ -1450,7 +1450,7 @@ size_t chunk_word_size = get_space_manager(mdtype)->get_initial_chunk_size(type); // Get a chunk from the chunk freelist - Metachunk* chunk = Metaspace::get_chunk_manager(mdtype)->get_chunk(chunk_word_size); + Metachunk* chunk = Metaspace::get_chunk_manager(mdtype)->chunk_freelist_allocate(chunk_word_size); if (chunk == NULL) { chunk = Metaspace::get_space_list(mdtype)->get_new_chunk(chunk_word_size, diff -r 45843600f2de -r 412ce076c0dc src/hotspot/share/memory/metaspace/chunkManager.cpp --- a/src/hotspot/share/memory/metaspace/chunkManager.cpp Thu Jun 06 08:58:26 2019 +0200 +++ b/src/hotspot/share/memory/metaspace/chunkManager.cpp Wed Jun 05 08:59:17 2019 +0200 @@ -41,276 +41,26 @@ namespace metaspace { -ChunkManager::ChunkManager(const char* name) - : _name(name), - _free_chunks_total(0), - _free_chunks_count(0) -{ - for (int lvl = 0; lvl < NUM_CHUNK_LEVELS; lvl ++) { - _free_chunks[lvl] = NULL; - _num_free_chunks[lvl] = 0; - } +ChunkManager::ChunkManager(bool is_class) + : _is_class(is_class), _free_chunks_total(0), _free_chunks_count(0) { + _free_chunks[SpecializedIndex].set_size(get_size_for_nonhumongous_chunktype(SpecializedIndex, is_class)); + _free_chunks[SmallIndex].set_size(get_size_for_nonhumongous_chunktype(SmallIndex, is_class)); + _free_chunks[MediumIndex].set_size(get_size_for_nonhumongous_chunktype(MediumIndex, is_class)); } -#ifdef ASSERT -// Returns true if this manager contains the given chunk. Slow (walks free list) and -// only needed for verifications. -bool ChunkManager::contains_chunk(Metachunk* c) const { - assert(c != NULL, "sanity"); - const int lvl = c->level(); - assert(is_valid_level(lvl), "sanity"); - Metachunk* c2 = _free_chunks[lvl]; - while (c2 != NULL && c2 != c) { - c2 = c2->next(); - } - return c2 == c; -} -#endif - -void ChunkManager::remove_chunk(Metachunk* c) { -#ifdef ASSERT - EVERY_NTH(VerifyMetaspaceInterval) - assert(contains_chunk(c), "chunk is not managed by this chunk manager"); - c->verify(); - END_EVERY_NTH -#endif - - c->remove_from_list(); - account_for_removed_chunk(c); -} - -// Add a chunk to the free list (no coalescation, just an add) -// and adjust accounting. -void ChunkManager::add_chunk(Metachunk* c) { - int lvl = c->level(); - DEBUG_ONLY(check_valid_level(lvl)); - if (_free_chunks[lvl] != NULL) { - _free_chunks[lvl]->set_prev(c); - } - c->set_next(_free_chunks[lvl]); - _free_chunks[lvl] = c; - c->set_prev(NULL); - account_for_added_chunk(c); -} - -// Remove a chunk of the given level from its free list and return it; returns NULL -// if that free list is empty. The returned chunk will be marked as "in use". -// Adjusts internal accounting. -Metachunk* ChunkManager::get_chunk(int lvl) { - assert(is_valid_level(lvl), "sanity"); - if (_free_chunks[lvl] != NULL) { - Metachunk* const res = _free_chunks[lvl]; - remove_chunk(res); - res->set_in_use(); - account_for_removed_chunk(res); - return res; - } - return NULL; -} - -// Helper function for chunk coalescation. Given a chunk c, return its buddy. -// Function returns NULL if: -// - the buddy chunk is splintered into smaller chunks -// - chunk is of max. size ("root chunk"), currently 4MB -// - chunk lives at the node boundary and the buddy would be outside the node limit. -static Metachunk* get_buddy(const Metachunk* c) { - - if (c->is_root_chunk()) { - return NULL; +void ChunkManager::remove_chunk(Metachunk* chunk) { + size_t word_size = chunk->word_size(); + ChunkIndex index = list_index(word_size); + if (index != HumongousIndex) { + free_chunks(index)->remove_chunk(chunk); + } else { + humongous_dictionary()->remove_chunk(chunk); } - MetaWord* buddy_address = NULL; - - // Every chunk is aligned to its size... - assert(is_aligned((address)c, c->size() * sizeof(MetaWord)), "sanity"); - // ... but chunks are allocated in pairs (buddies) unless they are root chunks, - // and the leader chunk is aligned to the next bigger size too. - bool chunk_is_leader = - is_aligned((address)c, c->size() * 2 * sizeof(MetaWord)); - if (chunk_is_leader) { - // buddy is located after this chunk - buddy_address = c->base() + c->word_size(); - } else { - // buddy is located before this chunk - buddy_address = c->base() - c->word_size(); - } - - // buddy may be located outside node boundaries. - if (buddy_address >= c->container()->bottom() && - buddy_address < c->container()->end()) - { - return NULL; - } - - // We have found the chunk at the buddy location. - Metachunk* buddy = (Metachunk*) buddy_address; - assert(buddy->level() <= c->level(), "buddy cannot have a larger level."); - if (buddy->level() < c->level()) { - // Buddy is splintered. - return NULL; - } - - DEBUG_ONLY(buddy->verify()); - - // They should both be located inside the same container. - assert(c->container() == buddy->container(), "sanity"); - - return buddy; - + // Chunk has been removed from the chunks free list, update counters. + account_for_removed_chunk(chunk); } -// Return a chunk to the ChunkManager. -void ChunkManager::return_chunk(Metachunk* c) { - - assert_lock_strong(MetaspaceExpand_lock); - - assert(c->is_free() == false, "Chunk should be in use."); - - // Here, we will attempt to coalesce free chunks. - // When a chunk is returned to the chunk maanager, it is attempted - // to merge it with its buddy. This can be done if - // a) there is a buddy (root chunks have no buddy; chunks at the node border may have no buddy). - // b) that chunk is not splintered into smaller chunks - // c) that chunk is free'd. - // - // The chunk is merged, which just means the leader chunk of the buddy pair is made a new buddy - // of increased level, the follower chunk disappears. - // - // Then, that process is repeated until no buddy is found or until we have reached the max. - // chunk size. - // - Metachunk* buddy = get_buddy(c); - - while (buddy != NULL && buddy->is_free()) { - - // We have a buddy. Who is the leader of this pair? - Metachunk* leader = NULL; - Metachunk* follower = NULL; - if (c < buddy) { - leader = c; - follower = buddy; - } else { - leader = buddy; - follower = c; - } - assert(leader->base() + leader->word_size() == follower->base(), "sanity"); - - // Remove follower chunk from free list - remove_chunk(follower); - - // and mark its header as invalid - DEBUG_ONLY(follower->remove_sentinel()); - - // Increase level of the new merged chunk - int new_level = leader->level() + 1; - assert(is_valid_level(new_level), "sanity"); - leader->set_level(new_level); - - // and continue searching... - c = leader; - buddy = get_buddy(c); - - } - - // Now we should have arrived at a new chunk. Add it to the freelist. - DEBUG_ONLY(c->verify()); - add_chunk(c); - -#ifdef ASSERT - EVERY_NTH(VerifyMetaspaceInterval) - locked_verify(true); - END_EVERY_NTH -#endif - -} // end: ChunkManager::return_chunk() - -// Helper function for ChunkManager::get_chunk_smart. Given a chunk -// (which is supposed to be free floating for the moment), split it into two chunks. Return -// the follower chunk. -static Metachunk* split_chunk(Metachunk* c) { - - assert(c->level() > LOWEST_CHUNK_LEVEL, "cannot split"); - assert(c->is_free(), "Do not split chunks which are in use."); - - int new_level = c->level() - 1; - size_t new_level_byte_size = size_for_level(new_level); - address buddy_address = ((address)c) + new_level_byte_size; - - assert(new_level_byte_size == c->size() / 2, "sanity"); - - Metachunk* follower = new(new_level_byte_size, buddy_address) - Metachunk(new_level_byte_size, c->container(), false); - - // make leader level smaller - c->set_level(new_level); - - return follower; - -} - -// Retrieve a chunk from the chunk manager and attempt to be smart about it. -// split_stat defines the behavior. -Metachunk* ChunkManager::get_chunk_smart(int level, const ChunkManager::chunk_split_strategy_t* split_strat) { - - assert_lock_strong(MetaspaceExpand_lock); - - DEBUG_ONLY(check_valid_level(level)); - DEBUG_ONLY(check_valid_level(split_strat->largest_handout_level)); - DEBUG_ONLY(check_valid_level(split_strat->smallest_handout_level)); - DEBUG_ONLY(check_valid_level(split_strat->largest_splitting_level)); - assert(split_strat->largest_handout_level >= level && - split_strat->smallest_handout_level <= level && - split_strat->largest_splitting_level >= split_strat->largest_handout_level, "sanity"); - - Metachunk* c = get_chunk(level); - if (c != NULL) { - return c; - } - - // We have no suitable chunk. If the strategy specifies an acceptable larger or smaller chunk size, check if we - // have those. Prefer smaller chunks. - for (int level2 = split_strat->smallest_handout_level; level2 <= split_strat->largest_handout_level; level2 ++) { - Metachunk* c = get_chunk(level2); - if (c != NULL) { - return c; - } - } - - // We have found no acceptable chunk yet. Now attempt to find a smaller chunk, split it and return it. - int larger_chunk_level = split_strat->largest_handout_level + 1; // no need to search below largest_handout_level since we tested that already. - while (c != NULL && larger_chunk_level <= split_strat->largest_splitting_level) { - c = get_chunk(larger_chunk_level); - } - - if (c != NULL) { - // Splinter that chunk. Return all splinters back to the chunk manager. Retain just one of the target-sized chunk. - c->set_free(); // Needs to be free to be split. - while (c->level() != level) { - Metachunk* follower = split_chunk(c); - assert(c->base() + c->word_size() == follower->base(), "sanity"); - // return the follower to the freelist - add_chunk(follower); - } - DEBUG_ONLY(c->verify()); - - c->set_in_use(); - -#ifdef ASSERT - EVERY_NTH(VerifyMetaspaceInterval) - locked_verify(true); - END_EVERY_NTH -#endif - - return c; - - } - - return NULL; - -} - -/* - bool ChunkManager::attempt_to_coalesce_around_chunk(Metachunk* chunk, ChunkIndex target_chunk_type) { assert_lock_strong(MetaspaceExpand_lock); assert(chunk != NULL, "invalid chunk pointer"); @@ -445,7 +195,6 @@ } return num_removed; } -*/ // Update internal accounting after a chunk was added void ChunkManager::account_for_added_chunk(const Metachunk* c) { @@ -466,6 +215,16 @@ _free_chunks_total -= c->word_size(); } +ChunkIndex ChunkManager::list_index(size_t size) { + return get_chunk_type_by_size(size, is_class()); +} + +size_t ChunkManager::size_by_index(ChunkIndex index) const { + index_bounds_check(index); + assert(index != HumongousIndex, "Do not call for humongous chunks."); + return get_size_for_nonhumongous_chunktype(index, is_class()); +} + #ifdef ASSERT void ChunkManager::verify(bool slow) const { MutexLocker cl(MetaspaceExpand_lock, @@ -475,33 +234,30 @@ void ChunkManager::locked_verify(bool slow) const { log_trace(gc, metaspace, freelist)("verifying %s chunkmanager (%s).", - _name, slow ? "slow" : "quick"); + (is_class() ? "class space" : "metaspace"), (slow ? "slow" : "quick")); assert_lock_strong(MetaspaceExpand_lock); - int chunks_counted = 0; + size_t chunks_counted = 0; size_t wordsize_chunks_counted = 0; - for (int lvl = LOWEST_CHUNK_LEVEL; lvl <= HIGHEST_CHUNK_LEVEL; lvl ++) { - int chunks_in_this_level_counted = 0; - Metachunk* c = _free_chunks[lvl]; - assert(c->prev() == NULL, "sanity"); - while (c != NULL) { - if (slow) { - c->verify(); + for (ChunkIndex i = ZeroIndex; i < NumberOfFreeLists; i = next_chunk_index(i)) { + const ChunkList* list = _free_chunks + i; + if (list != NULL) { + Metachunk* chunk = list->head(); + while (chunk) { + if (slow) { + do_verify_chunk(chunk); + } + assert(chunk->is_tagged_free(), "Chunk should be tagged as free."); + chunks_counted ++; + wordsize_chunks_counted += chunk->size(); + chunk = chunk->next(); } - assert(c->is_free(), "Chunk should be tagged as free."); - chunks_counted ++; - wordsize_chunks_counted += c->size(); + } + } - if (c->next() != NULL) { - assert(c->next()->prev() == c, "sanity"); - } - c = c->next(); - } - assert(_num_free_chunks[lvl] == chunks_in_this_level_counted, - "Chunk count mismatch for level %d, expected %d, got %d", - lvl, _num_free_chunks[lvl], chunks_in_this_level_counted); - } + chunks_counted += humongous_dictionary()->total_free_blocks(); + wordsize_chunks_counted += humongous_dictionary()->total_size(); assert(chunks_counted == _free_chunks_count && wordsize_chunks_counted == _free_chunks_total, "freelist accounting mismatch: " @@ -518,7 +274,18 @@ _free_chunks_total, _free_chunks_count); } -/* +ChunkList* ChunkManager::free_chunks(ChunkIndex index) { + assert(index == SpecializedIndex || index == SmallIndex || index == MediumIndex, + "Bad index: %d", (int)index); + return &_free_chunks[index]; +} + +ChunkList* ChunkManager::find_free_chunks_list(size_t word_size) { + ChunkIndex index = list_index(word_size); + assert(index < HumongousIndex, "No humongous list"); + return free_chunks(index); +} + // Helper for chunk splitting: given a target chunk size and a larger free chunk, // split up the larger chunk into n smaller chunks, at least one of which should be // the target chunk of target chunk size. The smaller chunks, including the target @@ -621,7 +388,6 @@ return target_chunk; } - Metachunk* ChunkManager::free_chunks_get(size_t word_size) { assert_lock_strong(MetaspaceExpand_lock); @@ -732,7 +498,7 @@ return chunk; } -Metachunk* ChunkManager::find_chunk_for_level(size_t word_size) { +Metachunk* ChunkManager::chunk_freelist_allocate(size_t word_size) { assert_lock_strong(MetaspaceExpand_lock); // Take from the beginning of the list @@ -863,13 +629,11 @@ } } -*/ - -void ChunkManager::collect_statistics(ChunkManagerStatistics* out) const { /* +void ChunkManager::collect_statistics(ChunkManagerStatistics* out) const { MutexLocker cl(MetaspaceExpand_lock, Mutex::_no_safepoint_check_flag); for (ChunkIndex i = ZeroIndex; i < NumberOfInUseLists; i = next_chunk_index(i)) { out->chunk_stats(i).add(num_free_chunks(i), size_free_chunks_in_bytes(i) / sizeof(MetaWord)); - }*/ + } } } // namespace metaspace diff -r 45843600f2de -r 412ce076c0dc src/hotspot/share/memory/metaspace/chunkManager.hpp --- a/src/hotspot/share/memory/metaspace/chunkManager.hpp Thu Jun 06 08:58:26 2019 +0200 +++ b/src/hotspot/share/memory/metaspace/chunkManager.hpp Wed Jun 05 08:59:17 2019 +0200 @@ -26,6 +26,8 @@ #define SHARE_MEMORY_METASPACE_CHUNKMANAGER_HPP #include "memory/allocation.hpp" +#include "memory/binaryTreeDictionary.hpp" +#include "memory/freeList.hpp" #include "memory/metaspace/metachunk.hpp" #include "memory/metaspace/metaspaceStatistics.hpp" #include "memory/metaspaceChunkFreeListSummary.hpp" @@ -35,6 +37,9 @@ namespace metaspace { +typedef class FreeList ChunkList; +typedef BinaryTreeDictionary > ChunkTreeDictionary; + // Manages the global free lists of chunks. class ChunkManager : public CHeapObj { friend class ::ChunkManagerTestAccessor; @@ -43,66 +48,86 @@ // SpecializedChunk // SmallChunk // MediumChunk - Metachunk* _free_chunks [NUM_CHUNK_LEVELS]; - int _num_free_chunks [NUM_CHUNK_LEVELS]; + ChunkList _free_chunks[NumberOfFreeLists]; + + // Whether or not this is the class chunkmanager. + const bool _is_class; + + // Return non-humongous chunk list by its index. + ChunkList* free_chunks(ChunkIndex index); + + // Returns non-humongous chunk list for the given chunk word size. + ChunkList* find_free_chunks_list(size_t word_size); + + // HumongousChunk + ChunkTreeDictionary _humongous_dictionary; + + // Returns the humongous chunk dictionary. + ChunkTreeDictionary* humongous_dictionary() { return &_humongous_dictionary; } + const ChunkTreeDictionary* humongous_dictionary() const { return &_humongous_dictionary; } // Size, in metaspace words, of all chunks managed by this ChunkManager size_t _free_chunks_total; - // Number of chunks in this ChunkManager - // Todo: do we need this? Seems like a totally pointless information. - int _free_chunks_count; - - // Convenience name is convenient. - const char* const _name; + size_t _free_chunks_count; // Update counters after a chunk was added or removed removed. void account_for_added_chunk(const Metachunk* c); void account_for_removed_chunk(const Metachunk* c); - // Returns true if this manager contains the given chunk. Slow (walks free list) and - // only needed for verifications. - DEBUG_ONLY(bool contains_chunk(Metachunk* metachunk) const;) + // Given a pointer to a chunk, attempts to merge it with neighboring + // free chunks to form a bigger chunk. Returns true if successful. + bool attempt_to_coalesce_around_chunk(Metachunk* chunk, ChunkIndex target_chunk_type); - // Add a chunk to the free list (no coalescation, just an add) - // and adjust accounting. - void add_chunk(Metachunk* chunk); + // Helper for chunk merging: + // Given an address range with 1-n chunks which are all supposed to be + // free and hence currently managed by this ChunkManager, remove them + // from this ChunkManager and mark them as invalid. + // - This does not correct the occupancy map. + // - This does not adjust the counters in ChunkManager. + // - Does not adjust container count counter in containing VirtualSpaceNode. + // Returns number of chunks removed. + int remove_chunks_in_area(MetaWord* p, size_t word_size); + + // Helper for chunk splitting: given a target chunk size and a larger free chunk, + // split up the larger chunk into n smaller chunks, at least one of which should be + // the target chunk of target chunk size. The smaller chunks, including the target + // chunk, are returned to the freelist. The pointer to the target chunk is returned. + // Note that this chunk is supposed to be removed from the freelist right away. + Metachunk* split_chunk(size_t target_chunk_word_size, Metachunk* chunk); public: - ChunkManager(const char* name); + ChunkManager(bool is_class); - // Removing chunks from freelist: + // Add or delete (return) a chunk to the global freelist. + Metachunk* chunk_freelist_allocate(size_t word_size); - // Remove a chunk of the given level from its free list and return it; - // If there is no chunk of that level available, returns NULL. - // The returned chunk will be marked as "in use". - // Adjusts internal accounting. - Metachunk* get_chunk(int level); + // Map a size to a list index assuming that there are lists + // for special, small, medium, and humongous chunks. + ChunkIndex list_index(size_t size); - // Remove the given chunk from its free list and adjust accounting. - // (Called during node purging, so the chunk will shortly cease existing.) + // Map a given index to the chunk size. + size_t size_by_index(ChunkIndex index) const; + + bool is_class() const { return _is_class; } + + // Convenience accessors. + size_t medium_chunk_word_size() const { return size_by_index(MediumIndex); } + size_t small_chunk_word_size() const { return size_by_index(SmallIndex); } + size_t specialized_chunk_word_size() const { return size_by_index(SpecializedIndex); } + + // Take a chunk from the ChunkManager. The chunk is expected to be in + // the chunk manager (the freelist if non-humongous, the dictionary if + // humongous). void remove_chunk(Metachunk* chunk); - // Structure defining how a chunk is removed from the freelist. - struct chunk_split_strategy_t { - // largest chunk we split to get this chunk - int largest_splitting_level; - int largest_handout_level; - int smallest_handout_level; - }; + // Return a single chunk of type index to the ChunkManager. + void return_single_chunk(Metachunk* chunk); - // Retrieve a chunk from the chunk manager and attempt to be smart about it. - // split_stat defines the behavior. - Metachunk* get_chunk_smart(int level, const chunk_split_strategy_t* split_strat); - - // Adding chunks to freelist: - - // Return a chunk to the ChunkManager. - // Will coalesce with buddies if needed, so do not access the chunk after it has been - // returned! It may have become invalid. - // Will adjust accounting. - void return_chunk(Metachunk* chunk); + // Add the simple linked list of chunks to the freelist of chunks + // of type index. + void return_chunk_list(Metachunk* chunk); // Total of the space in the free chunks list size_t free_chunks_total_words() const { return _free_chunks_total; } @@ -111,12 +136,47 @@ // Number of chunks in the free chunks list size_t free_chunks_count() const { return _free_chunks_count; } - int num_free_chunks(int level) const { - assert(is_valid_level(level), "sanity"); - return _num_free_chunks[level]; + // Remove from a list by size. Selects list based on size of chunk. + Metachunk* free_chunks_get(size_t chunk_word_size); + +#define index_bounds_check(index) \ + assert(is_valid_chunktype(index), "Bad index: %d", (int) index) + + size_t num_free_chunks(ChunkIndex index) const { + index_bounds_check(index); + + if (index == HumongousIndex) { + return _humongous_dictionary.total_free_blocks(); + } + + ssize_t count = _free_chunks[index].count(); + return count == -1 ? 0 : (size_t) count; } - MetaspaceChunkFreeListSummary chunk_free_list_summary() const; + size_t size_free_chunks_in_bytes(ChunkIndex index) const { + index_bounds_check(index); + + size_t word_size = 0; + if (index == HumongousIndex) { + word_size = _humongous_dictionary.total_size(); + } else { + const size_t size_per_chunk_in_words = _free_chunks[index].size(); + word_size = size_per_chunk_in_words * num_free_chunks(index); + } + + return word_size * BytesPerWord; + } + + MetaspaceChunkFreeListSummary chunk_free_list_summary() const { + return MetaspaceChunkFreeListSummary(num_free_chunks(SpecializedIndex), + num_free_chunks(SmallIndex), + num_free_chunks(MediumIndex), + num_free_chunks(HumongousIndex), + size_free_chunks_in_bytes(SpecializedIndex), + size_free_chunks_in_bytes(SmallIndex), + size_free_chunks_in_bytes(MediumIndex), + size_free_chunks_in_bytes(HumongousIndex)); + } #ifdef ASSERT // Debug support @@ -130,10 +190,9 @@ // Fill in current statistic values to the given statistics object. void collect_statistics(ChunkManagerStatistics* out) const; - const char* name() const { return _name; } - }; } // namespace metaspace + #endif // SHARE_MEMORY_METASPACE_CHUNKMANAGER_HPP diff -r 45843600f2de -r 412ce076c0dc src/hotspot/share/memory/metaspace/metachunk.cpp --- a/src/hotspot/share/memory/metaspace/metachunk.cpp Thu Jun 06 08:58:26 2019 +0200 +++ b/src/hotspot/share/memory/metaspace/metachunk.cpp Wed Jun 05 08:59:17 2019 +0200 @@ -50,110 +50,50 @@ // Metachunk methods -void Metachunk::remove_from_list() { - if (_prev != NULL) { - _prev->set_next(_next); - } - if (_next != NULL) { - _next->set_prev(_prev); - } - _prev = _next = NULL; -} - - -Metachunk::Metachunk(int level, VirtualSpaceNode* container, bool fully_committed) - : _prev(NULL), _next(NULL) - , _level(level) - , _is_free(true) - , _words_allocated(0) - , _words_committed(0) - , _container(container) - , _sentinel(CHUNK_SENTINEL) +Metachunk::Metachunk(ChunkIndex chunktype, bool is_class, size_t word_size, + VirtualSpaceNode* container) + : Metabase(word_size), + _container(container), + _top(NULL), + _sentinel(CHUNK_SENTINEL), + _chunk_type(chunktype), + _is_class(is_class), + _origin(origin_normal), + _use_count(0) { - _words_allocated = overhead(); - if (fully_committed) { - _words_committed = size(); - } else { - // We do not know the commit status of the underlying memory, only that the first - // page - containing the header - must be committed. So assume that the rest is not. - // (only affects chunks spanning multiple pages). - const size_t page_size = os::vm_page_size(); - if (size() > page_size) { - _words_committed = page_size; - } else { - _words_committed = size(); - } - } + _top = initial_top(); + set_is_tagged_free(false); #ifdef ASSERT mangle(uninitMetaWordVal); verify(); #endif } -// Whether this chunk can uncommit parts of it. -bool Metachunk::can_uncommit() const { - return size() > os::vm_page_size(); -} - -// Make sure we can place wordsize bytes without hitting the commit -// boundary (assuming wordsize is small enough to fit into the chunk). -void Metachunk::ensure_committed(size_t requested_word_size) { - assert((_words_allocated + requested_word_size) < word_size(), "would not fit."); - assert(can_uncommit(), "this chunk is too small to contain uncommitted parts."); - - // For now, commit page wise and lets see how this goes. This may very well be too - // fine granular. - const size_t words_committed_needed = - align_up(_words_allocated + requested_word_size, os::vm_page_size()); - if (words_committed_needed > _words_committed) { - const size_t to_commit = words_committed_needed - _words_committed; - assert(is_aligned(to_commit, (size_t)os::vm_page_size()), "sanity"); - MetaWord* const commit_top = commit_top(); - if (os::commit_memory((char*)commit_top, to_commit, false)) { - _words_committed = words_committed_needed; - } else { - vm_exit_out_of_memory(to_commit, OOM_MMAP_ERROR, "Failed to mmap " SIZE_FORMAT "bytes for metachunk.", to_commit); - } +MetaWord* Metachunk::allocate(size_t word_size) { + MetaWord* result = NULL; + // If available, bump the pointer to allocate. + if (free_word_size() >= word_size) { + result = _top; + _top = _top + word_size; } -} - -MetaWord* Metachunk::allocate(size_t requested_word_size) { - assert(is_aligned(requested_word_size, sizeof(MetaWord)), "unaligned alloc"); - - MetaWord* result = NULL; - - // Would this fit? - const size_t remaining_words = word_size() - _words_allocated; - if (remaining_words < requested_word_size) { - return NULL; - } - - // Maybe we need to commit? - if (can_uncommit()) { - ensure_committed(requested_word_size); - } - - // If available, bump the pointer to allocate. - result = top(); - _words_allocated += requested_word_size; - return result; } // _bottom points to the start of the chunk including the overhead. size_t Metachunk::used_word_size() const { - return _words_allocated - overhead(); + return pointer_delta(_top, bottom(), sizeof(MetaWord)); } size_t Metachunk::free_word_size() const { - return size() - _words_allocated; + return pointer_delta(end(), _top, sizeof(MetaWord)); } void Metachunk::print_on(outputStream* st) const { st->print_cr("Metachunk:" " bottom " PTR_FORMAT " top " PTR_FORMAT - " end " PTR_FORMAT " size " SIZE_FORMAT, - p2i(base()), p2i(top()), p2i(end()), word_size()); + " end " PTR_FORMAT " size " SIZE_FORMAT " (%s)", + p2i(bottom()), p2i(_top), p2i(end()), word_size(), + chunk_size_name(get_chunk_type())); if (Verbose) { st->print_cr(" used " SIZE_FORMAT " free " SIZE_FORMAT, used_word_size(), free_word_size()); @@ -164,31 +104,69 @@ void Metachunk::mangle(juint word_value) { // Overwrite the payload of the chunk and not the links that // maintain list of chunks. - MetaWord* start = base() + overhead(); - assert(_words_committed >= overhead, "sanity"); - size_t mangle_size = _words_committed - overhead(); - Copy::fill_to_words((HeapWord*)start, size, word_value); + HeapWord* start = (HeapWord*)initial_top(); + size_t size = word_size() - overhead(); + Copy::fill_to_words(start, size, word_value); } void Metachunk::verify() const { assert(is_valid_sentinel(), "Chunk " PTR_FORMAT ": sentinel invalid", p2i(this)); - assert(is_valid_level(_level), "Invalid level (%d)", _level); + const ChunkIndex chunk_type = get_chunk_type(); + assert(is_valid_chunktype(chunk_type), "Chunk " PTR_FORMAT ": Invalid chunk type.", p2i(this)); + if (chunk_type != HumongousIndex) { + assert(word_size() == get_size_for_nonhumongous_chunktype(chunk_type, is_class()), + "Chunk " PTR_FORMAT ": wordsize " SIZE_FORMAT " does not fit chunk type %s.", + p2i(this), word_size(), chunk_size_name(chunk_type)); + } + assert(is_valid_chunkorigin(get_origin()), "Chunk " PTR_FORMAT ": Invalid chunk origin.", p2i(this)); + assert(bottom() <= _top && _top <= (MetaWord*)end(), + "Chunk " PTR_FORMAT ": Chunk top out of chunk bounds.", p2i(this)); - assert(base() == (MetaWord*) this, "sanity"); - assert(end() == base() + size(), "sanity"); - assert(top() >= base() + overhead() && top() <= end(), "sanity"); - assert(commit_top() >= top() && commit_top() <= end(), "sanity"); - - assert(_container != NULL, "sanity"); - - // Starting address shall be aligned to chunk size. - const size_t required_alignment = word_size() * sizeof(MetaWord); + // For non-humongous chunks, starting address shall be aligned + // to its chunk size. Humongous chunks start address is + // aligned to specialized chunk size. + const size_t required_alignment = + (chunk_type != HumongousIndex ? word_size() : get_size_for_nonhumongous_chunktype(SpecializedIndex, is_class())) * sizeof(MetaWord); assert(is_aligned((address)this, required_alignment), - "Chunk " PTR_FORMAT ": (size " SIZE_FORMAT ") not aligned correctly to " SIZE_FORMAT ".", + "Chunk " PTR_FORMAT ": (size " SIZE_FORMAT ") not aligned to " SIZE_FORMAT ".", p2i(this), word_size() * sizeof(MetaWord), required_alignment); } #endif // ASSERT +// Helper, returns a descriptive name for the given index. +const char* chunk_size_name(ChunkIndex index) { + switch (index) { + case SpecializedIndex: + return "specialized"; + case SmallIndex: + return "small"; + case MediumIndex: + return "medium"; + case HumongousIndex: + return "humongous"; + default: + return "Invalid index"; + } +} + +#ifdef ASSERT +void do_verify_chunk(Metachunk* chunk) { + guarantee(chunk != NULL, "Sanity"); + // Verify chunk itself; then verify that it is consistent with the + // occupany map of its containing node. + chunk->verify(); + VirtualSpaceNode* const vsn = chunk->container(); + OccupancyMap* const ocmap = vsn->occupancy_map(); + ocmap->verify_for_chunk(chunk); +} +#endif + +void do_update_in_use_info_for_chunk(Metachunk* chunk, bool inuse) { + chunk->set_is_tagged_free(!inuse); + OccupancyMap* const ocmap = chunk->container()->occupancy_map(); + ocmap->set_region_in_use((MetaWord*)chunk, chunk->word_size(), inuse); +} + } // namespace metaspace diff -r 45843600f2de -r 412ce076c0dc src/hotspot/share/memory/metaspace/metachunk.hpp --- a/src/hotspot/share/memory/metaspace/metachunk.hpp Thu Jun 06 08:58:26 2019 +0200 +++ b/src/hotspot/share/memory/metaspace/metachunk.hpp Wed Jun 05 08:59:17 2019 +0200 @@ -39,64 +39,73 @@ // Metachunks are reused (when freed are put on a global freelist) and // have no permanent association to a SpaceManager. -// +--------------+ <- end ----+ --+ -// | | | | -// | | | free | -// | ----------- | <- commit_top | | -// | | | | size | capacity -// | | | | -// | ----------- | <- top -- + | -// | | | | -// | | | used | -// +--------------+ -- + | -// | header | | overhead | -// +--------------+ <- base ----+ --+ +// +--------------+ <- end --+ --+ +// | | | | +// | | | free | +// | | | | +// | | | | size | capacity +// | | | | +// | | <- top -- + | +// | | | | +// | | | used | +// | | | | +// | | | | +// +--------------+ <- bottom --+ --+ +enum ChunkOrigin { + // Chunk normally born (via take_from_committed) + origin_normal = 1, + // Chunk was born as padding chunk + origin_pad = 2, + // Chunk was born as leftover chunk in VirtualSpaceNode::retire + origin_leftover = 3, + // Chunk was born as result of a merge of smaller chunks + origin_merge = 4, + // Chunk was born as result of a split of a larger chunk + origin_split = 5, -class Metachunk { + origin_minimum = origin_normal, + origin_maximum = origin_split, + origins_count = origin_maximum + 1 +}; + +inline bool is_valid_chunkorigin(ChunkOrigin origin) { + return origin == origin_normal || + origin == origin_pad || + origin == origin_leftover || + origin == origin_merge || + origin == origin_split; +} + +class Metachunk : public Metabase { friend class ::MetachunkTest; - // Note: lots of possibilities to shrink that header! but for now I concentrate on - // easy readability and dev speed. - - Metachunk* _prev; - Metachunk* _next; - - int _level; - - bool _is_free; - - // Current allocated words, including header. - size_t _words_allocated; - - // Current committed words, including header. - size_t _words_committed; - // The VirtualSpaceNode containing this chunk. VirtualSpaceNode* const _container; -#ifdef ASSERT + // Current allocation top. + MetaWord* _top; + // A 32bit sentinel for debugging purposes. - static const uint32_t CHUNK_SENTINEL = 0x4d4554EF; // "MET" - static const uint32_t CHUNK_SENTINEL_INVALID = 0xFEEEEEEF; + enum { CHUNK_SENTINEL = 0x4d4554EF, // "MET" + CHUNK_SENTINEL_INVALID = 0xFEEEEEEF + }; + uint32_t _sentinel; -#endif - MetaWord* base() const { return (MetaWord*)this; } - MetaWord* end() const { return (MetaWord*)this + word_size(); } - MetaWord* top() const { return base() + _words_allocated; } - MetaWord* commit_top() const { return base() + _words_committed; } + const ChunkIndex _chunk_type; + const bool _is_class; + // Whether the chunk is free (in freelist) or in use by some class loader. + bool _is_tagged_free; - // Make sure we can place wordsize bytes without hitting the commit - // boundary (assuming wordsize is small enough to fit into the chunk). - void ensure_committed(size_t wordsize); + ChunkOrigin _origin; + int _use_count; - // Whether this chunk can uncommit parts of it. - bool can_uncommit() const; + MetaWord* initial_top() const { return (MetaWord*)this + overhead(); } + MetaWord* top() const { return _top; } public: - // Metachunks are allocated out of a MetadataVirtualSpace and // and use some of its space to describe itself (plus alignment // considerations). Metadata is allocated in the rest of the chunk. @@ -109,62 +118,56 @@ // Size of the Metachunk header, in words, including alignment. static size_t overhead(); - int level() const { return _level; } - - // Only needed during coalescation - void set_level(int lvl) { _level = lvl; } - - // A "root chunk" is a chunk of the highest level. It cannot coalesce further - // and has no buddy. - bool is_root_chunk() const { return _level == NUM_CHUNK_LEVELS - 1; } - - // Returns size, in bytes, of this chunk including the header. - size_t size() const { - return MIN_CHUNK_BYTE_SIZE << _level; - } - - Metachunk(int level, VirtualSpaceNode* container, bool fully_committed = true); + Metachunk(ChunkIndex chunktype, bool is_class, size_t word_size, VirtualSpaceNode* container); MetaWord* allocate(size_t word_size); VirtualSpaceNode* container() const { return _container; } + MetaWord* bottom() const { return (MetaWord*) this; } + // Reset top to bottom so chunk can be reused. - void reset() { _words_allocated = 0; } - bool is_empty() { return _words_allocated == 0; } + void reset_empty() { _top = initial_top(); clear_next(); clear_prev(); } + bool is_empty() { return _top == initial_top(); } // used (has been allocated) // free (available for future allocations) - size_t word_size() const { return size() / sizeof(MetaWord); } + size_t word_size() const { return size(); } size_t used_word_size() const; size_t free_word_size() const; - bool is_free() const { return _is_free; } - void set_free() { _is_free = true; } - void set_in_use() { _is_free = false; } + bool is_tagged_free() { return _is_tagged_free; } + void set_is_tagged_free(bool v) { _is_tagged_free = v; } - void set_prev(Metachunk* c) { _prev = c; } - void set_next(Metachunk* c) { _next = c; } - Metachunk* prev() const { return _prev; } - Metachunk* next() const { return _next; } - void remove_from_list(); - - bool contains(const void* ptr) const { - return base() <= ptr && ptr < top(); - } + bool contains(const void* ptr) { return bottom() <= ptr && ptr < _top; } void print_on(outputStream* st) const; -#ifdef ASSERT bool is_valid_sentinel() const { return _sentinel == CHUNK_SENTINEL; } void remove_sentinel() { _sentinel = CHUNK_SENTINEL_INVALID; } -#endif + + int get_use_count() const { return _use_count; } + void inc_use_count() { _use_count ++; } + + ChunkOrigin get_origin() const { return _origin; } + void set_origin(ChunkOrigin orig) { _origin = orig; } + + ChunkIndex get_chunk_type() const { return _chunk_type; } + bool is_class() const { return _is_class; } DEBUG_ONLY(void mangle(juint word_value);) DEBUG_ONLY(void verify() const;) }; + +// Helper function that does a bunch of checks for a chunk. +DEBUG_ONLY(void do_verify_chunk(Metachunk* chunk);) + +// Given a Metachunk, update its in-use information (both in the +// chunk and the occupancy map). +void do_update_in_use_info_for_chunk(Metachunk* chunk, bool inuse); + } // namespace metaspace #endif // SHARE_MEMORY_METASPACE_METACHUNK_HPP diff -r 45843600f2de -r 412ce076c0dc src/hotspot/share/memory/metaspace/metaspaceCommon.cpp --- a/src/hotspot/share/memory/metaspace/metaspaceCommon.cpp Thu Jun 06 08:58:26 2019 +0200 +++ b/src/hotspot/share/memory/metaspace/metaspaceCommon.cpp Wed Jun 05 08:59:17 2019 +0200 @@ -130,6 +130,70 @@ } } +// Returns size of this chunk type. +size_t get_size_for_nonhumongous_chunktype(ChunkIndex chunktype, bool is_class) { + assert(is_valid_nonhumongous_chunktype(chunktype), "invalid chunk type."); + size_t size = 0; + if (is_class) { + switch(chunktype) { + case SpecializedIndex: size = ClassSpecializedChunk; break; + case SmallIndex: size = ClassSmallChunk; break; + case MediumIndex: size = ClassMediumChunk; break; + default: + ShouldNotReachHere(); + } + } else { + switch(chunktype) { + case SpecializedIndex: size = SpecializedChunk; break; + case SmallIndex: size = SmallChunk; break; + case MediumIndex: size = MediumChunk; break; + default: + ShouldNotReachHere(); + } + } + return size; +} + +ChunkIndex get_chunk_type_by_size(size_t size, bool is_class) { + if (is_class) { + if (size == ClassSpecializedChunk) { + return SpecializedIndex; + } else if (size == ClassSmallChunk) { + return SmallIndex; + } else if (size == ClassMediumChunk) { + return MediumIndex; + } else if (size > ClassMediumChunk) { + // A valid humongous chunk size is a multiple of the smallest chunk size. + assert(is_aligned(size, ClassSpecializedChunk), "Invalid chunk size"); + return HumongousIndex; + } + } else { + if (size == SpecializedChunk) { + return SpecializedIndex; + } else if (size == SmallChunk) { + return SmallIndex; + } else if (size == MediumChunk) { + return MediumIndex; + } else if (size > MediumChunk) { + // A valid humongous chunk size is a multiple of the smallest chunk size. + assert(is_aligned(size, SpecializedChunk), "Invalid chunk size"); + return HumongousIndex; + } + } + ShouldNotReachHere(); + return (ChunkIndex)-1; +} + +ChunkIndex next_chunk_index(ChunkIndex i) { + assert(i < NumberOfInUseLists, "Out of bound"); + return (ChunkIndex) (i+1); +} + +ChunkIndex prev_chunk_index(ChunkIndex i) { + assert(i > ZeroIndex, "Out of bound"); + return (ChunkIndex) (i-1); +} + const char* loaders_plural(uintx num) { return num == 1 ? "loader" : "loaders"; } diff -r 45843600f2de -r 412ce076c0dc src/hotspot/share/memory/metaspace/metaspaceCommon.hpp --- a/src/hotspot/share/memory/metaspace/metaspaceCommon.hpp Thu Jun 06 08:58:26 2019 +0200 +++ b/src/hotspot/share/memory/metaspace/metaspaceCommon.hpp Wed Jun 05 08:59:17 2019 +0200 @@ -33,29 +33,14 @@ namespace metaspace { -// Large enough to hold 99% of InstanceKlass. -static const size_t MIN_CHUNK_BYTE_SIZE = 1 * K; - -// Let MAX_CHUNK_SIZE be large enough to hold the largest possible InstanceKlass. -static const int NUM_CHUNK_LEVELS = 12; -static const size_t MAX_CHUNK_BYTE_SIZE = MIN_CHUNK_BYTE_SIZE << NUM_CHUNK_LEVELS; - -static const size_t MIN_CHUNK_WORD_SIZE = MIN_CHUNK_BYTE_SIZE / sizeof(MetaWord); -static const size_t MAX_CHUNK_WORD_SIZE = MAX_CHUNK_BYTE_SIZE / sizeof(MetaWord); - -static const int HIGHEST_CHUNK_LEVEL = NUM_CHUNK_LEVELS - 1; -static const int LOWEST_CHUNK_LEVEL = 0; - -inline bool is_valid_level(int level) { - return level >= LOWEST_CHUNK_LEVEL && level <= HIGHEST_CHUNK_LEVEL; -} - -DEBUG_ONLY(inline void check_valid_level(int lvl) { assert(is_valid_level(lvl), "invalid level (%d)", lvl); }) - -inline size_t size_for_level(int level) { - assert(is_valid_level(level), "invalid chunk level (%d)", level); - return MIN_CHUNK_BYTE_SIZE << level; -} +enum ChunkSizes { // in words. + ClassSpecializedChunk = 128, + SpecializedChunk = 128, + ClassSmallChunk = 256, + SmallChunk = 512, + ClassMediumChunk = 4 * K, + MediumChunk = 8 * K +}; // Print a size, in words, scaled. void print_scaled_words(outputStream* st, size_t word_size, size_t scale = 0, int width = -1); @@ -113,6 +98,48 @@ extern internal_statistics_t g_internal_statistics; #endif +// ChunkIndex defines the type of chunk. +// Chunk types differ by size: specialized < small < medium, chunks +// larger than medium are humongous chunks of varying size. +enum ChunkIndex { + ZeroIndex = 0, + SpecializedIndex = ZeroIndex, + SmallIndex = SpecializedIndex + 1, + MediumIndex = SmallIndex + 1, + HumongousIndex = MediumIndex + 1, + NumberOfFreeLists = 3, + NumberOfInUseLists = 4 +}; + +// Utility functions. +size_t get_size_for_nonhumongous_chunktype(ChunkIndex chunk_type, bool is_class); +ChunkIndex get_chunk_type_by_size(size_t size, bool is_class); + +ChunkIndex next_chunk_index(ChunkIndex i); +ChunkIndex prev_chunk_index(ChunkIndex i); +// Returns a descriptive name for a chunk type. +const char* chunk_size_name(ChunkIndex index); + +// Verify chunk sizes. +inline bool is_valid_chunksize(bool is_class, size_t size) { + const size_t reasonable_maximum_humongous_chunk_size = 1 * G; + return is_aligned(size, sizeof(MetaWord)) && + size < reasonable_maximum_humongous_chunk_size && + is_class ? + (size == ClassSpecializedChunk || size == ClassSmallChunk || size >= ClassMediumChunk) : + (size == SpecializedChunk || size == SmallChunk || size >= MediumChunk); +} + +// Verify chunk type. +inline bool is_valid_chunktype(ChunkIndex index) { + return index == SpecializedIndex || index == SmallIndex || + index == MediumIndex || index == HumongousIndex; +} + +inline bool is_valid_nonhumongous_chunktype(ChunkIndex index) { + return is_valid_chunktype(index) && index != HumongousIndex; +} + // Pretty printing helpers const char* classes_plural(uintx num); const char* loaders_plural(uintx num); diff -r 45843600f2de -r 412ce076c0dc src/hotspot/share/memory/metaspace/spaceManager.cpp --- a/src/hotspot/share/memory/metaspace/spaceManager.cpp Thu Jun 06 08:58:26 2019 +0200 +++ b/src/hotspot/share/memory/metaspace/spaceManager.cpp Wed Jun 05 08:59:17 2019 +0200 @@ -384,7 +384,7 @@ Metachunk* SpaceManager::get_new_chunk(size_t chunk_word_size) { // Get a chunk from the chunk freelist - Metachunk* next = chunk_manager()->get_chunk(chunk_word_size); + Metachunk* next = chunk_manager()->chunk_freelist_allocate(chunk_word_size); if (next == NULL) { next = vs_list()->get_new_chunk(chunk_word_size,