< prev index next >

src/hotspot/share/memory/metaspace/metachunk.cpp

Print this page
rev 60538 : imported patch jep387-all.patch

*** 1,7 **** /* ! * Copyright (c) 2012, 2016, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License version 2 only, as * published by the Free Software Foundation. --- 1,8 ---- /* ! * Copyright (c) 2012, 2020, Oracle and/or its affiliates. All rights reserved. ! * Copyright (c) 2017, 2020 SAP SE. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License version 2 only, as * published by the Free Software Foundation.
*** 21,172 **** * questions. * */ #include "precompiled.hpp" ! #include "memory/allocation.hpp" #include "memory/metaspace/metachunk.hpp" ! #include "memory/metaspace/occupancyMap.hpp" #include "memory/metaspace/virtualSpaceNode.hpp" #include "utilities/align.hpp" #include "utilities/copy.hpp" #include "utilities/debug.hpp" namespace metaspace { ! size_t Metachunk::object_alignment() { ! // Must align pointers and sizes to 8, ! // so that 64 bit types get correctly aligned. ! const size_t alignment = 8; ! ! // Make sure that the Klass alignment also agree. ! STATIC_ASSERT(alignment == (size_t)KlassAlignmentInBytes); ! ! return alignment; ! } ! ! size_t Metachunk::overhead() { ! return align_up(sizeof(Metachunk), object_alignment()) / BytesPerWord; ! } ! ! // Metachunk methods ! ! Metachunk::Metachunk(ChunkIndex chunktype, bool is_class, size_t word_size, ! VirtualSpaceNode* container) ! : Metabase<Metachunk>(word_size), ! _container(container), ! _top(NULL), ! _sentinel(CHUNK_SENTINEL), ! _chunk_type(chunktype), ! _is_class(is_class), ! _origin(origin_normal), ! _use_count(0) ! { ! _top = initial_top(); ! set_is_tagged_free(false); #ifdef ASSERT ! mangle(uninitMetaWordVal); ! verify(); ! #endif } ! MetaWord* Metachunk::allocate(size_t word_size) { ! MetaWord* result = NULL; ! // If available, bump the pointer to allocate. ! if (free_word_size() >= word_size) { ! result = _top; ! _top = _top + word_size; } ! return result; } ! // _bottom points to the start of the chunk including the overhead. ! size_t Metachunk::used_word_size() const { ! return pointer_delta(_top, bottom(), sizeof(MetaWord)); } ! size_t Metachunk::free_word_size() const { ! return pointer_delta(end(), _top, sizeof(MetaWord)); } ! void Metachunk::print_on(outputStream* st) const { ! st->print_cr("Metachunk:" ! " bottom " PTR_FORMAT " top " PTR_FORMAT ! " end " PTR_FORMAT " size " SIZE_FORMAT " (%s)", ! p2i(bottom()), p2i(_top), p2i(end()), word_size(), ! chunk_size_name(get_chunk_type())); ! if (Verbose) { ! st->print_cr(" used " SIZE_FORMAT " free " SIZE_FORMAT, ! used_word_size(), free_word_size()); } } #ifdef ASSERT ! void Metachunk::mangle(juint word_value) { ! // Overwrite the payload of the chunk and not the links that ! // maintain list of chunks. ! HeapWord* start = (HeapWord*)initial_top(); ! size_t size = word_size() - overhead(); ! Copy::fill_to_words(start, size, word_value); ! } ! ! void Metachunk::verify() const { ! assert(is_valid_sentinel(), "Chunk " PTR_FORMAT ": sentinel invalid", p2i(this)); ! const ChunkIndex chunk_type = get_chunk_type(); ! assert(is_valid_chunktype(chunk_type), "Chunk " PTR_FORMAT ": Invalid chunk type.", p2i(this)); ! if (chunk_type != HumongousIndex) { ! assert(word_size() == get_size_for_nonhumongous_chunktype(chunk_type, is_class()), ! "Chunk " PTR_FORMAT ": wordsize " SIZE_FORMAT " does not fit chunk type %s.", ! p2i(this), word_size(), chunk_size_name(chunk_type)); ! } ! assert(is_valid_chunkorigin(get_origin()), "Chunk " PTR_FORMAT ": Invalid chunk origin.", p2i(this)); ! assert(bottom() <= _top && _top <= (MetaWord*)end(), ! "Chunk " PTR_FORMAT ": Chunk top out of chunk bounds.", p2i(this)); ! ! // For non-humongous chunks, starting address shall be aligned ! // to its chunk size. Humongous chunks start address is ! // aligned to specialized chunk size. ! const size_t required_alignment = ! (chunk_type != HumongousIndex ? word_size() : get_size_for_nonhumongous_chunktype(SpecializedIndex, is_class())) * sizeof(MetaWord); ! assert(is_aligned((address)this, required_alignment), ! "Chunk " PTR_FORMAT ": (size " SIZE_FORMAT ") not aligned to " SIZE_FORMAT ".", ! p2i(this), word_size() * sizeof(MetaWord), required_alignment); } ! #endif // ASSERT - // Helper, returns a descriptive name for the given index. - const char* chunk_size_name(ChunkIndex index) { - switch (index) { - case SpecializedIndex: - return "specialized"; - case SmallIndex: - return "small"; - case MediumIndex: - return "medium"; - case HumongousIndex: - return "humongous"; - default: - return "Invalid index"; } } ! #ifdef ASSERT ! void do_verify_chunk(Metachunk* chunk) { ! guarantee(chunk != NULL, "Sanity"); ! // Verify chunk itself; then verify that it is consistent with the ! // occupany map of its containing node. ! chunk->verify(); ! VirtualSpaceNode* const vsn = chunk->container(); ! OccupancyMap* const ocmap = vsn->occupancy_map(); ! ocmap->verify_for_chunk(chunk); } - #endif ! void do_update_in_use_info_for_chunk(Metachunk* chunk, bool inuse) { ! chunk->set_is_tagged_free(!inuse); ! OccupancyMap* const ocmap = chunk->container()->occupancy_map(); ! ocmap->set_region_in_use((MetaWord*)chunk, chunk->word_size(), inuse); } } // namespace metaspace --- 22,507 ---- * questions. * */ #include "precompiled.hpp" ! ! #include "logging/log.hpp" ! #include "memory/metaspace/chunkLevel.hpp" #include "memory/metaspace/metachunk.hpp" ! #include "memory/metaspace/metaspaceCommon.hpp" ! #include "memory/metaspace/settings.hpp" #include "memory/metaspace/virtualSpaceNode.hpp" + #include "runtime/mutexLocker.hpp" + #include "utilities/align.hpp" #include "utilities/copy.hpp" #include "utilities/debug.hpp" namespace metaspace { ! // Return a single char presentation of the state ('f', 'u', 'd') ! char Metachunk::get_state_char() const { ! switch (_state) { ! case state_free: return 'f'; ! case state_in_use: return 'u'; ! case state_dead: return 'd'; ! } ! return '?'; ! } ! #ifdef ASSERT ! void Metachunk::assert_have_expand_lock() { ! assert_lock_strong(MetaspaceExpand_lock); } + #endif + + // Commit uncommitted section of the chunk. + // Fails if we hit a commit limit. + bool Metachunk::commit_up_to(size_t new_committed_words) { + + // Please note: + // + // VirtualSpaceNode::ensure_range_is_committed(), when called over a range containing both committed and uncommitted parts, + // will replace the whole range with a new mapping, thus erasing the existing content in the committed parts. Therefore + // we must make sure never to call VirtualSpaceNode::ensure_range_is_committed() over a range containing live data. + // + // Luckily, this cannot happen by design. We have two cases: + // + // 1) chunks equal or larger than a commit granule. + // In this case, due to chunk geometry, the chunk should cover whole commit granules (in other words, a chunk equal or larger than + // a commit granule will never share a granule with a neighbor). That means whatever we commit or uncommit here does not affect + // neighboring chunks. We only have to take care not to re-commit used parts of ourself. We do this by moving the committed_words + // limit in multiple of commit granules. + // + // 2) chunks smaller than a commit granule. + // In this case, a chunk shares a single commit granule with its neighbors. But this never can be a problem: + // - Either the commit granule is already committed (and maybe the neighbors contain live data). In that case calling + // ensure_range_is_committed() will do nothing. + // - Or the commit granule is not committed, but in this case, the neighbors are uncommitted too and cannot contain live data. + + #ifdef ASSERT + if (word_size() >= Settings::commit_granule_words()) { + // case (1) + assert(is_aligned(base(), Settings::commit_granule_bytes()) && + is_aligned(end(), Settings::commit_granule_bytes()), + "Chunks larger than a commit granule must cover whole granules."); + assert(is_aligned(_committed_words, Settings::commit_granule_words()), + "The commit boundary must be aligned to commit granule size"); + assert(_used_words <= _committed_words, "Sanity"); + } else { + // case (2) + assert(_committed_words == 0 || _committed_words == word_size(), "Sanity"); + } + #endif + + // We should hold the expand lock at this point. + assert_lock_strong(MetaspaceExpand_lock); + + const size_t commit_from = _committed_words; + const size_t commit_to = MIN2(align_up(new_committed_words, Settings::commit_granule_words()), word_size()); + + assert(commit_from >= used_words(), "Sanity"); + assert(commit_to <= word_size(), "Sanity"); ! if (commit_to > commit_from) { ! log_debug(metaspace)("Chunk " METACHUNK_FORMAT ": attempting to move commit line to " ! SIZE_FORMAT " words.", METACHUNK_FORMAT_ARGS(this), commit_to); ! ! if (!_vsnode->ensure_range_is_committed(base() + commit_from, commit_to - commit_from)) { ! DEBUG_ONLY(verify(true);) ! return false; } ! } ! ! // Remember how far we have committed. ! _committed_words = commit_to; ! ! DEBUG_ONLY(verify(true);) ! ! return true; ! } ! ! // Ensure that chunk is committed up to at least new_committed_words words. ! // Fails if we hit a commit limit. ! bool Metachunk::ensure_committed(size_t new_committed_words) { ! ! bool rc = true; ! ! if (new_committed_words > committed_words()) { ! MutexLocker cl(MetaspaceExpand_lock, Mutex::_no_safepoint_check_flag); ! rc = commit_up_to(new_committed_words); ! } ! ! return rc; ! } ! bool Metachunk::ensure_committed_locked(size_t new_committed_words) { ! ! // the .._locked() variant should be called if we own the lock already. ! assert_lock_strong(MetaspaceExpand_lock); ! ! bool rc = true; ! ! if (new_committed_words > committed_words()) { ! rc = commit_up_to(new_committed_words); ! } ! ! return rc; ! } ! // Uncommit chunk area. The area must be a common multiple of the ! // commit granule size (in other words, we cannot uncommit chunks smaller than ! // a commit granule size). ! void Metachunk::uncommit() { ! MutexLocker cl(MetaspaceExpand_lock, Mutex::_no_safepoint_check_flag); ! uncommit_locked(); ! } ! ! void Metachunk::uncommit_locked() { ! // Only uncommit chunks which are free, have no used words set (extra precaution) and are equal or larger in size than a single commit granule. ! assert_lock_strong(MetaspaceExpand_lock); ! assert(_state == state_free && _used_words == 0 && word_size() >= Settings::commit_granule_words(), ! "Only free chunks equal or larger than commit granule size can be uncommitted " ! "(chunk " METACHUNK_FULL_FORMAT ").", METACHUNK_FULL_FORMAT_ARGS(this)); ! if (word_size() >= Settings::commit_granule_words()) { ! _vsnode->uncommit_range(base(), word_size()); ! _committed_words = 0; } } + void Metachunk::set_committed_words(size_t v) { + // Set committed words. Since we know that we only commit whole commit granules, we can round up v here. + v = MIN2(align_up(v, Settings::commit_granule_words()), word_size()); + _committed_words = v; + } + + + // Allocate word_size words from this chunk (word_size must be aligned to + // allocation_alignment_words). + // + // Caller must make sure the chunk is both large enough and committed far enough + // to hold the allocation. Will always work. + // + MetaWord* Metachunk::allocate(size_t request_word_size) { + + log_trace(metaspace)("Chunk " METACHUNK_FULL_FORMAT ": allocating " SIZE_FORMAT " words.", + METACHUNK_FULL_FORMAT_ARGS(this), request_word_size); + + // Caller must have made sure this works + assert(free_words() >= request_word_size, "Chunk too small."); + assert(free_below_committed_words() >= request_word_size, "Chunk not committed."); + + MetaWord* const p = top(); + + _used_words += request_word_size; + + SOMETIMES(verify(false);) + + return p; + + } #ifdef ASSERT ! ! // Zap this structure. ! void Metachunk::zap_header(uint8_t c) { ! memset(this, c, sizeof(Metachunk)); } ! void Metachunk::fill_with_pattern(MetaWord pattern, size_t word_size) { ! assert(word_size <= committed_words(), "Sanity"); ! for (size_t l = 0; l < word_size; l ++) { ! _base[l] = pattern; ! } ! } ! ! void Metachunk::check_pattern(MetaWord pattern, size_t word_size) { ! assert(word_size <= committed_words(), "Sanity"); ! for (size_t l = 0; l < word_size; l ++) { ! assert(_base[l] == pattern, ! "chunk " METACHUNK_FULL_FORMAT ": pattern change at " PTR_FORMAT ": expected " UINTX_FORMAT " but got " UINTX_FORMAT ".", ! METACHUNK_FULL_FORMAT_ARGS(this), p2i(_base + l), (uintx)pattern, (uintx)_base[l]); ! ! //////////////////////////////////////////// ! // A double-headed list of Metachunks. ! ! class AbstractMetachunkList { ! ! Metachunk* _first; ! Metachunk* _last; ! ! // Number of chunks ! IntCounter _num; ! ! protected: ! ! AbstractMetachunkList() : _first(NULL), _last(NULL), _num() {} ! ! Metachunk* first() const { return _first; } ! int count() const { return _num.get(); } ! ! // Add chunk to the front of the list. ! void add_front(Metachunk* c) { ! if (_first == NULL) { ! assert(_last == NULL && _num.get() == 0, "Sanity"); ! _first = _last = c; ! c->set_prev(NULL); ! c->set_next(NULL); ! } else { ! assert(_last != NULL && _num.get() > 0, "Sanity"); ! c->set_next(_first); ! c->set_prev(NULL); ! _first->set_prev(c); ! _first = c; ! } ! _num.increment(); ! } ! ! // Add chunk to the back of the list. ! void add_back(Metachunk* c) { ! if (_last == NULL) { ! assert(_first == NULL && _num.get() == 0, "Sanity"); ! _last = _first = c; ! c->set_prev(NULL); ! c->set_next(NULL); ! } else { ! assert(_first != NULL && _num.get() > 0, "Sanity"); ! c->set_next(NULL); ! c->set_prev(_last); ! _last->set_next(c); ! _last = c; ! } ! _num.increment(); ! } ! ! // Remove chunk from the front of the list. Returns NULL if list is empty. ! Metachunk* remove_front() { ! Metachunk* c = NULL; ! if (_first == NULL) { ! assert(_last == NULL && _num.get() == 0, "Sanity"); ! } else { ! c = _first; ! assert(c->prev() == NULL, "Sanity"); ! if (_first == _last) { ! assert(_num.get() == 1, "Sanity"); ! _first = _last = NULL; ! } else { ! assert(_num.get() > 1, "Sanity"); ! _first = _first->next(); ! _first->set_prev(NULL); ! } ! _num.decrement(); ! c->set_next(NULL); ! } ! return c; ! } ! ! // Remove chunk from the back of the list. Returns NULL if list is empty. ! Metachunk* remove_back() { ! Metachunk* c = NULL; ! if (_last == NULL) { ! assert(_first == NULL && _num.get() == 0, "Sanity"); ! } else { ! c = _last; ! assert(c->next() == NULL, "Sanity"); ! if (_first == _last) { ! assert(_num.get() == 1, "Sanity"); ! _first = _last = NULL; ! } else { ! assert(_num.get() > 1, "Sanity"); ! _last = _last->prev(); ! _last->set_next(NULL); ! } ! _num.decrement(); ! c->set_prev(NULL); ! } ! return c; ! } ! ! public: ! ! #ifdef ASSERT ! bool contains(const Metachunk* c) const; ! void verify() const; ! #endif ! ! // Returns size, in words, of committed space of all chunks in this list. ! // Note: walks list. ! size_t committed_word_size() const { ! size_t l = 0; ! for (const Metachunk* c = _first; c != NULL; c = c->next()) { ! l += c->committed_words(); ! } ! return l; ! } ! ! void print_on(outputStream* st) const; ! ! }; ! ! class UnsortedMetachunkList : public AbstractMetachunkList { ! public: ! ! ! ! ! ! }; ! } } ! ! // Verifies linking with neighbors in virtual space. ! // Can only be done under expand lock protection. ! void Metachunk::verify_neighborhood() const { ! ! assert_lock_strong(MetaspaceExpand_lock); ! assert(!is_dead(), "Do not call on dead chunks."); ! ! if (is_root_chunk()) { ! ! // Root chunks are all alone in the world. ! assert(next_in_vs() == NULL || prev_in_vs() == NULL, "Root chunks should have no neighbors"); ! ! } else { ! ! // Non-root chunks have neighbors, at least one, possibly two. ! ! assert(next_in_vs() != NULL || prev_in_vs() != NULL, ! "A non-root chunk should have neighbors (chunk @" PTR_FORMAT ! ", base " PTR_FORMAT ", level " CHKLVL_FORMAT ".", ! p2i(this), p2i(base()), level()); ! ! if (prev_in_vs() != NULL) { ! assert(prev_in_vs()->end() == base(), ! "Chunk " METACHUNK_FULL_FORMAT ": should be adjacent to predecessor: " METACHUNK_FULL_FORMAT ".", ! METACHUNK_FULL_FORMAT_ARGS(this), METACHUNK_FULL_FORMAT_ARGS(prev_in_vs())); ! assert(prev_in_vs()->next_in_vs() == this, ! "Chunk " METACHUNK_FULL_FORMAT ": broken link to left neighbor: " METACHUNK_FULL_FORMAT " (" PTR_FORMAT ").", ! METACHUNK_FULL_FORMAT_ARGS(this), METACHUNK_FULL_FORMAT_ARGS(prev_in_vs()), p2i(prev_in_vs()->next_in_vs())); ! } ! ! if (next_in_vs() != NULL) { ! assert(end() == next_in_vs()->base(), ! "Chunk " METACHUNK_FULL_FORMAT ": should be adjacent to successor: " METACHUNK_FULL_FORMAT ".", ! METACHUNK_FULL_FORMAT_ARGS(this), METACHUNK_FULL_FORMAT_ARGS(next_in_vs())); ! assert(next_in_vs()->prev_in_vs() == this, ! "Chunk " METACHUNK_FULL_FORMAT ": broken link to right neighbor: " METACHUNK_FULL_FORMAT " (" PTR_FORMAT ").", ! METACHUNK_FULL_FORMAT_ARGS(this), METACHUNK_FULL_FORMAT_ARGS(next_in_vs()), p2i(next_in_vs()->prev_in_vs())); ! } ! ! // One of the neighbors must be the buddy. It can be whole or splintered. ! ! // The chunk following us or preceeding us may be our buddy or a splintered part of it. ! Metachunk* buddy = is_leader() ? next_in_vs() : prev_in_vs(); ! ! assert(buddy != NULL, "Missing neighbor."); ! assert(!buddy->is_dead(), "Invalid buddy state."); ! ! // This neighbor is either or buddy (same level) or a splinter of our buddy - hence ! // the level can never be smaller (aka the chunk size cannot be larger). ! assert(buddy->level() >= level(), "Wrong level."); ! ! if (buddy->level() == level()) { ! ! // If the buddy is of the same size as us, it is unsplintered. ! assert(buddy->is_leader() == !is_leader(), ! "Only one chunk can be leader in a pair"); ! ! // When direct buddies are neighbors, one or both should be in use, otherwise they should ! // have been merged. ! ! // But since we call this verification function from internal functions where we are about to merge or just did split, ! // do not test this. We have RootChunkArea::verify_area_is_ideally_merged() for testing that. ! ! // assert(buddy->is_in_use() || is_in_use(), "incomplete merging?"); ! ! if (is_leader()) { ! assert(buddy->base() == end(), "Sanity"); ! assert(is_aligned(base(), word_size() * 2 * BytesPerWord), "Sanity"); ! } else { ! assert(buddy->end() == base(), "Sanity"); ! assert(is_aligned(buddy->base(), word_size() * 2 * BytesPerWord), "Sanity"); ! } ! ! } else { ! ! // Buddy, but splintered, and this is a part of it. ! if (is_leader()) { ! assert(buddy->base() == end(), "Sanity"); ! } else { ! assert(buddy->end() > (base() - word_size()), "Sanity"); ! } ! ! } ! } } ! volatile MetaWord dummy = 0; ! ! void Metachunk::verify(bool slow) const { ! ! // Note. This should be called under CLD lock protection. ! ! // We can verify everything except the _prev_in_vs/_next_in_vs pair. ! // This is because neighbor chunks may be added concurrently, so we cannot rely ! // on the content of _next_in_vs/_prev_in_vs unless we have the expand lock. ! ! assert(!is_dead(), "Do not call on dead chunks."); ! ! if (is_free()) { ! assert(used_words() == 0, "free chunks are not used."); ! } ! ! // Note: only call this on a life Metachunk. ! chunklevel::check_valid_level(level()); ! ! assert(base() != NULL, "No base ptr"); ! ! assert(committed_words() >= used_words(), ! "mismatch: committed: " SIZE_FORMAT ", used: " SIZE_FORMAT ".", ! committed_words(), used_words()); ! ! assert(word_size() >= committed_words(), ! "mismatch: word_size: " SIZE_FORMAT ", committed: " SIZE_FORMAT ".", ! word_size(), committed_words()); ! ! // Test base pointer ! assert(base() != NULL, "Base pointer NULL"); ! assert(vsnode() != NULL, "No space"); ! vsnode()->check_pointer(base()); ! ! // Starting address shall be aligned to chunk size. ! const size_t required_alignment = word_size() * sizeof(MetaWord); ! assert_is_aligned(base(), required_alignment); ! ! // If slow, test the committed area ! if (slow && _committed_words > 0) { ! for (const MetaWord* p = _base; p < _base + _committed_words; p += os::vm_page_size()) { ! dummy = *p; ! } ! dummy = *(_base + _committed_words - 1); ! } ! ! } ! #endif // ASSERT ! ! void Metachunk::print_on(outputStream* st) const { ! ! // Note: must also work with invalid/random data. (e.g. do not call word_size()) ! st->print("Chunk @" PTR_FORMAT ", state %c, base " PTR_FORMAT ", " ! "level " CHKLVL_FORMAT " (" SIZE_FORMAT " words), " ! "used " SIZE_FORMAT " words, committed " SIZE_FORMAT " words.", ! p2i(this), get_state_char(), p2i(base()), level(), ! (chunklevel::is_valid_level(level()) ? chunklevel::word_size_for_level(level()) : (size_t)-1), ! used_words(), committed_words()); ! } } // namespace metaspace
< prev index next >