# HG changeset patch # User stuefe # Date 1597251984 -7200 # Wed Aug 12 19:06:24 2020 +0200 # Node ID 9a3a277144a63c5a64bcd9696ac285b14407e250 # Parent ce770ba672fef24a214113780a16564c30c73851 imported patch jep387-core.patch diff --git a/src/hotspot/share/memory/metaspace.cpp b/src/hotspot/share/memory/metaspace.cpp --- a/src/hotspot/share/memory/metaspace.cpp +++ b/src/hotspot/share/memory/metaspace.cpp @@ -23,63 +23,190 @@ */ #include "precompiled.hpp" + #include "aot/aotLoader.hpp" -#include "classfile/classLoaderDataGraph.hpp" #include "gc/shared/collectedHeap.hpp" #include "logging/log.hpp" #include "logging/logStream.hpp" #include "memory/filemap.hpp" #include "memory/metaspace.hpp" -#include "memory/metaspace/chunkManager.hpp" -#include "memory/metaspace/metachunk.hpp" -#include "memory/metaspace/metaspaceCommon.hpp" -#include "memory/metaspace/printCLDMetaspaceInfoClosure.hpp" -#include "memory/metaspace/spaceManager.hpp" -#include "memory/metaspace/virtualSpaceList.hpp" #include "memory/metaspaceShared.hpp" #include "memory/metaspaceTracer.hpp" +#include "memory/metaspace/chunkHeaderPool.hpp" +#include "memory/metaspace/chunkManager.hpp" +#include "memory/metaspace/commitLimiter.hpp" +#include "memory/metaspace/metaspaceCommon.hpp" +#include "memory/metaspace/metaspaceContext.hpp" +#include "memory/metaspace/metaspaceEnums.hpp" +#include "memory/metaspace/metaspaceReport.hpp" +#include "memory/metaspace/metaspaceSizesSnapshot.hpp" +#include "memory/metaspace/runningCounters.hpp" +#include "memory/metaspace/settings.hpp" +#include "memory/metaspace/virtualSpaceList.hpp" #include "memory/universe.hpp" #include "oops/compressedOops.hpp" #include "runtime/atomic.hpp" #include "runtime/init.hpp" +#include "runtime/java.hpp" #include "services/memTracker.hpp" #include "utilities/copy.hpp" #include "utilities/debug.hpp" #include "utilities/formatBuffer.hpp" #include "utilities/globalDefinitions.hpp" -#include "utilities/vmError.hpp" -using namespace metaspace; +using metaspace::ChunkManager; +using metaspace::CommitLimiter; +using metaspace::MetaspaceContext; +using metaspace::MetaspaceReporter; +using metaspace::RunningCounters; +using metaspace::VirtualSpaceList; -MetaWord* last_allocated = 0; -size_t Metaspace::_compressed_class_space_size; -const MetaspaceTracer* Metaspace::_tracer = NULL; +size_t MetaspaceUtils::used_words() { + return RunningCounters::used_words(); +} -DEBUG_ONLY(bool Metaspace::_frozen = false;) +size_t MetaspaceUtils::used_words(Metaspace::MetadataType mdtype) { + return metaspace::is_class(mdtype) ? RunningCounters::used_words_class() : RunningCounters::used_words_nonclass(); +} -static const char* space_type_name(Metaspace::MetaspaceType t) { - const char* s = NULL; - switch (t) { - case Metaspace::StandardMetaspaceType: s = "Standard"; break; - case Metaspace::BootMetaspaceType: s = "Boot"; break; - case Metaspace::ClassMirrorHolderMetaspaceType: s = "ClassMirrorHolder"; break; - case Metaspace::ReflectionMetaspaceType: s = "Reflection"; break; - default: ShouldNotReachHere(); +size_t MetaspaceUtils::reserved_words() { + return RunningCounters::reserved_words(); +} + +size_t MetaspaceUtils::reserved_words(Metaspace::MetadataType mdtype) { + return metaspace::is_class(mdtype) ? RunningCounters::reserved_words_class() : RunningCounters::reserved_words_nonclass(); +} + +size_t MetaspaceUtils::committed_words() { + return RunningCounters::committed_words(); +} + +size_t MetaspaceUtils::committed_words(Metaspace::MetadataType mdtype) { + return metaspace::is_class(mdtype) ? RunningCounters::committed_words_class() : RunningCounters::committed_words_nonclass(); +} + + + +void MetaspaceUtils::print_metaspace_change(const metaspace::MetaspaceSizesSnapshot& pre_meta_values) { + const metaspace::MetaspaceSizesSnapshot meta_values; + + // We print used and committed since these are the most useful at-a-glance vitals for Metaspace: + // - used tells you how much memory is actually used for metadata + // - committed tells you how much memory is committed for the purpose of metadata + // The difference between those two would be waste, which can have various forms (freelists, + // unused parts of committed chunks etc) + // + // Left out is reserved, since this is not as exciting as the first two values: for class space, + // it is a constant (to uninformed users, often confusingly large). For non-class space, it would + // be interesting since free chunks can be uncommitted, but for now it is left out. + + if (Metaspace::using_class_space()) { + log_info(gc, metaspace)(HEAP_CHANGE_FORMAT" " + HEAP_CHANGE_FORMAT" " + HEAP_CHANGE_FORMAT, + HEAP_CHANGE_FORMAT_ARGS("Metaspace", + pre_meta_values.used(), + pre_meta_values.committed(), + meta_values.used(), + meta_values.committed()), + HEAP_CHANGE_FORMAT_ARGS("NonClass", + pre_meta_values.non_class_used(), + pre_meta_values.non_class_committed(), + meta_values.non_class_used(), + meta_values.non_class_committed()), + HEAP_CHANGE_FORMAT_ARGS("Class", + pre_meta_values.class_used(), + pre_meta_values.class_committed(), + meta_values.class_used(), + meta_values.class_committed())); + } else { + log_info(gc, metaspace)(HEAP_CHANGE_FORMAT, + HEAP_CHANGE_FORMAT_ARGS("Metaspace", + pre_meta_values.used(), + pre_meta_values.committed(), + meta_values.used(), + meta_values.committed())); } - return s; } +// This will print out a basic metaspace usage report but +// unlike print_report() is guaranteed not to lock or to walk the CLDG. +void MetaspaceUtils::print_basic_report(outputStream* out, size_t scale) { + MetaspaceReporter::print_basic_report(out, scale); +} + +// Prints a report about the current metaspace state. +// Optional parts can be enabled via flags. +// Function will walk the CLDG and will lock the expand lock; if that is not +// convenient, use print_basic_report() instead. +void MetaspaceUtils::print_report(outputStream* out, size_t scale) { + const int flags = + MetaspaceReporter::rf_show_loaders | + MetaspaceReporter::rf_break_down_by_chunktype | + MetaspaceReporter::rf_show_classes; + MetaspaceReporter::print_report(out, scale, flags); +} + +void MetaspaceUtils::print_on(outputStream* out) { + + // Used from all GCs. It first prints out totals, then, separately, the class space portion. + + out->print_cr(" Metaspace " + "used " SIZE_FORMAT "K, " + "committed " SIZE_FORMAT "K, " + "reserved " SIZE_FORMAT "K", + used_bytes()/K, + committed_bytes()/K, + reserved_bytes()/K); + + if (Metaspace::using_class_space()) { + const Metaspace::MetadataType ct = Metaspace::ClassType; + out->print_cr(" class space " + "used " SIZE_FORMAT "K, " + "committed " SIZE_FORMAT "K, " + "reserved " SIZE_FORMAT "K", + used_bytes(ct)/K, + committed_bytes(ct)/K, + reserved_bytes(ct)/K); + } +} + +#ifdef ASSERT +void MetaspaceUtils::verify(bool slow) { + if (Metaspace::initialized()) { + + // Verify non-class chunkmanager... + ChunkManager* cm = ChunkManager::chunkmanager_nonclass(); + cm->verify(slow); + + // ... and space list. + VirtualSpaceList* vsl = VirtualSpaceList::vslist_nonclass(); + vsl->verify(slow); + + if (Metaspace::using_class_space()) { + // If we use compressed class pointers, verify class chunkmanager... + cm = ChunkManager::chunkmanager_class(); + assert(cm != NULL, "Sanity"); + cm->verify(slow); + + // ... and class spacelist. + VirtualSpaceList* vsl = VirtualSpaceList::vslist_nonclass(); + assert(vsl != NULL, "Sanity"); + vsl->verify(slow); + } + + } +} +#endif + +////////////////////////////////7 +// MetaspaceGC methods + volatile size_t MetaspaceGC::_capacity_until_GC = 0; uint MetaspaceGC::_shrink_factor = 0; -// BlockFreelist methods - -// VirtualSpaceNode methods - -// MetaspaceGC methods - // VM_CollectForMetadataAllocation is the vm operation used to GC. // Within the VM operation after the GC the attempt to allocate the metadata // should succeed. If the GC did not free enough space for the metaspace @@ -351,643 +478,37 @@ } } -// MetaspaceUtils -size_t MetaspaceUtils::_capacity_words [Metaspace:: MetadataTypeCount] = {0, 0}; -size_t MetaspaceUtils::_overhead_words [Metaspace:: MetadataTypeCount] = {0, 0}; -volatile size_t MetaspaceUtils::_used_words [Metaspace:: MetadataTypeCount] = {0, 0}; -// Collect used metaspace statistics. This involves walking the CLDG. The resulting -// output will be the accumulated values for all live metaspaces. -// Note: method does not do any locking. -void MetaspaceUtils::collect_statistics(ClassLoaderMetaspaceStatistics* out) { - out->reset(); - ClassLoaderDataGraphMetaspaceIterator iter; - while (iter.repeat()) { - ClassLoaderMetaspace* msp = iter.get_next(); - if (msp != NULL) { - msp->add_to_statistics(out); - } - } + +////// Metaspace methods ///// + +const MetaspaceTracer* Metaspace::_tracer = NULL; + +DEBUG_ONLY(bool Metaspace::_frozen = false;) + +bool Metaspace::initialized() { + return metaspace::MetaspaceContext::context_nonclass() != NULL && + (using_class_space() ? metaspace::MetaspaceContext::context_class() != NULL : true); } -size_t MetaspaceUtils::free_in_vs_bytes(Metaspace::MetadataType mdtype) { - VirtualSpaceList* list = Metaspace::get_space_list(mdtype); - return list == NULL ? 0 : list->free_bytes(); -} - -size_t MetaspaceUtils::free_in_vs_bytes() { - return free_in_vs_bytes(Metaspace::ClassType) + free_in_vs_bytes(Metaspace::NonClassType); -} - -static void inc_stat_nonatomically(size_t* pstat, size_t words) { - assert_lock_strong(MetaspaceExpand_lock); - (*pstat) += words; -} - -static void dec_stat_nonatomically(size_t* pstat, size_t words) { - assert_lock_strong(MetaspaceExpand_lock); - const size_t size_now = *pstat; - assert(size_now >= words, "About to decrement counter below zero " - "(current value: " SIZE_FORMAT ", decrement value: " SIZE_FORMAT ".", - size_now, words); - *pstat = size_now - words; -} - -static void inc_stat_atomically(volatile size_t* pstat, size_t words) { - Atomic::add(pstat, words); -} - -static void dec_stat_atomically(volatile size_t* pstat, size_t words) { - const size_t size_now = *pstat; - assert(size_now >= words, "About to decrement counter below zero " - "(current value: " SIZE_FORMAT ", decrement value: " SIZE_FORMAT ".", - size_now, words); - Atomic::sub(pstat, words); -} - -void MetaspaceUtils::dec_capacity(Metaspace::MetadataType mdtype, size_t words) { - dec_stat_nonatomically(&_capacity_words[mdtype], words); -} -void MetaspaceUtils::inc_capacity(Metaspace::MetadataType mdtype, size_t words) { - inc_stat_nonatomically(&_capacity_words[mdtype], words); -} -void MetaspaceUtils::dec_used(Metaspace::MetadataType mdtype, size_t words) { - dec_stat_atomically(&_used_words[mdtype], words); -} -void MetaspaceUtils::inc_used(Metaspace::MetadataType mdtype, size_t words) { - inc_stat_atomically(&_used_words[mdtype], words); -} -void MetaspaceUtils::dec_overhead(Metaspace::MetadataType mdtype, size_t words) { - dec_stat_nonatomically(&_overhead_words[mdtype], words); -} -void MetaspaceUtils::inc_overhead(Metaspace::MetadataType mdtype, size_t words) { - inc_stat_nonatomically(&_overhead_words[mdtype], words); -} - -size_t MetaspaceUtils::reserved_bytes(Metaspace::MetadataType mdtype) { - VirtualSpaceList* list = Metaspace::get_space_list(mdtype); - return list == NULL ? 0 : list->reserved_bytes(); -} - -size_t MetaspaceUtils::committed_bytes(Metaspace::MetadataType mdtype) { - VirtualSpaceList* list = Metaspace::get_space_list(mdtype); - return list == NULL ? 0 : list->committed_bytes(); -} - -size_t MetaspaceUtils::min_chunk_size_words() { return Metaspace::first_chunk_word_size(); } - -size_t MetaspaceUtils::free_chunks_total_words(Metaspace::MetadataType mdtype) { - ChunkManager* chunk_manager = Metaspace::get_chunk_manager(mdtype); - if (chunk_manager == NULL) { - return 0; - } - return chunk_manager->free_chunks_total_words(); -} - -size_t MetaspaceUtils::free_chunks_total_bytes(Metaspace::MetadataType mdtype) { - return free_chunks_total_words(mdtype) * BytesPerWord; -} - -size_t MetaspaceUtils::free_chunks_total_words() { - return free_chunks_total_words(Metaspace::ClassType) + - free_chunks_total_words(Metaspace::NonClassType); -} - -size_t MetaspaceUtils::free_chunks_total_bytes() { - return free_chunks_total_words() * BytesPerWord; -} - -bool MetaspaceUtils::has_chunk_free_list(Metaspace::MetadataType mdtype) { - return Metaspace::get_chunk_manager(mdtype) != NULL; -} - -MetaspaceChunkFreeListSummary MetaspaceUtils::chunk_free_list_summary(Metaspace::MetadataType mdtype) { - if (!has_chunk_free_list(mdtype)) { - return MetaspaceChunkFreeListSummary(); - } - - const ChunkManager* cm = Metaspace::get_chunk_manager(mdtype); - return cm->chunk_free_list_summary(); -} - -void MetaspaceUtils::print_metaspace_change(const metaspace::MetaspaceSizesSnapshot& pre_meta_values) { - const metaspace::MetaspaceSizesSnapshot meta_values; - - if (Metaspace::using_class_space()) { - log_info(gc, metaspace)(HEAP_CHANGE_FORMAT" " - HEAP_CHANGE_FORMAT" " - HEAP_CHANGE_FORMAT, - HEAP_CHANGE_FORMAT_ARGS("Metaspace", - pre_meta_values.used(), - pre_meta_values.committed(), - meta_values.used(), - meta_values.committed()), - HEAP_CHANGE_FORMAT_ARGS("NonClass", - pre_meta_values.non_class_used(), - pre_meta_values.non_class_committed(), - meta_values.non_class_used(), - meta_values.non_class_committed()), - HEAP_CHANGE_FORMAT_ARGS("Class", - pre_meta_values.class_used(), - pre_meta_values.class_committed(), - meta_values.class_used(), - meta_values.class_committed())); - } else { - log_info(gc, metaspace)(HEAP_CHANGE_FORMAT, - HEAP_CHANGE_FORMAT_ARGS("Metaspace", - pre_meta_values.used(), - pre_meta_values.committed(), - meta_values.used(), - meta_values.committed())); - } -} - -void MetaspaceUtils::print_on(outputStream* out) { - Metaspace::MetadataType nct = Metaspace::NonClassType; - - out->print_cr(" Metaspace " - "used " SIZE_FORMAT "K, " - "capacity " SIZE_FORMAT "K, " - "committed " SIZE_FORMAT "K, " - "reserved " SIZE_FORMAT "K", - used_bytes()/K, - capacity_bytes()/K, - committed_bytes()/K, - reserved_bytes()/K); - - if (Metaspace::using_class_space()) { - Metaspace::MetadataType ct = Metaspace::ClassType; - out->print_cr(" class space " - "used " SIZE_FORMAT "K, " - "capacity " SIZE_FORMAT "K, " - "committed " SIZE_FORMAT "K, " - "reserved " SIZE_FORMAT "K", - used_bytes(ct)/K, - capacity_bytes(ct)/K, - committed_bytes(ct)/K, - reserved_bytes(ct)/K); - } -} - - -void MetaspaceUtils::print_vs(outputStream* out, size_t scale) { - const size_t reserved_nonclass_words = reserved_bytes(Metaspace::NonClassType) / sizeof(MetaWord); - const size_t committed_nonclass_words = committed_bytes(Metaspace::NonClassType) / sizeof(MetaWord); - { - if (Metaspace::using_class_space()) { - out->print(" Non-class space: "); - } - print_scaled_words(out, reserved_nonclass_words, scale, 7); - out->print(" reserved, "); - print_scaled_words_and_percentage(out, committed_nonclass_words, reserved_nonclass_words, scale, 7); - out->print_cr(" committed "); - - if (Metaspace::using_class_space()) { - const size_t reserved_class_words = reserved_bytes(Metaspace::ClassType) / sizeof(MetaWord); - const size_t committed_class_words = committed_bytes(Metaspace::ClassType) / sizeof(MetaWord); - out->print(" Class space: "); - print_scaled_words(out, reserved_class_words, scale, 7); - out->print(" reserved, "); - print_scaled_words_and_percentage(out, committed_class_words, reserved_class_words, scale, 7); - out->print_cr(" committed "); - - const size_t reserved_words = reserved_nonclass_words + reserved_class_words; - const size_t committed_words = committed_nonclass_words + committed_class_words; - out->print(" Both: "); - print_scaled_words(out, reserved_words, scale, 7); - out->print(" reserved, "); - print_scaled_words_and_percentage(out, committed_words, reserved_words, scale, 7); - out->print_cr(" committed "); - } - } -} - -static void print_basic_switches(outputStream* out, size_t scale) { - out->print("MaxMetaspaceSize: "); - if (MaxMetaspaceSize >= (max_uintx) - (2 * os::vm_page_size())) { - // aka "very big". Default is max_uintx, but due to rounding in arg parsing the real - // value is smaller. - out->print("unlimited"); - } else { - print_human_readable_size(out, MaxMetaspaceSize, scale); - } - out->cr(); - if (Metaspace::using_class_space()) { - out->print("CompressedClassSpaceSize: "); - print_human_readable_size(out, CompressedClassSpaceSize, scale); - } - out->cr(); -} - -// This will print out a basic metaspace usage report but -// unlike print_report() is guaranteed not to lock or to walk the CLDG. -void MetaspaceUtils::print_basic_report(outputStream* out, size_t scale) { - - if (!Metaspace::initialized()) { - out->print_cr("Metaspace not yet initialized."); - return; - } - - out->cr(); - out->print_cr("Usage:"); - - if (Metaspace::using_class_space()) { - out->print(" Non-class: "); - } - - // In its most basic form, we do not require walking the CLDG. Instead, just print the running totals from - // MetaspaceUtils. - const size_t cap_nc = MetaspaceUtils::capacity_words(Metaspace::NonClassType); - const size_t overhead_nc = MetaspaceUtils::overhead_words(Metaspace::NonClassType); - const size_t used_nc = MetaspaceUtils::used_words(Metaspace::NonClassType); - const size_t free_and_waste_nc = cap_nc - overhead_nc - used_nc; - - print_scaled_words(out, cap_nc, scale, 5); - out->print(" capacity, "); - print_scaled_words_and_percentage(out, used_nc, cap_nc, scale, 5); - out->print(" used, "); - print_scaled_words_and_percentage(out, free_and_waste_nc, cap_nc, scale, 5); - out->print(" free+waste, "); - print_scaled_words_and_percentage(out, overhead_nc, cap_nc, scale, 5); - out->print(" overhead. "); - out->cr(); - - if (Metaspace::using_class_space()) { - const size_t cap_c = MetaspaceUtils::capacity_words(Metaspace::ClassType); - const size_t overhead_c = MetaspaceUtils::overhead_words(Metaspace::ClassType); - const size_t used_c = MetaspaceUtils::used_words(Metaspace::ClassType); - const size_t free_and_waste_c = cap_c - overhead_c - used_c; - out->print(" Class: "); - print_scaled_words(out, cap_c, scale, 5); - out->print(" capacity, "); - print_scaled_words_and_percentage(out, used_c, cap_c, scale, 5); - out->print(" used, "); - print_scaled_words_and_percentage(out, free_and_waste_c, cap_c, scale, 5); - out->print(" free+waste, "); - print_scaled_words_and_percentage(out, overhead_c, cap_c, scale, 5); - out->print(" overhead. "); - out->cr(); - - out->print(" Both: "); - const size_t cap = cap_nc + cap_c; - - print_scaled_words(out, cap, scale, 5); - out->print(" capacity, "); - print_scaled_words_and_percentage(out, used_nc + used_c, cap, scale, 5); - out->print(" used, "); - print_scaled_words_and_percentage(out, free_and_waste_nc + free_and_waste_c, cap, scale, 5); - out->print(" free+waste, "); - print_scaled_words_and_percentage(out, overhead_nc + overhead_c, cap, scale, 5); - out->print(" overhead. "); - out->cr(); - } - - out->cr(); - out->print_cr("Virtual space:"); - - print_vs(out, scale); - - out->cr(); - out->print_cr("Chunk freelists:"); - - if (Metaspace::using_class_space()) { - out->print(" Non-Class: "); - } - print_human_readable_size(out, Metaspace::chunk_manager_metadata()->free_chunks_total_bytes(), scale); - out->cr(); - if (Metaspace::using_class_space()) { - out->print(" Class: "); - print_human_readable_size(out, Metaspace::chunk_manager_class()->free_chunks_total_bytes(), scale); - out->cr(); - out->print(" Both: "); - print_human_readable_size(out, Metaspace::chunk_manager_class()->free_chunks_total_bytes() + - Metaspace::chunk_manager_metadata()->free_chunks_total_bytes(), scale); - out->cr(); - } - - out->cr(); - - // Print basic settings - print_basic_switches(out, scale); - - out->cr(); - -} - -void MetaspaceUtils::print_report(outputStream* out, size_t scale, int flags) { - - if (!Metaspace::initialized()) { - out->print_cr("Metaspace not yet initialized."); - return; - } - - const bool print_loaders = (flags & rf_show_loaders) > 0; - const bool print_classes = (flags & rf_show_classes) > 0; - const bool print_by_chunktype = (flags & rf_break_down_by_chunktype) > 0; - const bool print_by_spacetype = (flags & rf_break_down_by_spacetype) > 0; - - // Some report options require walking the class loader data graph. - PrintCLDMetaspaceInfoClosure cl(out, scale, print_loaders, print_classes, print_by_chunktype); - if (print_loaders) { - out->cr(); - out->print_cr("Usage per loader:"); - out->cr(); - } - - ClassLoaderDataGraph::loaded_cld_do(&cl); // collect data and optionally print - - // Print totals, broken up by space type. - if (print_by_spacetype) { - out->cr(); - out->print_cr("Usage per space type:"); - out->cr(); - for (int space_type = (int)Metaspace::ZeroMetaspaceType; - space_type < (int)Metaspace::MetaspaceTypeCount; space_type ++) - { - uintx num_loaders = cl._num_loaders_by_spacetype[space_type]; - uintx num_classes = cl._num_classes_by_spacetype[space_type]; - out->print("%s - " UINTX_FORMAT " %s", - space_type_name((Metaspace::MetaspaceType)space_type), - num_loaders, loaders_plural(num_loaders)); - if (num_classes > 0) { - out->print(", "); - print_number_of_classes(out, num_classes, cl._num_classes_shared_by_spacetype[space_type]); - out->print(":"); - cl._stats_by_spacetype[space_type].print_on(out, scale, print_by_chunktype); - } else { - out->print("."); - out->cr(); - } - out->cr(); - } - } - - // Print totals for in-use data: - out->cr(); - { - uintx num_loaders = cl._num_loaders; - out->print("Total Usage - " UINTX_FORMAT " %s, ", - num_loaders, loaders_plural(num_loaders)); - print_number_of_classes(out, cl._num_classes, cl._num_classes_shared); - out->print(":"); - cl._stats_total.print_on(out, scale, print_by_chunktype); - out->cr(); - } - - // -- Print Virtual space. - out->cr(); - out->print_cr("Virtual space:"); - - print_vs(out, scale); - - // -- Print VirtualSpaceList details. - if ((flags & rf_show_vslist) > 0) { - out->cr(); - out->print_cr("Virtual space list%s:", Metaspace::using_class_space() ? "s" : ""); - - if (Metaspace::using_class_space()) { - out->print_cr(" Non-Class:"); - } - Metaspace::space_list()->print_on(out, scale); - if (Metaspace::using_class_space()) { - out->print_cr(" Class:"); - Metaspace::class_space_list()->print_on(out, scale); - } - } - out->cr(); - - // -- Print VirtualSpaceList map. - if ((flags & rf_show_vsmap) > 0) { - out->cr(); - out->print_cr("Virtual space map:"); - - if (Metaspace::using_class_space()) { - out->print_cr(" Non-Class:"); - } - Metaspace::space_list()->print_map(out); - if (Metaspace::using_class_space()) { - out->print_cr(" Class:"); - Metaspace::class_space_list()->print_map(out); - } - } - out->cr(); - - // -- Print Freelists (ChunkManager) details - out->cr(); - out->print_cr("Chunk freelist%s:", Metaspace::using_class_space() ? "s" : ""); - - ChunkManagerStatistics non_class_cm_stat; - Metaspace::chunk_manager_metadata()->collect_statistics(&non_class_cm_stat); - - if (Metaspace::using_class_space()) { - out->print_cr(" Non-Class:"); - } - non_class_cm_stat.print_on(out, scale); - - if (Metaspace::using_class_space()) { - ChunkManagerStatistics class_cm_stat; - Metaspace::chunk_manager_class()->collect_statistics(&class_cm_stat); - out->print_cr(" Class:"); - class_cm_stat.print_on(out, scale); - } - - // As a convenience, print a summary of common waste. - out->cr(); - out->print("Waste "); - // For all wastages, print percentages from total. As total use the total size of memory committed for metaspace. - const size_t committed_words = committed_bytes() / BytesPerWord; - - out->print("(percentages refer to total committed size "); - print_scaled_words(out, committed_words, scale); - out->print_cr("):"); - - // Print space committed but not yet used by any class loader - const size_t unused_words_in_vs = MetaspaceUtils::free_in_vs_bytes() / BytesPerWord; - out->print(" Committed unused: "); - print_scaled_words_and_percentage(out, unused_words_in_vs, committed_words, scale, 6); - out->cr(); - - // Print waste for in-use chunks. - UsedChunksStatistics ucs_nonclass = cl._stats_total.nonclass_sm_stats().totals(); - UsedChunksStatistics ucs_class = cl._stats_total.class_sm_stats().totals(); - UsedChunksStatistics ucs_all; - ucs_all.add(ucs_nonclass); - ucs_all.add(ucs_class); - - out->print(" Waste in chunks in use: "); - print_scaled_words_and_percentage(out, ucs_all.waste(), committed_words, scale, 6); - out->cr(); - out->print(" Free in chunks in use: "); - print_scaled_words_and_percentage(out, ucs_all.free(), committed_words, scale, 6); - out->cr(); - out->print(" Overhead in chunks in use: "); - print_scaled_words_and_percentage(out, ucs_all.overhead(), committed_words, scale, 6); - out->cr(); - - // Print waste in free chunks. - const size_t total_capacity_in_free_chunks = - Metaspace::chunk_manager_metadata()->free_chunks_total_words() + - (Metaspace::using_class_space() ? Metaspace::chunk_manager_class()->free_chunks_total_words() : 0); - out->print(" In free chunks: "); - print_scaled_words_and_percentage(out, total_capacity_in_free_chunks, committed_words, scale, 6); - out->cr(); - - // Print waste in deallocated blocks. - const uintx free_blocks_num = - cl._stats_total.nonclass_sm_stats().free_blocks_num() + - cl._stats_total.class_sm_stats().free_blocks_num(); - const size_t free_blocks_cap_words = - cl._stats_total.nonclass_sm_stats().free_blocks_cap_words() + - cl._stats_total.class_sm_stats().free_blocks_cap_words(); - out->print("Deallocated from chunks in use: "); - print_scaled_words_and_percentage(out, free_blocks_cap_words, committed_words, scale, 6); - out->print(" (" UINTX_FORMAT " blocks)", free_blocks_num); - out->cr(); - - // Print total waste. - const size_t total_waste = ucs_all.waste() + ucs_all.free() + ucs_all.overhead() + total_capacity_in_free_chunks - + free_blocks_cap_words + unused_words_in_vs; - out->print(" -total-: "); - print_scaled_words_and_percentage(out, total_waste, committed_words, scale, 6); - out->cr(); - - // Print internal statistics -#ifdef ASSERT - out->cr(); - out->cr(); - out->print_cr("Internal statistics:"); - out->cr(); - out->print_cr("Number of allocations: " UINTX_FORMAT ".", g_internal_statistics.num_allocs); - out->print_cr("Number of space births: " UINTX_FORMAT ".", g_internal_statistics.num_metaspace_births); - out->print_cr("Number of space deaths: " UINTX_FORMAT ".", g_internal_statistics.num_metaspace_deaths); - out->print_cr("Number of virtual space node births: " UINTX_FORMAT ".", g_internal_statistics.num_vsnodes_created); - out->print_cr("Number of virtual space node deaths: " UINTX_FORMAT ".", g_internal_statistics.num_vsnodes_purged); - out->print_cr("Number of times virtual space nodes were expanded: " UINTX_FORMAT ".", g_internal_statistics.num_committed_space_expanded); - out->print_cr("Number of deallocations: " UINTX_FORMAT " (" UINTX_FORMAT " external).", g_internal_statistics.num_deallocs, g_internal_statistics.num_external_deallocs); - out->print_cr("Allocations from deallocated blocks: " UINTX_FORMAT ".", g_internal_statistics.num_allocs_from_deallocated_blocks); - out->print_cr("Number of chunks added to freelist: " UINTX_FORMAT ".", - g_internal_statistics.num_chunks_added_to_freelist); - out->print_cr("Number of chunks removed from freelist: " UINTX_FORMAT ".", - g_internal_statistics.num_chunks_removed_from_freelist); - out->print_cr("Number of chunk merges: " UINTX_FORMAT ", split-ups: " UINTX_FORMAT ".", - g_internal_statistics.num_chunk_merges, g_internal_statistics.num_chunk_splits); - - out->cr(); -#endif - - // Print some interesting settings - out->cr(); - out->cr(); - print_basic_switches(out, scale); - - out->cr(); - out->print("InitialBootClassLoaderMetaspaceSize: "); - print_human_readable_size(out, InitialBootClassLoaderMetaspaceSize, scale); - - out->cr(); - out->cr(); - -} // MetaspaceUtils::print_report() - -// Prints an ASCII representation of the given space. -void MetaspaceUtils::print_metaspace_map(outputStream* out, Metaspace::MetadataType mdtype) { - MutexLocker cl(MetaspaceExpand_lock, Mutex::_no_safepoint_check_flag); - const bool for_class = mdtype == Metaspace::ClassType ? true : false; - VirtualSpaceList* const vsl = for_class ? Metaspace::class_space_list() : Metaspace::space_list(); - if (vsl != NULL) { - if (for_class) { - if (!Metaspace::using_class_space()) { - out->print_cr("No Class Space."); - return; - } - out->print_raw("---- Metaspace Map (Class Space) ----"); - } else { - out->print_raw("---- Metaspace Map (Non-Class Space) ----"); - } - // Print legend: - out->cr(); - out->print_cr("Chunk Types (uppercase chunks are in use): x-specialized, s-small, m-medium, h-humongous."); - out->cr(); - VirtualSpaceList* const vsl = for_class ? Metaspace::class_space_list() : Metaspace::space_list(); - vsl->print_map(out); - out->cr(); - } -} - -void MetaspaceUtils::verify_free_chunks() { -#ifdef ASSERT - Metaspace::chunk_manager_metadata()->verify(false); - if (Metaspace::using_class_space()) { - Metaspace::chunk_manager_class()->verify(false); - } -#endif -} - -void MetaspaceUtils::verify_metrics() { -#ifdef ASSERT - // Please note: there are time windows where the internal counters are out of sync with - // reality. For example, when a newly created ClassLoaderMetaspace creates its first chunk - - // the ClassLoaderMetaspace is not yet attached to its ClassLoaderData object and hence will - // not be counted when iterating the CLDG. So be careful when you call this method. - ClassLoaderMetaspaceStatistics total_stat; - collect_statistics(&total_stat); - UsedChunksStatistics nonclass_chunk_stat = total_stat.nonclass_sm_stats().totals(); - UsedChunksStatistics class_chunk_stat = total_stat.class_sm_stats().totals(); - - bool mismatch = false; - for (int i = 0; i < Metaspace::MetadataTypeCount; i ++) { - Metaspace::MetadataType mdtype = (Metaspace::MetadataType)i; - UsedChunksStatistics chunk_stat = total_stat.sm_stats(mdtype).totals(); - if (capacity_words(mdtype) != chunk_stat.cap() || - used_words(mdtype) != chunk_stat.used() || - overhead_words(mdtype) != chunk_stat.overhead()) { - mismatch = true; - tty->print_cr("MetaspaceUtils::verify_metrics: counter mismatch for mdtype=%u:", mdtype); - tty->print_cr("Expected cap " SIZE_FORMAT ", used " SIZE_FORMAT ", overhead " SIZE_FORMAT ".", - capacity_words(mdtype), used_words(mdtype), overhead_words(mdtype)); - tty->print_cr("Got cap " SIZE_FORMAT ", used " SIZE_FORMAT ", overhead " SIZE_FORMAT ".", - chunk_stat.cap(), chunk_stat.used(), chunk_stat.overhead()); - tty->flush(); - } - } - assert(mismatch == false, "MetaspaceUtils::verify_metrics: counter mismatch."); -#endif -} - -// Metaspace methods - -size_t Metaspace::_first_chunk_word_size = 0; -size_t Metaspace::_first_class_chunk_word_size = 0; - -size_t Metaspace::_commit_alignment = 0; -size_t Metaspace::_reserve_alignment = 0; - -VirtualSpaceList* Metaspace::_space_list = NULL; -VirtualSpaceList* Metaspace::_class_space_list = NULL; - -ChunkManager* Metaspace::_chunk_manager_metadata = NULL; -ChunkManager* Metaspace::_chunk_manager_class = NULL; - -bool Metaspace::_initialized = false; - -#define VIRTUALSPACEMULTIPLIER 2 - #ifdef _LP64 void Metaspace::print_compressed_class_space(outputStream* st) { - if (_class_space_list != NULL) { - address base = (address)_class_space_list->current_virtual_space()->bottom(); - address top = base + compressed_class_space_size(); - st->print("Compressed class space mapped at: " PTR_FORMAT "-" PTR_FORMAT ", size: " SIZE_FORMAT, - p2i(base), p2i(top), top - base); + if (VirtualSpaceList::vslist_class() != NULL) { + MetaWord* base = VirtualSpaceList::vslist_class()->base_of_first_node(); + size_t size = VirtualSpaceList::vslist_class()->word_size_of_first_node(); + MetaWord* top = base + size; + st->print("Compressed class space mapped at: " PTR_FORMAT "-" PTR_FORMAT ", reserved size: " SIZE_FORMAT, + p2i(base), p2i(top), (top - base) * BytesPerWord); st->cr(); } } // Given a prereserved space, use that to set up the compressed class space list. void Metaspace::initialize_class_space(ReservedSpace rs) { + assert(rs.size() >= CompressedClassSpaceSize, + SIZE_FORMAT " != " SIZE_FORMAT, rs.size(), CompressedClassSpaceSize); assert(using_class_space(), "Must be using class space"); - assert(_class_space_list == NULL && _chunk_manager_class == NULL, "Only call once"); assert(rs.size() == CompressedClassSpaceSize, SIZE_FORMAT " != " SIZE_FORMAT, rs.size(), CompressedClassSpaceSize); @@ -995,18 +516,18 @@ is_aligned(rs.size(), Metaspace::reserve_alignment()), "wrong alignment"); - _class_space_list = new VirtualSpaceList(rs); - _chunk_manager_class = new ChunkManager(true/*is_class*/); + MetaspaceContext::initialize_class_space_context(rs); // This does currently not work because rs may be the result of a split // operation and NMT seems not to be able to handle splits. // Will be fixed with JDK-8243535. // MemTracker::record_virtual_memory_type((address)rs.base(), mtClass); - if (!_class_space_list->initialization_succeeded()) { - vm_exit_during_initialization("Failed to setup compressed class space virtual space list."); - } +} +// Returns true if class space has been setup (initialize_class_space). +bool Metaspace::class_space_is_initialized() { + return MetaspaceContext::context_class() != NULL; } // Reserve a range of memory at an address suitable for en/decoding narrow @@ -1063,72 +584,90 @@ #endif // _LP64 +size_t Metaspace::reserve_alignment_words() { + return metaspace::Settings::virtual_space_node_reserve_alignment_words(); +} + +size_t Metaspace::commit_alignment_words() { + return metaspace::Settings::commit_granule_words(); +} + void Metaspace::ergo_initialize() { - if (DumpSharedSpaces) { - // Using large pages when dumping the shared archive is currently not implemented. - FLAG_SET_ERGO(UseLargePagesInMetaspace, false); + + // Must happen before using any setting from Settings::--- + metaspace::Settings::ergo_initialize(); + + // MaxMetaspaceSize and CompressedClassSpaceSize: + // + // MaxMetaspaceSize is the maximum size, in bytes, of memory we are allowed + // to commit for the Metaspace. + // It is just a number; a limit we compare against before committing. It + // does not have to be aligned to anything. + // It gets used as compare value in class CommitLimiter. + // It is set to max_uintx in globals.hpp by default, so by default it does + // not limit anything. + // + // CompressedClassSpaceSize is the size, in bytes, of the address range we + // pre-reserve for the compressed class space (if we use class space). + // This size has to be aligned to the metaspace reserve alignment (to the + // size of a root chunk). It gets aligned up from whatever value the caller + // gave us to the next multiple of root chunk size. + // + // Note: Strictly speaking MaxMetaspaceSize and CompressedClassSpaceSize have + // very little to do with each other. The notion often encountered: + // MaxMetaspaceSize = CompressedClassSpaceSize + + // is subtly wrong: MaxMetaspaceSize can besmaller than CompressedClassSpaceSize, + // in which case we just would not be able to fully commit the class space range. + // + // We still adjust CompressedClassSpaceSize to reasonable limits, mainly to + // save on reserved space, and to make ergnonomics less confusing. + + // (aligned just for cleanliness:) + MaxMetaspaceSize = MAX2(align_down(MaxMetaspaceSize, commit_alignment()), commit_alignment()); + + if (UseCompressedClassPointers) { + // Let CCS size not be larger than 80% of MaxMetaspaceSize. Note that is + // grossly over-dimensioned for most usage scenarios; typical ratio of + // class space : non class space usage is about 1:6. With many small classes, + // it can get as low as 1:2. It is not a big deal though since ccs is only + // reserved and will be committed on demand only. + size_t max_ccs_size = MaxMetaspaceSize * 0.8; + size_t adjusted_ccs_size = MIN2(CompressedClassSpaceSize, max_ccs_size); + + // CCS must be aligned to root chunk size, and be at least the size of one + // root chunk. + adjusted_ccs_size = align_up(adjusted_ccs_size, reserve_alignment()); + adjusted_ccs_size = MAX2(adjusted_ccs_size, reserve_alignment()); + + // Note: re-adjusting may have us left with a CompressedClassSpaceSize + // larger than MaxMetaspaceSize for very small values of MaxMetaspaceSize. + // Lets just live with that, its not a big deal. + + if (adjusted_ccs_size != CompressedClassSpaceSize) { + FLAG_SET_ERGO(CompressedClassSpaceSize, adjusted_ccs_size); + log_info(metaspace)("Setting CompressedClassSpaceSize to " SIZE_FORMAT ".", + CompressedClassSpaceSize); + } } - size_t page_size = os::vm_page_size(); - if (UseLargePages && UseLargePagesInMetaspace) { - page_size = os::large_page_size(); - } - - _commit_alignment = page_size; - _reserve_alignment = MAX2(page_size, (size_t)os::vm_allocation_granularity()); - - // The upcoming Metaspace rewrite will impose a higher alignment granularity. - // To prepare for that and to catch/prevent any misuse of Metaspace alignment - // which may creep in, up the alignment a bit. - if (_reserve_alignment == 4 * K) { - _reserve_alignment *= 4; - } - - // Do not use FLAG_SET_ERGO to update MaxMetaspaceSize, since this will - // override if MaxMetaspaceSize was set on the command line or not. - // This information is needed later to conform to the specification of the - // java.lang.management.MemoryUsage API. - // - // Ideally, we would be able to set the default value of MaxMetaspaceSize in - // globals.hpp to the aligned value, but this is not possible, since the - // alignment depends on other flags being parsed. - MaxMetaspaceSize = align_down_bounded(MaxMetaspaceSize, _reserve_alignment); - + // Set MetaspaceSize, MinMetaspaceExpansion and MaxMetaspaceExpansion if (MetaspaceSize > MaxMetaspaceSize) { MetaspaceSize = MaxMetaspaceSize; } - MetaspaceSize = align_down_bounded(MetaspaceSize, _commit_alignment); + MetaspaceSize = align_down_bounded(MetaspaceSize, commit_alignment()); assert(MetaspaceSize <= MaxMetaspaceSize, "MetaspaceSize should be limited by MaxMetaspaceSize"); - MinMetaspaceExpansion = align_down_bounded(MinMetaspaceExpansion, _commit_alignment); - MaxMetaspaceExpansion = align_down_bounded(MaxMetaspaceExpansion, _commit_alignment); + MinMetaspaceExpansion = align_down_bounded(MinMetaspaceExpansion, commit_alignment()); + MaxMetaspaceExpansion = align_down_bounded(MaxMetaspaceExpansion, commit_alignment()); - CompressedClassSpaceSize = align_down_bounded(CompressedClassSpaceSize, _reserve_alignment); - - // Initial virtual space size will be calculated at global_initialize() - size_t min_metaspace_sz = - VIRTUALSPACEMULTIPLIER * InitialBootClassLoaderMetaspaceSize; - if (UseCompressedClassPointers) { - if ((min_metaspace_sz + CompressedClassSpaceSize) > MaxMetaspaceSize) { - if (min_metaspace_sz >= MaxMetaspaceSize) { - vm_exit_during_initialization("MaxMetaspaceSize is too small."); - } else { - FLAG_SET_ERGO(CompressedClassSpaceSize, - MaxMetaspaceSize - min_metaspace_sz); - } - } - } else if (min_metaspace_sz >= MaxMetaspaceSize) { - FLAG_SET_ERGO(InitialBootClassLoaderMetaspaceSize, - min_metaspace_sz); - } - - set_compressed_class_space_size(CompressedClassSpaceSize); } void Metaspace::global_initialize() { - MetaspaceGC::initialize(); + MetaspaceGC::initialize(); // <- since we do not prealloc init chunks anymore is this still needed? + + metaspace::ChunkHeaderPool::initialize(); // If UseCompressedClassPointers=1, we have two cases: // a) if CDS is active (either dump time or runtime), it will create the ccs @@ -1188,7 +727,7 @@ if (!rs.is_reserved()) { vm_exit_during_initialization( err_msg("Could not allocate compressed class space: " SIZE_FORMAT " bytes", - compressed_class_space_size())); + CompressedClassSpaceSize)); } // Initialize space @@ -1200,31 +739,24 @@ #endif - // Initialize these before initializing the VirtualSpaceList - _first_chunk_word_size = InitialBootClassLoaderMetaspaceSize / BytesPerWord; - _first_chunk_word_size = align_word_size_up(_first_chunk_word_size); - // Make the first class chunk bigger than a medium chunk so it's not put - // on the medium chunk list. The next chunk will be small and progress - // from there. This size calculated by -version. - _first_class_chunk_word_size = MIN2((size_t)MediumChunk*6, - (CompressedClassSpaceSize/BytesPerWord)*2); - _first_class_chunk_word_size = align_word_size_up(_first_class_chunk_word_size); - // Arbitrarily set the initial virtual space to a multiple - // of the boot class loader size. - size_t word_size = VIRTUALSPACEMULTIPLIER * _first_chunk_word_size; - word_size = align_up(word_size, Metaspace::reserve_alignment_words()); - - // Initialize the list of virtual spaces. - _space_list = new VirtualSpaceList(word_size); - _chunk_manager_metadata = new ChunkManager(false/*metaspace*/); - - if (!_space_list->initialization_succeeded()) { - vm_exit_during_initialization("Unable to setup metadata virtual space list.", NULL); - } + // Initialize non-class virtual space list, and its chunk manager: + MetaspaceContext::initialize_nonclass_space_context(); _tracer = new MetaspaceTracer(); - _initialized = true; + // We must prevent the very first address of the ccs from being used to store + // metadata, since that address would translate to a narrow pointer of 0, and the + // VM does not distinguish between "narrow 0 as in NULL" and "narrow 0 as in start + // of ccs". + // Before Elastic Metaspace that did not happen due to the fact that every Metachunk + // had a header and therefore could not allocate anything at offset 0. +#ifdef _LP64 + if (using_class_space()) { + // The simplest way to fix this is to allocate a tiny dummy chunk right at the + // start of ccs and do not use it for anything. + MetaspaceContext::context_class()->cm()->get_chunk(metaspace::chunklevel::HIGHEST_CHUNK_LEVEL); + } +#endif #ifdef _LP64 if (UseCompressedClassPointers) { @@ -1246,23 +778,15 @@ MetaspaceGC::post_initialize(); } -void Metaspace::verify_global_initialization() { - assert(space_list() != NULL, "Metadata VirtualSpaceList has not been initialized"); - assert(chunk_manager_metadata() != NULL, "Metadata ChunkManager has not been initialized"); - - if (using_class_space()) { - assert(class_space_list() != NULL, "Class VirtualSpaceList has not been initialized"); - assert(chunk_manager_class() != NULL, "Class ChunkManager has not been initialized"); - } -} - -size_t Metaspace::align_word_size_up(size_t word_size) { - size_t byte_size = word_size * wordSize; - return ReservedSpace::allocation_align_size_up(byte_size) / wordSize; +size_t Metaspace::max_allocation_word_size() { + const size_t max_overhead_words = metaspace::get_raw_word_size_for_requested_word_size(1); + return metaspace::chunklevel::MAX_CHUNK_WORD_SIZE - max_overhead_words; } MetaWord* Metaspace::allocate(ClassLoaderData* loader_data, size_t word_size, MetaspaceObj::Type type, TRAPS) { + assert(word_size <= Metaspace::max_allocation_word_size(), + "allocation size too large (" SIZE_FORMAT ")", word_size); assert(!_frozen, "sanity"); assert(!(DumpSharedSpaces && THREAD->is_VM_thread()), "sanity"); @@ -1274,7 +798,7 @@ assert(loader_data != NULL, "Should never pass around a NULL loader_data. " "ClassLoaderData::the_null_class_loader_data() should have been used."); - MetadataType mdtype = (type == MetaspaceObj::ClassType) ? ClassType : NonClassType; + Metaspace::MetadataType mdtype = (type == MetaspaceObj::ClassType) ? Metaspace::ClassType : Metaspace::NonClassType; // Try to allocate metadata. MetaWord* result = loader_data->metaspace_non_null()->allocate(word_size, mdtype); @@ -1307,6 +831,8 @@ // Zero initialize. Copy::fill_to_words((HeapWord*)result, word_size, 0); + log_trace(metaspace)("Metaspace::allocate: type %d return " PTR_FORMAT ".", (int)type, p2i(result)); + return result; } @@ -1317,7 +843,7 @@ Log(gc, metaspace, freelist, oom) log; if (log.is_info()) { log.info("Metaspace (%s) allocation failed for size " SIZE_FORMAT, - is_class_space_allocation(mdtype) ? "class" : "data", word_size); + metaspace::is_class(mdtype) ? "class" : "data", word_size); ResourceMark rm; if (log.is_debug()) { if (loader_data->metaspace_or_null() != NULL) { @@ -1330,12 +856,17 @@ MetaspaceUtils::print_basic_report(&ls, 0); } + // Which limit did we hit? CompressedClassSpaceSize or MaxMetaspaceSize? bool out_of_compressed_class_space = false; - if (is_class_space_allocation(mdtype)) { + if (metaspace::is_class(mdtype)) { ClassLoaderMetaspace* metaspace = loader_data->metaspace_non_null(); out_of_compressed_class_space = MetaspaceUtils::committed_bytes(Metaspace::ClassType) + - (metaspace->class_chunk_size(word_size) * BytesPerWord) > + // TODO: Okay this is just cheesy. + // Of course this may fail and return incorrect results. + // Think this over - we need some clean way to remember which limit + // exactly we hit during an allocation. Some sort of allocation context structure? + align_up(word_size * BytesPerWord, 4 * M) > CompressedClassSpaceSize; } @@ -1362,26 +893,16 @@ } } -const char* Metaspace::metadata_type_name(Metaspace::MetadataType mdtype) { - switch (mdtype) { - case Metaspace::ClassType: return "Class"; - case Metaspace::NonClassType: return "Metadata"; - default: - assert(false, "Got bad mdtype: %d", (int) mdtype); - return NULL; +void Metaspace::purge() { + ChunkManager* cm = ChunkManager::chunkmanager_nonclass(); + if (cm != NULL) { + cm->purge(); } -} - -void Metaspace::purge(MetadataType mdtype) { - get_space_list(mdtype)->purge(get_chunk_manager(mdtype)); -} - -void Metaspace::purge() { - MutexLocker cl(MetaspaceExpand_lock, - Mutex::_no_safepoint_check_flag); - purge(NonClassType); if (using_class_space()) { - purge(ClassType); + cm = ChunkManager::chunkmanager_class(); + if (cm != NULL) { + cm->purge(); + } } } @@ -1393,214 +914,9 @@ } bool Metaspace::contains_non_shared(const void* ptr) { - if (using_class_space() && get_space_list(ClassType)->contains(ptr)) { + if (using_class_space() && VirtualSpaceList::vslist_class()->contains((MetaWord*)ptr)) { return true; } - return get_space_list(NonClassType)->contains(ptr); + return VirtualSpaceList::vslist_nonclass()->contains((MetaWord*)ptr); } - -// ClassLoaderMetaspace - -ClassLoaderMetaspace::ClassLoaderMetaspace(Mutex* lock, Metaspace::MetaspaceType type) - : _space_type(type) - , _lock(lock) - , _vsm(NULL) - , _class_vsm(NULL) -{ - initialize(lock, type); -} - -ClassLoaderMetaspace::~ClassLoaderMetaspace() { - Metaspace::assert_not_frozen(); - DEBUG_ONLY(Atomic::inc(&g_internal_statistics.num_metaspace_deaths)); - delete _vsm; - if (Metaspace::using_class_space()) { - delete _class_vsm; - } -} - -void ClassLoaderMetaspace::initialize_first_chunk(Metaspace::MetaspaceType type, Metaspace::MetadataType mdtype) { - Metachunk* chunk = get_initialization_chunk(type, mdtype); - if (chunk != NULL) { - // Add to this manager's list of chunks in use and make it the current_chunk(). - get_space_manager(mdtype)->add_chunk(chunk, true); - } -} - -Metachunk* ClassLoaderMetaspace::get_initialization_chunk(Metaspace::MetaspaceType type, Metaspace::MetadataType mdtype) { - size_t chunk_word_size = get_space_manager(mdtype)->get_initial_chunk_size(type); - - // Get a chunk from the chunk freelist - Metachunk* chunk = Metaspace::get_chunk_manager(mdtype)->chunk_freelist_allocate(chunk_word_size); - - if (chunk == NULL) { - chunk = Metaspace::get_space_list(mdtype)->get_new_chunk(chunk_word_size, - get_space_manager(mdtype)->medium_chunk_bunch()); - } - - return chunk; -} - -void ClassLoaderMetaspace::initialize(Mutex* lock, Metaspace::MetaspaceType type) { - Metaspace::verify_global_initialization(); - - DEBUG_ONLY(Atomic::inc(&g_internal_statistics.num_metaspace_births)); - - // Allocate SpaceManager for metadata objects. - _vsm = new SpaceManager(Metaspace::NonClassType, type, lock); - - if (Metaspace::using_class_space()) { - // Allocate SpaceManager for classes. - _class_vsm = new SpaceManager(Metaspace::ClassType, type, lock); - } - - MutexLocker cl(MetaspaceExpand_lock, Mutex::_no_safepoint_check_flag); - - // Allocate chunk for metadata objects - initialize_first_chunk(type, Metaspace::NonClassType); - - // Allocate chunk for class metadata objects - if (Metaspace::using_class_space()) { - initialize_first_chunk(type, Metaspace::ClassType); - } -} - -MetaWord* ClassLoaderMetaspace::allocate(size_t word_size, Metaspace::MetadataType mdtype) { - Metaspace::assert_not_frozen(); - - DEBUG_ONLY(Atomic::inc(&g_internal_statistics.num_allocs)); - - // Don't use class_vsm() unless UseCompressedClassPointers is true. - if (Metaspace::is_class_space_allocation(mdtype)) { - return class_vsm()->allocate(word_size); - } else { - return vsm()->allocate(word_size); - } -} - -MetaWord* ClassLoaderMetaspace::expand_and_allocate(size_t word_size, Metaspace::MetadataType mdtype) { - Metaspace::assert_not_frozen(); - size_t delta_bytes = MetaspaceGC::delta_capacity_until_GC(word_size * BytesPerWord); - assert(delta_bytes > 0, "Must be"); - - size_t before = 0; - size_t after = 0; - bool can_retry = true; - MetaWord* res; - bool incremented; - - // Each thread increments the HWM at most once. Even if the thread fails to increment - // the HWM, an allocation is still attempted. This is because another thread must then - // have incremented the HWM and therefore the allocation might still succeed. - do { - incremented = MetaspaceGC::inc_capacity_until_GC(delta_bytes, &after, &before, &can_retry); - res = allocate(word_size, mdtype); - } while (!incremented && res == NULL && can_retry); - - if (incremented) { - Metaspace::tracer()->report_gc_threshold(before, after, - MetaspaceGCThresholdUpdater::ExpandAndAllocate); - log_trace(gc, metaspace)("Increase capacity to GC from " SIZE_FORMAT " to " SIZE_FORMAT, before, after); - } - - return res; -} - -size_t ClassLoaderMetaspace::allocated_blocks_bytes() const { - return (vsm()->used_words() + - (Metaspace::using_class_space() ? class_vsm()->used_words() : 0)) * BytesPerWord; -} - -size_t ClassLoaderMetaspace::allocated_chunks_bytes() const { - return (vsm()->capacity_words() + - (Metaspace::using_class_space() ? class_vsm()->capacity_words() : 0)) * BytesPerWord; -} - -void ClassLoaderMetaspace::deallocate(MetaWord* ptr, size_t word_size, bool is_class) { - Metaspace::assert_not_frozen(); - assert(!SafepointSynchronize::is_at_safepoint() - || Thread::current()->is_VM_thread(), "should be the VM thread"); - - DEBUG_ONLY(Atomic::inc(&g_internal_statistics.num_external_deallocs)); - - MutexLocker ml(vsm()->lock(), Mutex::_no_safepoint_check_flag); - - if (is_class && Metaspace::using_class_space()) { - class_vsm()->deallocate(ptr, word_size); - } else { - vsm()->deallocate(ptr, word_size); - } -} - -size_t ClassLoaderMetaspace::class_chunk_size(size_t word_size) { - assert(Metaspace::using_class_space(), "Has to use class space"); - return class_vsm()->calc_chunk_size(word_size); -} - -void ClassLoaderMetaspace::print_on(outputStream* out) const { - // Print both class virtual space counts and metaspace. - if (Verbose) { - vsm()->print_on(out); - if (Metaspace::using_class_space()) { - class_vsm()->print_on(out); - } - } -} - -void ClassLoaderMetaspace::verify() { - vsm()->verify(); - if (Metaspace::using_class_space()) { - class_vsm()->verify(); - } -} - -void ClassLoaderMetaspace::add_to_statistics_locked(ClassLoaderMetaspaceStatistics* out) const { - assert_lock_strong(lock()); - vsm()->add_to_statistics_locked(&out->nonclass_sm_stats()); - if (Metaspace::using_class_space()) { - class_vsm()->add_to_statistics_locked(&out->class_sm_stats()); - } -} - -void ClassLoaderMetaspace::add_to_statistics(ClassLoaderMetaspaceStatistics* out) const { - MutexLocker cl(lock(), Mutex::_no_safepoint_check_flag); - add_to_statistics_locked(out); -} - -/////////////// Unit tests /////////////// - -struct chunkmanager_statistics_t { - int num_specialized_chunks; - int num_small_chunks; - int num_medium_chunks; - int num_humongous_chunks; -}; - -extern void test_metaspace_retrieve_chunkmanager_statistics(Metaspace::MetadataType mdType, chunkmanager_statistics_t* out) { - ChunkManager* const chunk_manager = Metaspace::get_chunk_manager(mdType); - ChunkManagerStatistics stat; - chunk_manager->collect_statistics(&stat); - out->num_specialized_chunks = (int)stat.chunk_stats(SpecializedIndex).num(); - out->num_small_chunks = (int)stat.chunk_stats(SmallIndex).num(); - out->num_medium_chunks = (int)stat.chunk_stats(MediumIndex).num(); - out->num_humongous_chunks = (int)stat.chunk_stats(HumongousIndex).num(); -} - -struct chunk_geometry_t { - size_t specialized_chunk_word_size; - size_t small_chunk_word_size; - size_t medium_chunk_word_size; -}; - -extern void test_metaspace_retrieve_chunk_geometry(Metaspace::MetadataType mdType, chunk_geometry_t* out) { - if (mdType == Metaspace::NonClassType) { - out->specialized_chunk_word_size = SpecializedChunk; - out->small_chunk_word_size = SmallChunk; - out->medium_chunk_word_size = MediumChunk; - } else { - out->specialized_chunk_word_size = ClassSpecializedChunk; - out->small_chunk_word_size = ClassSmallChunk; - out->medium_chunk_word_size = ClassMediumChunk; - } -} diff --git a/src/hotspot/share/memory/metaspace.hpp b/src/hotspot/share/memory/metaspace.hpp --- a/src/hotspot/share/memory/metaspace.hpp +++ b/src/hotspot/share/memory/metaspace.hpp @@ -28,65 +28,23 @@ #include "memory/memRegion.hpp" #include "memory/metaspaceChunkFreeListSummary.hpp" #include "memory/virtualspace.hpp" -#include "memory/metaspace/metaspaceSizesSnapshot.hpp" #include "runtime/globals.hpp" #include "utilities/exceptions.hpp" - -// Metaspace -// -// Metaspaces are Arenas for the VM's metadata. -// They are allocated one per class loader object, and one for the null -// bootstrap class loader -// -// block X ---+ +-------------------+ -// | | Virtualspace | -// | | | -// | | | -// | |-------------------| -// | || Chunk | -// | || | -// | ||---------- | -// +------>||| block 0 | | -// ||---------- | -// ||| block 1 | | -// ||---------- | -// || | -// |-------------------| -// | | -// | | -// +-------------------+ -// +#include "utilities/globalDefinitions.hpp" class ClassLoaderData; +class MetaspaceShared; class MetaspaceTracer; class Mutex; class outputStream; -class CollectedHeap; - namespace metaspace { - class ChunkManager; - class ClassLoaderMetaspaceStatistics; - class Metablock; - class Metachunk; - class PrintCLDMetaspaceInfoClosure; - class SpaceManager; - class VirtualSpaceList; - class VirtualSpaceNode; + class MetaspaceArena; + class MetaspaceSizesSnapshot; + struct clms_stats_t; } -// Metaspaces each have a SpaceManager and allocations -// are done by the SpaceManager. Allocations are done -// out of the current Metachunk. When the current Metachunk -// is exhausted, the SpaceManager gets a new one from -// the current VirtualSpace. When the VirtualSpace is exhausted -// the SpaceManager gets a new one. The SpaceManager -// also manages freelists of available Chunks. -// -// Currently the space manager maintains the list of -// virtual spaces and the list of chunks in use. Its -// allocate() method returns a block for use as a -// quantum of metadata. +////////////////// Metaspace /////////////////////// // Namespace for important central static functions // (auxiliary stuff goes into MetaspaceUtils) @@ -94,7 +52,7 @@ friend class MetaspaceShared; - public: +public: enum MetadataType { ClassType, NonClassType, @@ -109,59 +67,15 @@ MetaspaceTypeCount }; - private: +private: - // Align up the word size to the allocation word size - static size_t align_word_size_up(size_t); - - // Aligned size of the metaspace. - static size_t _compressed_class_space_size; - - static size_t compressed_class_space_size() { - return _compressed_class_space_size; - } - - static void set_compressed_class_space_size(size_t size) { - _compressed_class_space_size = size; - } - - static size_t _first_chunk_word_size; - static size_t _first_class_chunk_word_size; - - static size_t _commit_alignment; - static size_t _reserve_alignment; DEBUG_ONLY(static bool _frozen;) - // Virtual Space lists for both classes and other metadata - static metaspace::VirtualSpaceList* _space_list; - static metaspace::VirtualSpaceList* _class_space_list; - - static metaspace::ChunkManager* _chunk_manager_metadata; - static metaspace::ChunkManager* _chunk_manager_class; - static const MetaspaceTracer* _tracer; static bool _initialized; - public: - static metaspace::VirtualSpaceList* space_list() { return _space_list; } - static metaspace::VirtualSpaceList* class_space_list() { return _class_space_list; } - static metaspace::VirtualSpaceList* get_space_list(MetadataType mdtype) { - assert(mdtype != MetadataTypeCount, "MetadaTypeCount can't be used as mdtype"); - return mdtype == ClassType ? class_space_list() : space_list(); - } - - static metaspace::ChunkManager* chunk_manager_metadata() { return _chunk_manager_metadata; } - static metaspace::ChunkManager* chunk_manager_class() { return _chunk_manager_class; } - static metaspace::ChunkManager* get_chunk_manager(MetadataType mdtype) { - assert(mdtype != MetadataTypeCount, "MetadaTypeCount can't be used as mdtype"); - return mdtype == ClassType ? chunk_manager_class() : chunk_manager_metadata(); - } - - // convenience function - static metaspace::ChunkManager* get_chunk_manager(bool is_class) { - return is_class ? chunk_manager_class() : chunk_manager_metadata(); - } +public: static const MetaspaceTracer* tracer() { return _tracer; } static void freeze() { @@ -188,7 +102,7 @@ static void initialize_class_space(ReservedSpace rs); // Returns true if class space has been setup (initialize_class_space). - static bool class_space_is_initialized() { return _class_space_list != NULL; } + static bool class_space_is_initialized(); #endif @@ -198,15 +112,18 @@ static void global_initialize(); static void post_initialize(); - static void verify_global_initialization(); + // Alignment, in bytes, of metaspace mappings + static size_t reserve_alignment() { return reserve_alignment_words() * BytesPerWord; } + // Alignment, in words, of metaspace mappings + static size_t reserve_alignment_words(); - static size_t first_chunk_word_size() { return _first_chunk_word_size; } - static size_t first_class_chunk_word_size() { return _first_class_chunk_word_size; } + // The granularity at which Metaspace is committed and uncommitted. + // (Todo: Why does this have to be exposed?) + static size_t commit_alignment() { return commit_alignment_words() * BytesPerWord; } + static size_t commit_alignment_words(); - static size_t reserve_alignment() { return _reserve_alignment; } - static size_t reserve_alignment_words() { return _reserve_alignment / BytesPerWord; } - static size_t commit_alignment() { return _commit_alignment; } - static size_t commit_alignment_words() { return _commit_alignment / BytesPerWord; } + // The largest possible single allocation + static size_t max_allocation_word_size(); static MetaWord* allocate(ClassLoaderData* loader_data, size_t word_size, MetaspaceObj::Type type, TRAPS); @@ -215,13 +132,10 @@ static bool contains_non_shared(const void* ptr); // Free empty virtualspaces - static void purge(MetadataType mdtype); static void purge(); static void report_metadata_oome(ClassLoaderData* loader_data, size_t word_size, - MetaspaceObj::Type type, MetadataType mdtype, TRAPS); - - static const char* metadata_type_name(Metaspace::MetadataType mdtype); + MetaspaceObj::Type type, Metaspace::MetadataType mdtype, TRAPS); static void print_compressed_class_space(outputStream* st) NOT_LP64({}); @@ -230,216 +144,116 @@ return NOT_LP64(false) LP64_ONLY(UseCompressedClassPointers); } - static bool is_class_space_allocation(MetadataType mdType) { - return mdType == ClassType && using_class_space(); - } - - static bool initialized() { return _initialized; } + static bool initialized(); }; -// Manages the metaspace portion belonging to a class loader +// ClassLoaderMetaspace is an inbetween-object between a CLD and its MetaspaceArena(s). +// +// A CLD owns one MetaspaceArena if compressed class space is off, two if its one +// (one for allocations of Klass* structures from class space, one for the rest from +// non-class space). +// +// ClassLoaderMetaspace only exists to hide this logic from upper layers: +// +// +------+ +----------------------+ +-------------------+ +// | CLD | ---> | ClassLoaderMetaspace | ----> | (non class) Arena | +// +------+ +----------------------+ | +-------------------+ allocation top +// | | v +// | + chunk -- chunk ... -- chunk +// | +// | +-------------------+ +// +--> | (class) Arena | +// +-------------------+ +// | +// + chunk ... chunk +// ^ +// alloc top +// class ClassLoaderMetaspace : public CHeapObj { - friend class CollectedHeap; // For expand_and_allocate() - friend class ZCollectedHeap; // For expand_and_allocate() - friend class ShenandoahHeap; // For expand_and_allocate() - friend class Metaspace; - friend class MetaspaceUtils; - friend class metaspace::PrintCLDMetaspaceInfoClosure; - friend class VM_CollectForMetadataAllocation; // For expand_and_allocate() - private: - - void initialize(Mutex* lock, Metaspace::MetaspaceType type); - - // Initialize the first chunk for a Metaspace. Used for - // special cases such as the boot class loader, reflection - // class loader and hidden class loader. - void initialize_first_chunk(Metaspace::MetaspaceType type, Metaspace::MetadataType mdtype); - metaspace::Metachunk* get_initialization_chunk(Metaspace::MetaspaceType type, Metaspace::MetadataType mdtype); + // A reference to an outside lock, held by the CLD. + Mutex* const _lock; const Metaspace::MetaspaceType _space_type; - Mutex* const _lock; - metaspace::SpaceManager* _vsm; - metaspace::SpaceManager* _class_vsm; - metaspace::SpaceManager* vsm() const { return _vsm; } - metaspace::SpaceManager* class_vsm() const { return _class_vsm; } - metaspace::SpaceManager* get_space_manager(Metaspace::MetadataType mdtype) { - assert(mdtype != Metaspace::MetadataTypeCount, "MetadaTypeCount can't be used as mdtype"); - return mdtype == Metaspace::ClassType ? class_vsm() : vsm(); + // Arena for allocations from non-class metaspace + // (resp. for all allocations if -XX:-UseCompressedClassPointers). + metaspace::MetaspaceArena* _non_class_space_arena; + + // Arena for allocations from class space + // (NULL if -XX:-UseCompressedClassPointers). + metaspace::MetaspaceArena* _class_space_arena; + + Mutex* lock() const { return _lock; } + metaspace::MetaspaceArena* non_class_space_arena() const { return _non_class_space_arena; } + metaspace::MetaspaceArena* class_space_arena() const { return _class_space_arena; } + + metaspace::MetaspaceArena* get_arena(bool is_class) { + return is_class ? class_space_arena() : non_class_space_arena(); } - Mutex* lock() const { return _lock; } +public: - MetaWord* expand_and_allocate(size_t size, Metaspace::MetadataType mdtype); + ClassLoaderMetaspace(Mutex* lock, Metaspace::MetaspaceType space_type); - size_t class_chunk_size(size_t word_size); - - // Adds to the given statistic object. Must be locked with CLD metaspace lock. - void add_to_statistics_locked(metaspace::ClassLoaderMetaspaceStatistics* out) const; + ~ClassLoaderMetaspace(); Metaspace::MetaspaceType space_type() const { return _space_type; } - public: + // Allocate word_size words from Metaspace. + MetaWord* allocate(size_t word_size, Metaspace::MetadataType mdType); - ClassLoaderMetaspace(Mutex* lock, Metaspace::MetaspaceType type); - ~ClassLoaderMetaspace(); + // Attempt to expand the GC threshold to be good for at least another word_size words + // and allocate. Returns NULL if failure. Used during Metaspace GC. + MetaWord* expand_and_allocate(size_t word_size, Metaspace::MetadataType mdType); - // Allocate space for metadata of type mdtype. This is space - // within a Metachunk and is used by - // allocate(ClassLoaderData*, size_t, bool, MetadataType, TRAPS) - MetaWord* allocate(size_t word_size, Metaspace::MetadataType mdtype); + // Prematurely returns a metaspace allocation to the _block_freelists + // because it is not needed anymore. + void deallocate(MetaWord* ptr, size_t word_size, bool is_class); - size_t allocated_blocks_bytes() const; - size_t allocated_chunks_bytes() const; + // Update statistics. This walks all in-use chunks. + void add_to_statistics(metaspace::clms_stats_t* out) const; - void deallocate(MetaWord* ptr, size_t byte_size, bool is_class); + DEBUG_ONLY(void verify() const;) - void print_on(outputStream* st) const; - // Debugging support - void verify(); + // This only exists for JFR and jcmd VM.classloader_stats. We may want to + // change this. Capacity as a stat is of questionable use since it may + // contain committed and uncommitted areas. For now we do this to maintain + // backward compatibility with JFR. + void calculate_jfr_stats(size_t* p_used_bytes, size_t* p_capacity_bytes) const; - // Adds to the given statistic object. Will lock with CLD metaspace lock. - void add_to_statistics(metaspace::ClassLoaderMetaspaceStatistics* out) const; +}; // end: ClassLoaderMetaspace -}; // ClassLoaderMetaspace -class MetaspaceUtils : AllStatic { - - // Spacemanager updates running counters. - friend class metaspace::SpaceManager; - - // Special access for error reporting (checks without locks). - friend class oopDesc; - friend class Klass; - - // Running counters for statistics concerning in-use chunks. - // Note: capacity = used + free + waste + overhead. Note that we do not - // count free and waste. Their sum can be deduces from the three other values. - // For more details, one should call print_report() from within a safe point. - static size_t _capacity_words [Metaspace:: MetadataTypeCount]; - static size_t _overhead_words [Metaspace:: MetadataTypeCount]; - static volatile size_t _used_words [Metaspace:: MetadataTypeCount]; - - // Atomically decrement or increment in-use statistic counters - static void dec_capacity(Metaspace::MetadataType mdtype, size_t words); - static void inc_capacity(Metaspace::MetadataType mdtype, size_t words); - static void dec_used(Metaspace::MetadataType mdtype, size_t words); - static void inc_used(Metaspace::MetadataType mdtype, size_t words); - static void dec_overhead(Metaspace::MetadataType mdtype, size_t words); - static void inc_overhead(Metaspace::MetadataType mdtype, size_t words); - - - // Getters for the in-use counters. - static size_t capacity_words(Metaspace::MetadataType mdtype) { return _capacity_words[mdtype]; } - static size_t used_words(Metaspace::MetadataType mdtype) { return _used_words[mdtype]; } - static size_t overhead_words(Metaspace::MetadataType mdtype) { return _overhead_words[mdtype]; } - - static size_t free_chunks_total_words(Metaspace::MetadataType mdtype); - - // Helper for print_xx_report. - static void print_vs(outputStream* out, size_t scale); - -public: - - // Collect used metaspace statistics. This involves walking the CLDG. The resulting - // output will be the accumulated values for all live metaspaces. - // Note: method does not do any locking. - static void collect_statistics(metaspace::ClassLoaderMetaspaceStatistics* out); - - // Used by MetaspaceCounters - static size_t free_chunks_total_words(); - static size_t free_chunks_total_bytes(); - static size_t free_chunks_total_bytes(Metaspace::MetadataType mdtype); - - static size_t capacity_words() { - return capacity_words(Metaspace::NonClassType) + - capacity_words(Metaspace::ClassType); - } - static size_t capacity_bytes(Metaspace::MetadataType mdtype) { - return capacity_words(mdtype) * BytesPerWord; - } - static size_t capacity_bytes() { - return capacity_words() * BytesPerWord; - } - - static size_t used_words() { - return used_words(Metaspace::NonClassType) + - used_words(Metaspace::ClassType); - } - static size_t used_bytes(Metaspace::MetadataType mdtype) { - return used_words(mdtype) * BytesPerWord; - } - static size_t used_bytes() { - return used_words() * BytesPerWord; - } - - // Space committed but yet unclaimed by any class loader. - static size_t free_in_vs_bytes(); - static size_t free_in_vs_bytes(Metaspace::MetadataType mdtype); - - static size_t reserved_bytes(Metaspace::MetadataType mdtype); - static size_t reserved_bytes() { - return reserved_bytes(Metaspace::ClassType) + - reserved_bytes(Metaspace::NonClassType); - } - - static size_t committed_bytes(Metaspace::MetadataType mdtype); - static size_t committed_bytes() { - return committed_bytes(Metaspace::ClassType) + - committed_bytes(Metaspace::NonClassType); - } - - static size_t min_chunk_size_words(); - - // Flags for print_report(). - enum ReportFlag { - // Show usage by class loader. - rf_show_loaders = (1 << 0), - // Breaks report down by chunk type (small, medium, ...). - rf_break_down_by_chunktype = (1 << 1), - // Breaks report down by space type (hidden, reflection, ...). - rf_break_down_by_spacetype = (1 << 2), - // Print details about the underlying virtual spaces. - rf_show_vslist = (1 << 3), - // Print metaspace map. - rf_show_vsmap = (1 << 4), - // If show_loaders: show loaded classes for each loader. - rf_show_classes = (1 << 5) - }; - - // This will print out a basic metaspace usage report but - // unlike print_report() is guaranteed not to lock or to walk the CLDG. - static void print_basic_report(outputStream* st, size_t scale); - - // Prints a report about the current metaspace state. - // Optional parts can be enabled via flags. - // Function will walk the CLDG and will lock the expand lock; if that is not - // convenient, use print_basic_report() instead. - static void print_report(outputStream* out, size_t scale = 0, int flags = 0); - - static bool has_chunk_free_list(Metaspace::MetadataType mdtype); - static MetaspaceChunkFreeListSummary chunk_free_list_summary(Metaspace::MetadataType mdtype); - - // Log change in used metadata. - static void print_metaspace_change(const metaspace::MetaspaceSizesSnapshot& pre_meta_values); - static void print_on(outputStream * out); - - // Prints an ASCII representation of the given space. - static void print_metaspace_map(outputStream* out, Metaspace::MetadataType mdtype); - - static void dump(outputStream* out); - static void verify_free_chunks(); - // Check internal counters (capacity, used). - static void verify_metrics(); -}; +////////////////// MetaspaceGC /////////////////////// // Metaspace are deallocated when their class loader are GC'ed. // This class implements a policy for inducing GC's to recover // Metaspaces. -class MetaspaceGC : AllStatic { +class MetaspaceGCThresholdUpdater : public AllStatic { + public: + enum Type { + ComputeNewSize, + ExpandAndAllocate, + Last + }; + + static const char* to_string(MetaspaceGCThresholdUpdater::Type updater) { + switch (updater) { + case ComputeNewSize: + return "compute_new_size"; + case ExpandAndAllocate: + return "expand_and_allocate"; + default: + assert(false, "Got bad updater: %d", (int) updater); + return NULL; + }; + } +}; + +class MetaspaceGC : public AllStatic { // The current high-water-mark for inducing a GC. // When committed memory of all metaspaces reaches this value, @@ -477,4 +291,53 @@ static void compute_new_size(); }; + + + +class MetaspaceUtils : AllStatic { +public: + + // Committed space actually in use by Metadata + static size_t used_words(); + static size_t used_words(Metaspace::MetadataType mdtype); + + // Space committed for Metaspace + static size_t committed_words(); + static size_t committed_words(Metaspace::MetadataType mdtype); + + // Space reserved for Metaspace + static size_t reserved_words(); + static size_t reserved_words(Metaspace::MetadataType mdtype); + + // _bytes() variants for convenience... + static size_t used_bytes() { return used_words() * BytesPerWord; } + static size_t used_bytes(Metaspace::MetadataType mdtype) { return used_words(mdtype) * BytesPerWord; } + static size_t committed_bytes() { return committed_words() * BytesPerWord; } + static size_t committed_bytes(Metaspace::MetadataType mdtype) { return committed_words(mdtype) * BytesPerWord; } + static size_t reserved_bytes() { return reserved_words() * BytesPerWord; } + static size_t reserved_bytes(Metaspace::MetadataType mdtype) { return reserved_words(mdtype) * BytesPerWord; } + + // (See JDK-8251342). Implement or Consolidate. + static MetaspaceChunkFreeListSummary chunk_free_list_summary(Metaspace::MetadataType mdtype) { + return MetaspaceChunkFreeListSummary(0,0,0,0,0,0,0,0); + } + + // Log change in used metadata. + static void print_metaspace_change(const metaspace::MetaspaceSizesSnapshot& pre_meta_values); + + // This will print out a basic metaspace usage report but + // unlike print_report() is guaranteed not to lock or to walk the CLDG. + static void print_basic_report(outputStream* st, size_t scale = 0); + + // Prints a report about the current metaspace state. + // Function will walk the CLDG and will lock the expand lock; if that is not + // convenient, use print_basic_report() instead. + static void print_report(outputStream* out, size_t scale = 0); + + static void print_on(outputStream * out); + + DEBUG_ONLY(static void verify(bool slow);) + +}; + #endif // SHARE_MEMORY_METASPACE_HPP diff --git a/src/hotspot/share/memory/metaspace/allocationGuard.hpp b/src/hotspot/share/memory/metaspace/allocationGuard.hpp new file mode 100644 --- /dev/null +++ b/src/hotspot/share/memory/metaspace/allocationGuard.hpp @@ -0,0 +1,120 @@ +/* + * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020 SAP SE. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_MEMORY_METASPACE_ALLOCATION_GUARD_HPP +#define SHARE_MEMORY_METASPACE_ALLOCATION_GUARD_HPP + +#include "memory/allocation.hpp" +#include "memory/metaspace/chunkLevel.hpp" +#include "utilities/debug.hpp" +#include "utilities/globalDefinitions.hpp" + +// In Debug builds, Metadata in Metaspace can be optionally guarded - enclosed in canaries - +// to detect memory overwriters. +// +// These canaries are periodically checked, e.g. when the Metaspace is purged in a context +// of a GC. + +// The canaries precede any allocated block... +// +// +---------------+ +// | 'METAMETA' | +// +---------------+ +// | block size | +// +---------------+ +// | block... | +// . . +// . . +// . . +// | | +// +---------------+ +// . . +// +---------------+ +// | 'METAMETA' | +// +---------------+ +// | block size | +// +---------------+ +// | block... | + +// ... and since the blocks are allocated via pointer bump and closely follow each other, +// one block's prefix is its predecessor's suffix, so apart from the last block all +// blocks have an overwriter canary on both ends. +// + +// Note: this feature is only available in debug, and is activated using +// -XX:+MetaspaceGuardAllocations. When active, it disables deallocation handling - since +// freeblock handling in the freeblock lists would get too complex - so one may run leaks +// in deallocation-heavvy scenarios (e.g. lots of class redefinitions). +// + + +namespace metaspace { + +#ifdef ASSERT + +struct prefix_t { + uintx mark; + size_t word_size; // raw word size including prefix + // MetaWord payload [0]; // varsized (but unfortunately not all our compilers understand that) +}; + +// The prefix structure must be aligned to MetaWord size. +STATIC_ASSERT((sizeof(prefix_t) & WordAlignmentMask) == 0); + +inline prefix_t* prefix_from_payload(MetaWord* p) { + return (prefix_t*)((address)p - sizeof(prefix_t)); +} + +inline MetaWord* payload_from_prefix(prefix_t* pp) { + // return pp->payload; + return (MetaWord*)((address)pp + sizeof(prefix_t)); +} + +inline size_t prefix_size() { + return sizeof(prefix_t); +} + +#define EYECATCHER NOT_LP64(0x77698465) LP64_ONLY(0x7769846577698465ULL) // "META" resp "METAMETA" + +// Given a pointer to a memory area, establish the prefix at the start of that area and +// return the starting pointer to the payload. +inline MetaWord* establish_prefix(MetaWord* p_raw, size_t raw_word_size) { + prefix_t* pp = (prefix_t*)p_raw; + pp->mark = EYECATCHER; + pp->word_size = raw_word_size; + return payload_from_prefix(pp); +} + +inline void check_prefix(const prefix_t* pp) { + assert(pp->mark == EYECATCHER, "corrupt block at " PTR_FORMAT ".", p2i(pp)); + assert(pp->word_size > 0 && pp->word_size < chunklevel::MAX_CHUNK_WORD_SIZE, + "Invalid size " SIZE_FORMAT " in block at " PTR_FORMAT ".", pp->word_size, p2i(pp)); +} + +#endif + +} // namespace metaspace + +#endif // SHARE_MEMORY_METASPACE_ALLOCATION_GUARD_HPP diff --git a/src/hotspot/share/memory/metaspace/arenaGrowthPolicy.cpp b/src/hotspot/share/memory/metaspace/arenaGrowthPolicy.cpp new file mode 100644 --- /dev/null +++ b/src/hotspot/share/memory/metaspace/arenaGrowthPolicy.cpp @@ -0,0 +1,128 @@ +/* + * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020 SAP SE. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" + +#include "memory/metaspace/arenaGrowthPolicy.hpp" +#include "memory/metaspace/chunkLevel.hpp" +#include "utilities/globalDefinitions.hpp" + +namespace metaspace { + +// hard-coded chunk allocation sequences for various space types +// (Note: when modifying this, don't add jumps of more than double the +// last chunk size. There is a gtest testing this, see test_arenagrowthpolicy.cpp) + +static const chunklevel_t g_sequ_standard_non_class[] = { + chunklevel::CHUNK_LEVEL_4K, + chunklevel::CHUNK_LEVEL_4K, + chunklevel::CHUNK_LEVEL_4K, + chunklevel::CHUNK_LEVEL_8K, + chunklevel::CHUNK_LEVEL_16K + // .. repeat last +}; + +static const chunklevel_t g_sequ_standard_class[] = { + chunklevel::CHUNK_LEVEL_2K, + chunklevel::CHUNK_LEVEL_2K, + chunklevel::CHUNK_LEVEL_4K, + chunklevel::CHUNK_LEVEL_8K, + chunklevel::CHUNK_LEVEL_16K + // .. repeat last +}; + +static const chunklevel_t g_sequ_anon_non_class[] = { + chunklevel::CHUNK_LEVEL_1K, + // .. repeat last +}; + +static const chunklevel_t g_sequ_anon_class[] = { + chunklevel::CHUNK_LEVEL_1K, + // .. repeat last +}; + +static const chunklevel_t g_sequ_refl_non_class[] = { + chunklevel::CHUNK_LEVEL_2K, + chunklevel::CHUNK_LEVEL_1K + // .. repeat last +}; + +static const chunklevel_t g_sequ_refl_class[] = { + chunklevel::CHUNK_LEVEL_1K, + // .. repeat last +}; + +// Boot class loader: give it large chunks: beyond commit granule size +// (typically 64K) the costs for large chunks largely diminishes since +// they are committed on the fly. +static const chunklevel_t g_sequ_boot_non_class[] = { + chunklevel::CHUNK_LEVEL_4M, + chunklevel::CHUNK_LEVEL_1M + // .. repeat last +}; + +static const chunklevel_t g_sequ_boot_class[] = { + chunklevel::CHUNK_LEVEL_256K + // .. repeat last +}; + +const ArenaGrowthPolicy* ArenaGrowthPolicy::policy_for_space_type(Metaspace::MetaspaceType space_type, bool is_class) { + +#define DEFINE_CLASS_FOR_ARRAY(what) \ + static ArenaGrowthPolicy chunk_alloc_sequence_##what (g_sequ_##what, sizeof(g_sequ_##what)/sizeof(chunklevel_t)); + + DEFINE_CLASS_FOR_ARRAY(standard_non_class) + DEFINE_CLASS_FOR_ARRAY(standard_class) + DEFINE_CLASS_FOR_ARRAY(anon_non_class) + DEFINE_CLASS_FOR_ARRAY(anon_class) + DEFINE_CLASS_FOR_ARRAY(refl_non_class) + DEFINE_CLASS_FOR_ARRAY(refl_class) + DEFINE_CLASS_FOR_ARRAY(boot_non_class) + DEFINE_CLASS_FOR_ARRAY(boot_class) + + if (is_class) { + switch(space_type) { + case Metaspace::StandardMetaspaceType: return &chunk_alloc_sequence_standard_class; + case Metaspace::ReflectionMetaspaceType: return &chunk_alloc_sequence_refl_class; + case Metaspace::ClassMirrorHolderMetaspaceType: return &chunk_alloc_sequence_anon_class; + case Metaspace::BootMetaspaceType: return &chunk_alloc_sequence_boot_class; + default: ShouldNotReachHere(); + } + } else { + switch(space_type) { + case Metaspace::StandardMetaspaceType: return &chunk_alloc_sequence_standard_non_class; + case Metaspace::ReflectionMetaspaceType: return &chunk_alloc_sequence_refl_non_class; + case Metaspace::ClassMirrorHolderMetaspaceType: return &chunk_alloc_sequence_anon_non_class; + case Metaspace::BootMetaspaceType: return &chunk_alloc_sequence_boot_non_class; + default: ShouldNotReachHere(); + } + } + + return NULL; + +} + +} // namespace + diff --git a/src/hotspot/share/memory/metaspace/arenaGrowthPolicy.hpp b/src/hotspot/share/memory/metaspace/arenaGrowthPolicy.hpp new file mode 100644 --- /dev/null +++ b/src/hotspot/share/memory/metaspace/arenaGrowthPolicy.hpp @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020 SAP SE. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_MEMORY_METASPACE_CHUNKALLOCSEQUENCE_HPP +#define SHARE_MEMORY_METASPACE_CHUNKALLOCSEQUENCE_HPP + +#include "memory/metaspace.hpp" // For Metaspace::MetaspaceType +#include "memory/metaspace/chunkLevel.hpp" +#include "utilities/debug.hpp" + +namespace metaspace { + + +// ArenaGrowthPolicy encodes the growth policy of a MetaspaceArena. +// +// These arenas grow in steps (by allocating new chunks). The coarseness of growth +// (chunk size, level) depends on what the arena is used for. Used for a class loader +// which is expected to load only one or very few classes should grow in tiny steps. +// For normal classloaders, it can grow in coarser steps, and arenas used by +// the boot loader will grow in even larger steps since we expect it to load a lot of +// classes. +// Note that when growing in large steps (in steps larger than a commit granule, +// by default 64K), costs diminish somewhat since we do not commit the whole space +// immediately. + +class ArenaGrowthPolicy { + + // const array specifying chunk level allocation progression (growth steps). Last + // chunk is to be an endlessly repeated allocation. + const chunklevel_t* const _entries; + const int _num_entries; + + ArenaGrowthPolicy(const chunklevel_t* array, int num_entries) + : _entries(array), _num_entries(num_entries) { + assert(_num_entries > 0, "must not be empty."); + } + +public: + + chunklevel_t get_level_at_step(int num_allocated) const { + if (num_allocated >= _num_entries) { + // Caller shall repeat last allocation + return _entries[_num_entries - 1]; + } + return _entries[num_allocated]; + } + + // Given a space type, return the correct policy to use. + // The returned object is static and read only. + static const ArenaGrowthPolicy* policy_for_space_type(Metaspace::MetaspaceType space_type, bool is_class); + +}; + + +} // namespace metaspace + +#endif // SHARE_MEMORY_METASPACE_CHUNKALLOCSEQUENCE_HPP diff --git a/src/hotspot/share/memory/metaspace/binlist.hpp b/src/hotspot/share/memory/metaspace/binlist.hpp new file mode 100644 --- /dev/null +++ b/src/hotspot/share/memory/metaspace/binlist.hpp @@ -0,0 +1,224 @@ +/* + * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020 SAP SE. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_MEMORY_METASPACE_BINLIST_HPP +#define SHARE_MEMORY_METASPACE_BINLIST_HPP + +#include "utilities/align.hpp" +#include "utilities/debug.hpp" +#include "utilities/globalDefinitions.hpp" +#include "memory/metaspace/counter.hpp" +#include "memory/metaspace/metaspaceCommon.hpp" + +namespace metaspace { + +// BinList is a data structure to manage small to very small memory blocks +// (only a few words). It is used to manage deallocated blocks - see +// class FreeBlocks. + +// Memory blocks are kept in linked lists. Each list +// contains blocks of only one size. There is a list for blocks of two words, +// for blocks of three words, etc. The list heads are kept in a vector, +// ordered by block size. +// + +// wordsize +// +// +---+ +---+ +---+ +---+ +// 1 | |-->| |-->| |-...->| | +// +---+ +---+ +---+ +---+ +// +// +----+ +----+ +----+ +----+ +// 2 | |-->| |-->| |-...->| | +// +----+ +----+ +----+ +----+ +// +// +-----+ +-----+ +-----+ +-----+ +// 3 | |-->| |-->| |-...->| | +// +-----+ +-----+ +-----+ +-----+ +// . +// . +// . +// +// +----------+ +----------+ +----------+ +----------+ +// n | |-->| |-->| |-...->| | +// +----------+ +----------+ +----------+ +----------+ + + +// Insertion is of course fast, O(1). +// +// On retrieval, we attempt to find the closest fit to a given size, walking the +// list head vector (a bitmask is used to speed that part up). +// +// This structure is a bit expensive in memory costs (we pay one pointer per managed +// block size) so we only use it for a small number of sizes. + +template +class BinListImpl { + + struct block_t { block_t* next; size_t size; }; + + // a mask to speed up searching for populated lists. + // 0 marks an empty list, 1 for a non-empty one. + typedef uint32_t mask_t; + STATIC_ASSERT(num_lists <= sizeof(mask_t) * 8); + + mask_t _mask; + + // minimal block size must be large enough to hold a block. + STATIC_ASSERT(smallest_size * sizeof(MetaWord) >= sizeof(block_t)); + +public: + + // block sizes this structure can keep are limited by [_min_block_size, _max_block_size) + const static size_t minimal_word_size = smallest_size; + const static size_t maximal_word_size = minimal_word_size + num_lists; + +private: + + block_t* _v[num_lists]; + + MemRangeCounter _counter; + + static int index_for_word_size(size_t word_size) { + int index = (int)(word_size - minimal_word_size); + assert(index >= 0 && index < num_lists, "Invalid index %d", index); + return index; + } + + static size_t word_size_for_index(int index) { + assert(index >= 0 && index < num_lists, "Invalid index %d", index); + return minimal_word_size + index; + } + + // Search the range [index, _num_lists) for the smallest non-empty list. Returns -1 on fail. + int index_for_next_non_empty_list(int index) { + assert(index >= 0 && index < num_lists, "Invalid index %d", index); + int i2 = index; + mask_t m = _mask >> i2; + if (m > 0) { + // count leading zeros would be helpful. + while ((m & 1) == 0) { + assert(_v[i2] == NULL, "mask mismatch"); + i2 ++; + m >>= 1; + } + // We must have found something. + assert(i2 < num_lists, "sanity."); + assert(_v[i2] != NULL, "mask mismatch"); + return i2; + } + return -1; + } + + void mask_set_bit(int bit) { _mask |= (((mask_t)1) << bit); } + void mask_clr_bit(int bit) { _mask &= ~(((mask_t)1) << bit); } + +public: + + BinListImpl() : _mask(0) { + for (int i = 0; i < num_lists; i ++) { + _v[i] = NULL; + } + } + + void add_block(MetaWord* p, size_t word_size) { + assert(word_size >= minimal_word_size && + word_size < maximal_word_size, "bad block size"); + const int index = index_for_word_size(word_size); + block_t* b = (block_t*)p; + b->size = word_size; + b->next = _v[index]; + _v[index] = b; + _counter.add(word_size); + mask_set_bit(index); + } + + // Given a word_size, searches and returns a block of at least that size. + // Block may be larger. Real block size is returned in *p_real_word_size. + MetaWord* get_block(size_t word_size, size_t* p_real_word_size) { + assert(word_size >= minimal_word_size && + word_size < maximal_word_size, "bad block size " SIZE_FORMAT ".", word_size); + int index = index_for_word_size(word_size); + index = index_for_next_non_empty_list(index); + if (index != -1) { + assert(_v[index] != NULL && + _v[index]->size >= word_size, "sanity"); + + MetaWord* const p = (MetaWord*)_v[index]; + const size_t real_word_size = word_size_for_index(index); + + _v[index] = _v[index]->next; + if (_v[index] == NULL) { + mask_clr_bit(index); + } + + _counter.sub(real_word_size); + *p_real_word_size = real_word_size; + + return p; + + } else { + + *p_real_word_size = 0; + return NULL; + + } + } + + + // Returns number of blocks in this structure + unsigned count() const { return _counter.count(); } + + // Returns total size, in words, of all elements. + size_t total_size() const { return _counter.total_size(); } + + bool is_empty() const { return _mask == 0; } + +#ifdef ASSERT + void verify() const { + MemRangeCounter local_counter; + for (int i = 0; i < num_lists; i ++) { + assert(((_mask >> i) & 1) == ((_v[i] == 0) ? 0 : 1), "sanity"); + const size_t s = minimal_word_size + i; + for (block_t* b = _v[i]; b != NULL; b = b->next) { + assert(b->size == s, "bad block size"); + local_counter.add(s); + } + } + local_counter.check(_counter); + } +#endif // ASSERT + + +}; + +typedef BinListImpl<2, 8> BinList8; +typedef BinListImpl<2, 16> BinList16; +typedef BinListImpl<2, 32> BinList32; +typedef BinListImpl<2, 64> BinList64; + +} // namespace metaspace + +#endif // SHARE_MEMORY_METASPACE_BINLIST_HPP diff --git a/src/hotspot/share/memory/metaspace/blockFreelist.cpp b/src/hotspot/share/memory/metaspace/blockFreelist.cpp deleted file mode 100644 --- a/src/hotspot/share/memory/metaspace/blockFreelist.cpp +++ /dev/null @@ -1,109 +0,0 @@ -/* - * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ -#include "precompiled.hpp" - -#include "logging/log.hpp" -#include "memory/binaryTreeDictionary.inline.hpp" -#include "memory/metaspace/blockFreelist.hpp" -#include "utilities/ostream.hpp" -#include "utilities/globalDefinitions.hpp" - - -namespace metaspace { - - -BlockFreelist::BlockFreelist() : _dictionary(new BlockTreeDictionary()), _small_blocks(NULL) {} - -BlockFreelist::~BlockFreelist() { - delete _dictionary; - if (_small_blocks != NULL) { - delete _small_blocks; - } -} - -void BlockFreelist::return_block(MetaWord* p, size_t word_size) { - assert(word_size >= SmallBlocks::small_block_min_size(), "never return dark matter"); - - Metablock* free_chunk = ::new (p) Metablock(word_size); - if (word_size < SmallBlocks::small_block_max_size()) { - small_blocks()->return_block(free_chunk, word_size); - } else { - dictionary()->return_chunk(free_chunk); -} - log_trace(gc, metaspace, freelist, blocks)("returning block at " INTPTR_FORMAT " size = " - SIZE_FORMAT, p2i(free_chunk), word_size); -} - -MetaWord* BlockFreelist::get_block(size_t word_size) { - assert(word_size >= SmallBlocks::small_block_min_size(), "never get dark matter"); - - // Try small_blocks first. - if (word_size < SmallBlocks::small_block_max_size()) { - // Don't create small_blocks() until needed. small_blocks() allocates the small block list for - // this space manager. - MetaWord* new_block = (MetaWord*) small_blocks()->get_block(word_size); - if (new_block != NULL) { - log_trace(gc, metaspace, freelist, blocks)("getting block at " INTPTR_FORMAT " size = " SIZE_FORMAT, - p2i(new_block), word_size); - return new_block; - } - } - - if (word_size < BlockFreelist::min_dictionary_size()) { - // If allocation in small blocks fails, this is Dark Matter. Too small for dictionary. - return NULL; - } - - Metablock* free_block = dictionary()->get_chunk(word_size); - if (free_block == NULL) { - return NULL; - } - - const size_t block_size = free_block->size(); - if (block_size > WasteMultiplier * word_size) { - return_block((MetaWord*)free_block, block_size); - return NULL; - } - - MetaWord* new_block = (MetaWord*)free_block; - assert(block_size >= word_size, "Incorrect size of block from freelist"); - const size_t unused = block_size - word_size; - if (unused >= SmallBlocks::small_block_min_size()) { - return_block(new_block + word_size, unused); - } - - log_trace(gc, metaspace, freelist, blocks)("getting block at " INTPTR_FORMAT " size = " SIZE_FORMAT, - p2i(new_block), word_size); - return new_block; -} - -void BlockFreelist::print_on(outputStream* st) const { - dictionary()->print_free_lists(st); - if (_small_blocks != NULL) { - _small_blocks->print_on(st); - } -} - -} // namespace metaspace - diff --git a/src/hotspot/share/memory/metaspace/blockFreelist.hpp b/src/hotspot/share/memory/metaspace/blockFreelist.hpp deleted file mode 100644 --- a/src/hotspot/share/memory/metaspace/blockFreelist.hpp +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Copyright (c) 2018, 2019, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_MEMORY_METASPACE_BLOCKFREELIST_HPP -#define SHARE_MEMORY_METASPACE_BLOCKFREELIST_HPP - -#include "memory/allocation.hpp" -#include "memory/binaryTreeDictionary.hpp" -#include "memory/freeList.hpp" -#include "memory/metaspace/smallBlocks.hpp" -#include "memory/metaspace/metablock.hpp" -#include "utilities/globalDefinitions.hpp" - -namespace metaspace { - -typedef BinaryTreeDictionary > BlockTreeDictionary; - -// Used to manage the free list of Metablocks (a block corresponds -// to the allocation of a quantum of metadata). -class BlockFreelist : public CHeapObj { - - BlockTreeDictionary* const _dictionary; - SmallBlocks* _small_blocks; - - // Only allocate and split from freelist if the size of the allocation - // is at least 1/4th the size of the available block. - const static int WasteMultiplier = 4; - - // Accessors - BlockTreeDictionary* dictionary() const { return _dictionary; } - SmallBlocks* small_blocks() { - if (_small_blocks == NULL) { - _small_blocks = new SmallBlocks(); - } - return _small_blocks; - } - - public: - - BlockFreelist(); - ~BlockFreelist(); - - // Get and return a block to the free list - MetaWord* get_block(size_t word_size); - void return_block(MetaWord* p, size_t word_size); - - // Returns the total size, in words, of all blocks kept in this structure. - size_t total_size() const { - size_t result = dictionary()->total_size(); - if (_small_blocks != NULL) { - result = result + _small_blocks->total_size(); - } - return result; - } - - // Returns the number of all blocks kept in this structure. - uintx num_blocks() const { - uintx result = dictionary()->total_free_blocks(); - if (_small_blocks != NULL) { - result = result + _small_blocks->total_num_blocks(); - } - return result; - } - - static size_t min_dictionary_size() { return TreeChunk >::min_size(); } - void print_on(outputStream* st) const; -}; - -} // namespace metaspace - -#endif // SHARE_MEMORY_METASPACE_BLOCKFREELIST_HPP diff --git a/src/hotspot/share/memory/metaspace/blocktree.cpp b/src/hotspot/share/memory/metaspace/blocktree.cpp new file mode 100644 --- /dev/null +++ b/src/hotspot/share/memory/metaspace/blocktree.cpp @@ -0,0 +1,169 @@ +/* + * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020 SAP SE. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" + +#include "memory/metaspace/blocktree.hpp" +#include "utilities/debug.hpp" +#include "utilities/globalDefinitions.hpp" +#include "utilities/ostream.hpp" + +namespace metaspace { + + +#ifdef ASSERT + +// Tree verification + +// These asserts prints the tree, then asserts +#define assrt(cond, format, ...) \ + if (!(cond)) { \ + print_tree(tty); \ + assert(cond, format, __VA_ARGS__); \ + } + + // This assert prints the tree, then stops (generic message) +#define assrt0(cond) \ + if (!(cond)) { \ + print_tree(tty); \ + assert(cond, "sanity"); \ + } + +struct BlockTree::veri_data_t { + MemRangeCounter counter; + int max_edge; + size_t largest; +}; + +// Given a node, check that all siblings have the same size and that we have no +// (direct) circularities. +void BlockTree::verify_node_siblings(node_t* n, veri_data_t* vd) const { + const size_t size = n->size; + node_t* n2 = n->next; + node_t* prev_sib = NULL; + while (n2 != NULL) { + assrt0(n2->size == size); + vd->counter.add(n2->size); + if (prev_sib != NULL) { + assrt0(prev_sib->next == n2); + assrt0(prev_sib != n2); + } + prev_sib = n2; + n2 = n2->next; + } +} + +// Given a node and outer bounds applying to it and all children, check it and all children recursively. +void BlockTree::verify_node(node_t* n, size_t left_limit, size_t right_limit, + veri_data_t* vd, int lvl) const { + + if (lvl > vd->max_edge) { + vd->max_edge = lvl; + } + + if (n->size > vd->largest) { + vd->largest = n->size; + } + + assrt0((n == _root && n->parent == NULL) || (n != _root && n->parent != NULL)); + + // check all siblings + if (n->next != NULL) { + verify_node_siblings(n, vd); + } + + // check order + assrt(n->size >= minimal_word_size && n->size <= maximal_word_size, + "bad node size " SIZE_FORMAT, n->size); + assrt0(n->size < right_limit); + assrt0(n->size > left_limit); + + vd->counter.add(n->size); + + if (n->left != NULL) { + assrt0(n != n->left); + assrt0(n->left->parent == n); + assrt0(n->left->size < n->size); + assrt0(n->left->size > left_limit); + verify_node(n->left, left_limit, n->size, vd, lvl + 1); + } + + if (n->right != NULL) { + assrt0(n != n->right); + assrt0(n->right->parent == n); + assrt0(n->right->size < right_limit); + assrt0(n->right->size > n->size); + verify_node(n->right, n->size, right_limit, vd, lvl + 1); + } + +} + +void BlockTree::verify_tree() const { + int num = 0; + size_t size = 0; + veri_data_t vd; + vd.max_edge = 0; + vd.largest = 0; + if (_root != NULL) { + assrt0(_root->parent == NULL); + verify_node(_root, 0, maximal_word_size + 1, &vd, 0); + assrt0(vd.largest == _largest_size_added); + vd.counter.check(_counter); + assrt0(vd.counter.count() > 0); + } +} + +void BlockTree::zap_range(MetaWord* p, size_t word_size) { + memset(p, 0xF3, word_size * sizeof(MetaWord)); +} + +#undef assrt +#undef assrt0 + +#endif // ASSERT + + +void BlockTree::print_node(outputStream* st, node_t* n, int lvl) { + for (int i = 0; i < lvl; i ++) { + st->print("---"); + } + st->print_cr("<" PTR_FORMAT " (size " SIZE_FORMAT ")", p2i(n), n->size); + if (n->left) { + print_node(st, n->left, lvl + 1); + } + if (n->right) { + print_node(st, n->right, lvl + 1); + } +} + +void BlockTree::print_tree(outputStream* st) const { + if (_root != NULL) { + print_node(st, _root, 0); + } else { + st->print_cr(""); + } +} + +} // namespace metaspace diff --git a/src/hotspot/share/memory/metaspace/blocktree.hpp b/src/hotspot/share/memory/metaspace/blocktree.hpp new file mode 100644 --- /dev/null +++ b/src/hotspot/share/memory/metaspace/blocktree.hpp @@ -0,0 +1,448 @@ +/* + * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020 SAP SE. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_MEMORY_METASPACE_BLOCKTREE_HPP +#define SHARE_MEMORY_METASPACE_BLOCKTREE_HPP + +#include "memory/allocation.hpp" +#include "memory/metaspace/counter.hpp" +#include "utilities/debug.hpp" +#include "utilities/globalDefinitions.hpp" + +namespace metaspace { + + +// BlockTree is a rather simple binary search tree. It is used to +// manage small to medium free memory blocks (see class FreeBlocks). +// +// There is no separation between payload (managed blocks) and nodes: the +// memory blocks themselves are the nodes, with the block size being the key. +// +// We store node pointer information in these blocks when storing them. That +// imposes a minimum size to the managed memory blocks. +// See MetaspaceArene::get_raw_allocation_word_size(). +// +// We want to manage many memory blocks of the same size, but we want +// to prevent the tree from blowing up and degenerating into a list. Therefore +// there is only one node for each unique block size; subsequent blocks of the +// same size are stacked below that first node: +// +// +-----+ +// | 100 | +// +-----+ +// / \ +// +-----+ +// | 80 | +// +-----+ +// / | \ +// / +-----+ \ +// +-----+ | 80 | +-----+ +// | 70 | +-----+ | 85 | +// +-----+ | +-----+ +// +-----+ +// | 80 | +// +-----+ +// +// +// Todo: This tree is unbalanced. It would be a good fit for a red-black tree. +// In order to make this a red-black tree, we need an algorithm which can deal +// with nodes which are their own payload (most red-black tree implementations +// swap payloads of their nodes at some point, see e.g. j.u.TreeSet). +// A good example is the Linux kernel rbtree, which is a clean, easy-to-read +// implementation. + +class BlockTree: public CHeapObj { + + + struct node_t { + + // Normal tree node stuff... + node_t* parent; + node_t* left; + node_t* right; + + // blocks with the same size are put in a list with this node as head. + node_t* next; + + // word size of node. Note that size cannot be larger than max metaspace size, + // so this could be very well a 32bit value (in case we ever make this a balancing + // tree and need additional space for weighting information). + size_t size; + + }; + +public: + + // Largest node size, (bit arbitrarily) capped at 4M since we know this to + // be the max possible metaspace allocation size. TODO. Do this better. + const static size_t maximal_word_size = 4 * M; + + // We need nodes to be at least large enough to hold a node_t + const static size_t minimal_word_size = + (sizeof(node_t) + sizeof(MetaWord) - 1) / sizeof(MetaWord); + +private: + + node_t* _root; + + // As a performance optimization, we keep the size of the largest node. + size_t _largest_size_added; + + MemRangeCounter _counter; + + // given a node n, add it to the list starting at head + static void add_to_list(node_t* n, node_t* head) { + assert(head->size == n->size, "sanity"); + n->next = head->next; + head->next = n; + DEBUG_ONLY(n->left = n->right = n->parent = NULL;) + } + + // given a node list starting at head, remove one node from it and return it. + // List must contain at least one other node. + static node_t* remove_from_list(node_t* head) { + assert(head->next != NULL, "sanity"); + node_t* n = head->next; + if (n != NULL) { + head->next = n->next; + } + return n; + } + + // Given a node n and a node p, wire up n as left child of p. + static void set_left_child(node_t* p, node_t* c) { + p->left = c; + if (c != NULL) { + assert(c->size < p->size, "sanity"); + c->parent = p; + } + } + + // Given a node n and a node p, wire up n as right child of p. + static void set_right_child(node_t* p, node_t* c) { + p->right = c; + if (c != NULL) { + assert(c->size > p->size, "sanity"); + c->parent = p; + } + } + + // Given a node n, return its predecessor in the tree + // (node with the next-smaller size). + static node_t* predecessor(node_t* n) { + node_t* pred = NULL; + if (n->left != NULL) { + pred = n->left; + while (pred->right != NULL) { + pred = pred->right; + } + } else { + pred = n->parent; + node_t* n2 = n; + while (pred != NULL && n2 == pred->left) { + n2 = pred; + pred = pred->parent; + } + } + return pred; + } + + // Given a node n, return its predecessor in the tree + // (node with the next-smaller size). + static node_t* successor(node_t* n) { + node_t* succ = NULL; + if (n->right != NULL) { + // If there is a right child, search the left-most + // child of that child. + succ = n->right; + while (succ->left != NULL) { + succ = succ->left; + } + } else { + succ = n->parent; + node_t* n2 = n; + // As long as I am the right child of my parent, search upward + while (succ != NULL && n2 == succ->right) { + n2 = succ; + succ = succ->parent; + } + } + return succ; + } + + // Given a node, replace it with a replacement node as a child for its parent. + // If the node is root and has no parent, sets it as root. + void replace_node_in_parent(node_t* child, node_t* replace) { + node_t* parent = child->parent; + if (parent != NULL) { + if (parent->left == child) { // I am a left child + set_left_child(parent, replace); + } else { + set_right_child(parent, replace); + } + } else { + assert(child == _root, "must be root"); + _root = replace; + if (replace != NULL) { + replace->parent = NULL; + } + } + return; + } + + // Given a node n and a node forebear, insert n under forebear + void insert(node_t* forebear, node_t* n) { + if (n->size == forebear->size) { + add_to_list(n, forebear); // parent stays NULL in this case. + } else { + if (n->size < forebear->size) { + if (forebear->left == NULL) { + set_left_child(forebear, n); + } else { + insert(forebear->left, n); + } + } else { + assert(n->size > forebear->size, "sanity"); + if (forebear->right == NULL) { + set_right_child(forebear, n); + if (_largest_size_added < n->size) { + _largest_size_added = n->size; + } + } else { + insert(forebear->right, n); + } + } + } + } + + // Given a node and a wish size, search this node and all children for + // the node closest (equal or larger sized) to the size s. + static node_t* find_closest_fit(node_t* n, size_t s) { + + if (n->size == s) { + // Perfect fit. + return n; + + } else if (n->size < s) { + // too small, dive down right side + if (n->right != NULL) { + return find_closest_fit(n->right, s); + } else { + return NULL; + } + } else { + // n is a possible fit + assert(n->size > s, "Sanity"); + if (n->left != NULL && n->left->size >= s) { + // but not the best - dive down left side. + return find_closest_fit(n->left, s); + } else { + // n is the best fit. + return n; + } + } + + } + + // Given a wish size, search the whole tree for a + // node closest (equal or larger sized) to the size s. + node_t* find_closest_fit(size_t s) { + if (_root != NULL) { + return find_closest_fit(_root, s); + } + return NULL; + } + + // Given a node n, remove it from the tree and repair tree. + void remove_node_from_tree(node_t* n) { + + assert(n->next == NULL, "do not delete a node which has a non-empty list"); + + // Maintain largest size node to speed up lookup + if (n->size == _largest_size_added) { + node_t* pred = predecessor(n); + if (pred != NULL) { + _largest_size_added = pred->size; + } else { + _largest_size_added = 0; + } + } + + if (n->left == NULL && n->right == NULL) { + replace_node_in_parent(n, NULL); + + } else if (n->left == NULL && n->right != NULL) { + replace_node_in_parent(n, n->right); + + } else if (n->left != NULL && n->right == NULL) { + replace_node_in_parent(n, n->left); + + } else { + + // Node has two children. + + // 1) Find direct successor (the next larger node). + node_t* succ = successor(n); + + // There has to be a successor since n->right was != NULL... + assert(succ != NULL, "must be"); + + // ... and it should not have a left child since successor + // is supposed to be the next larger node, so it must be the mostleft node + // in the sub tree rooted at n->right + assert(succ->left == NULL, "must be"); + + assert(succ->size > n->size, "sanity"); + + node_t* successor_parent = succ->parent; + node_t* successor_right_child = succ->right; + + // Remove successor from its parent. + if (successor_parent == n) { + + // special case: successor is a direct child of n. Has to be the right child then. + assert(n->right == succ, "sanity"); + + // Just replace n with this successor. + replace_node_in_parent(n, succ); + + // Take over n's old left child, too. + // We keep the successor's right child. + set_left_child(succ, n->left); + + } else { + + // If the successors parent is not n, we are deeper in the tree, + // the successor has to be the left child of its parent. + assert(successor_parent->left == succ, "sanity"); + + // The right child of the successor (if there was one) replaces the successor at its parent's left child. + set_left_child(successor_parent, succ->right); + + // and the successor replaces n at its parent + replace_node_in_parent(n, succ); + + // and takes over n's old children + set_left_child(succ, n->left); + set_right_child(succ, n->right); + + } + } + } + +#ifdef ASSERT + + struct veri_data_t; + void verify_node_siblings(node_t* n, veri_data_t* vd) const; + void verify_node(node_t* n, size_t left_limit, size_t right_limit, veri_data_t* vd, int lvl) const; + void verify_tree() const; + + void zap_range(MetaWord* p, size_t word_size); + +#endif // ASSERT + + + static void print_node(outputStream* st, node_t* n, int lvl); + +public: + + BlockTree() : _root(NULL), _largest_size_added(0) {} + + // Add a memory block to the tree. Memory block will be used to store + // node information. + void add_block(MetaWord* p, size_t word_size) { + DEBUG_ONLY(zap_range(p, word_size)); + assert(word_size >= minimal_word_size && word_size < maximal_word_size, + "invalid block size " SIZE_FORMAT, word_size); + node_t* n = (node_t*)p; + n->size = word_size; + n->next = n->left = n->right = n->parent = NULL; + if (_root == NULL) { + _root = n; + } else { + insert(_root, n); + } + _counter.add(word_size); + + // Maintain largest node to speed up lookup + if (_largest_size_added < n->size) { + _largest_size_added = n->size; + } + + } + + // Given a word_size, searches and returns a block of at least that size. + // Block may be larger. Real block size is returned in *p_real_word_size. + MetaWord* get_block(size_t word_size, size_t* p_real_word_size) { + assert(word_size >= minimal_word_size && word_size < maximal_word_size, + "invalid block size " SIZE_FORMAT, word_size); + + if (_largest_size_added < word_size) { + return NULL; + } + + node_t* n = find_closest_fit(word_size); + + if (n != NULL) { + assert(n->size >= word_size, "sanity"); + + // If the node has siblings, remove one of them, + // otherwise remove this node from the tree. + if (n->next != NULL) { + n = remove_from_list(n); + } else { + remove_node_from_tree(n); + } + + MetaWord* p = (MetaWord*)n; + *p_real_word_size = n->size; + + _counter.sub(n->size); + + DEBUG_ONLY(zap_range(p, n->size)); + + return p; + } + return NULL; + } + + + // Returns number of blocks in this structure + unsigned count() const { return _counter.count(); } + + // Returns total size, in words, of all elements. + size_t total_size() const { return _counter.total_size(); } + + bool is_empty() const { return _root == NULL; } + + void print_tree(outputStream* st) const; + + DEBUG_ONLY(void verify() const { verify_tree(); }) + +}; + +} // namespace metaspace + +#endif // SHARE_MEMORY_METASPACE_BLOCKTREE_HPP diff --git a/src/hotspot/share/memory/metaspace/chunkHeaderPool.cpp b/src/hotspot/share/memory/metaspace/chunkHeaderPool.cpp new file mode 100644 --- /dev/null +++ b/src/hotspot/share/memory/metaspace/chunkHeaderPool.cpp @@ -0,0 +1,94 @@ +/* + * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020 SAP SE. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "memory/metaspace/chunkHeaderPool.hpp" +#include "runtime/os.hpp" +#include "utilities/debug.hpp" +#include "utilities/globalDefinitions.hpp" + +namespace metaspace { + + +// Returns reference to the one global chunk header pool. +ChunkHeaderPool* ChunkHeaderPool::_chunkHeaderPool = NULL; + + +ChunkHeaderPool::ChunkHeaderPool() + : _num_slabs(), _first_slab(NULL), _current_slab(NULL) +{ +} + +// Note: the global chunk header pool gets never deleted; so this destructor only +// exists for the sake of tests. +ChunkHeaderPool::~ChunkHeaderPool() { + slab_t* s = _first_slab; + while (s != NULL) { + slab_t* next_slab = s->next; + os::free(s); + s = next_slab; + } +} + +void ChunkHeaderPool::allocate_new_slab() { + slab_t* slab = new slab_t(); + if (_current_slab != NULL) { + _current_slab->next = slab; + } + _current_slab = slab; + if (_first_slab == NULL) { + _first_slab = slab; + } + _num_slabs.increment(); +} + +// Returns size of memory used. +size_t ChunkHeaderPool::memory_footprint_words() const { + return (_num_slabs.get() * sizeof(slab_t)) / BytesPerWord; +} + +void ChunkHeaderPool::initialize() { + assert(_chunkHeaderPool == NULL, "only once"); + _chunkHeaderPool = new ChunkHeaderPool(); +} + +#ifdef ASSERT +void ChunkHeaderPool::verify(bool slow) const { + const slab_t* s = _first_slab; + int num = 0; + while (s != NULL) { + assert(s->top >= 0 && s->top <= slab_capacity, + "invalid slab at " PTR_FORMAT ", top: %d, slab cap: %d", + p2i(s), s->top, slab_capacity ); + s = s->next; + num ++; + } + _num_slabs.check(num); +} +#endif + +} // namespace metaspace + + diff --git a/src/hotspot/share/memory/metaspace/chunkHeaderPool.hpp b/src/hotspot/share/memory/metaspace/chunkHeaderPool.hpp new file mode 100644 --- /dev/null +++ b/src/hotspot/share/memory/metaspace/chunkHeaderPool.hpp @@ -0,0 +1,149 @@ +/* + * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020 SAP SE. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_MEMORY_METASPACE_CHUNKHEADERPOOL_HPP +#define SHARE_MEMORY_METASPACE_CHUNKHEADERPOOL_HPP + +#include "memory/allocation.hpp" +#include "memory/metaspace/metachunk.hpp" +#include "memory/metaspace/metachunkList.hpp" +#include "utilities/debug.hpp" +#include "utilities/globalDefinitions.hpp" + +namespace metaspace { + +// Chunk headers (Metachunk objects) are separate entities from their payload. +// Since they are allocated and released frequently in the course of buddy allocation +// (splitting, merging chunks happens often) we want allocation of them fast. Therefore +// we keep them in a simple pool (somewhat like a primitive slab allocator). + +class ChunkHeaderPool : public CHeapObj { + + static const int slab_capacity = 128; + + struct slab_t : public CHeapObj { + slab_t* next; + int top; + Metachunk elems [slab_capacity]; + slab_t() : next(NULL), top(0) { + for (int i = 0; i < slab_capacity; i++) { + elems[i].clear(); + } + } + }; + + IntCounter _num_slabs; + slab_t* _first_slab; + slab_t* _current_slab; + + IntCounter _num_handed_out; + + MetachunkList _freelist; + + void allocate_new_slab(); + + static ChunkHeaderPool* _chunkHeaderPool; + + +public: + + ChunkHeaderPool(); + + ~ChunkHeaderPool(); + + // Allocates a Metachunk structure. The structure is uninitialized. + Metachunk* allocate_chunk_header() { + + Metachunk* c = NULL; + + DEBUG_ONLY(verify(false)); + + c = _freelist.remove_first(); + assert(c == NULL || c->is_dead(), "Not a freelist chunk header?"); + + if (c == NULL) { + + if (_current_slab == NULL || + _current_slab->top == slab_capacity) { + allocate_new_slab(); + assert(_current_slab->top < slab_capacity, "Sanity"); + } + + c = _current_slab->elems + _current_slab->top; + _current_slab->top ++; + + } + + _num_handed_out.increment(); + + // By contract, the returned structure is uninitialized. + // Zap to make this clear. + DEBUG_ONLY(c->zap_header(0xBB);) + + return c; + + } + + void return_chunk_header(Metachunk* c) { + // We only ever should return free chunks, since returning chunks + // happens only on merging and merging only works with free chunks. + assert(c != NULL && c->is_free(), "Sanity"); +#ifdef ASSERT + // In debug, fill dead header with pattern. + c->zap_header(0xCC); + c->set_next(NULL); + c->set_prev(NULL); +#endif + c->set_dead(); + _freelist.add(c); + _num_handed_out.decrement(); + + } + + // Returns number of allocated elements. + int used() const { return _num_handed_out.get(); } + + // Returns number of elements in free list. + int freelist_size() const { return _freelist.count(); } + + // Returns size of memory used. + size_t memory_footprint_words() const; + + DEBUG_ONLY(void verify(bool slow) const;) + + static void initialize(); + + // Returns reference to the one global chunk header pool. + static ChunkHeaderPool* pool() { return _chunkHeaderPool; } + +}; + + +} // namespace metaspace + + + + +#endif // SHARE_MEMORY_METASPACE_CHUNKHEADERPOOL_HPP diff --git a/src/hotspot/share/memory/metaspace/chunkLevel.cpp b/src/hotspot/share/memory/metaspace/chunkLevel.cpp new file mode 100644 --- /dev/null +++ b/src/hotspot/share/memory/metaspace/chunkLevel.cpp @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020 SAP SE. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "memory/metaspace/chunkLevel.hpp" +#include "utilities/debug.hpp" +#include "utilities/globalDefinitions.hpp" +#include "utilities/ostream.hpp" +#include "utilities/powerOfTwo.hpp" + +namespace metaspace { + +using namespace chunklevel; + +chunklevel_t chunklevel::level_fitting_word_size(size_t word_size) { + + assert(MAX_CHUNK_WORD_SIZE >= word_size, + SIZE_FORMAT " - too large allocation size.", word_size * BytesPerWord); + + if (word_size <= MIN_CHUNK_WORD_SIZE) { + return HIGHEST_CHUNK_LEVEL; + } + + const size_t v_pow2 = round_up_power_of_2(word_size); + const chunklevel_t lvl = (chunklevel_t)(exact_log2(MAX_CHUNK_WORD_SIZE) - exact_log2(v_pow2)); + + return lvl; + +} + +void chunklevel::print_chunk_size(outputStream* st, chunklevel_t lvl) { + if (chunklevel::is_valid_level(lvl)) { + const size_t s = chunklevel::word_size_for_level(lvl) * BytesPerWord; + if (s < 1 * M) { + st->print("%3uk", (unsigned)(s / K)); + } else { + st->print("%3um", (unsigned)(s / M)); + } + } else { + st->print("?-?"); + } + +} + +} // namespace metaspace + + diff --git a/src/hotspot/share/memory/metaspace/chunkLevel.hpp b/src/hotspot/share/memory/metaspace/chunkLevel.hpp new file mode 100644 --- /dev/null +++ b/src/hotspot/share/memory/metaspace/chunkLevel.hpp @@ -0,0 +1,132 @@ +/* + * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020 SAP SE. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_MEMORY_METASPACE_CHUNKLEVEL_HPP +#define SHARE_MEMORY_METASPACE_CHUNKLEVEL_HPP + +#include "utilities/globalDefinitions.hpp" + +// Constants for the chunk levels and some utility functions. + +class outputStream; + +namespace metaspace { + + +// Chunks are managed by a binary buddy allocator. + +// Chunk sizes range from 1K to 4MB (64bit). +// + +// Each chunk has a level; the level corresponds to its position in the tree +// and describes its size. +// +// The largest chunks are called root chunks, of 4MB in size, and have level 0. +// From there on it goes: +// +// size level +// 4MB 0 +// 2MB 1 +// 1MB 2 +// 512K 3 +// 256K 4 +// 128K 5 +// 64K 6 +// 32K 7 +// 16K 8 +// 8K 9 +// 4K 10 +// 2K 11 +// 1K 12 + +// Metachunk level (must be signed) +typedef signed char chunklevel_t; + +#define CHKLVL_FORMAT "lv%.2d" + +namespace chunklevel { + +static const size_t MAX_CHUNK_BYTE_SIZE = 4 * M; +static const int NUM_CHUNK_LEVELS = 13; +static const size_t MIN_CHUNK_BYTE_SIZE = (MAX_CHUNK_BYTE_SIZE >> ((size_t)NUM_CHUNK_LEVELS - 1)); + +static const size_t MIN_CHUNK_WORD_SIZE = MIN_CHUNK_BYTE_SIZE / sizeof(MetaWord); +static const size_t MAX_CHUNK_WORD_SIZE = MAX_CHUNK_BYTE_SIZE / sizeof(MetaWord); + +static const chunklevel_t ROOT_CHUNK_LEVEL = 0; + +static const chunklevel_t HIGHEST_CHUNK_LEVEL = NUM_CHUNK_LEVELS - 1; +static const chunklevel_t LOWEST_CHUNK_LEVEL = 0; + +static const chunklevel_t INVALID_CHUNK_LEVEL = (chunklevel_t) -1; + +inline bool is_valid_level(chunklevel_t level) { + return level >= LOWEST_CHUNK_LEVEL && + level <= HIGHEST_CHUNK_LEVEL; +} + +inline void check_valid_level(chunklevel_t lvl) { + assert(is_valid_level(lvl), "invalid level (%d)", (int)lvl); +} + +// Given a level return the chunk size, in words. +inline size_t word_size_for_level(chunklevel_t level) { + return (MAX_CHUNK_BYTE_SIZE >> level) / BytesPerWord; +} + +// Given an arbitrary word size smaller than the highest chunk size, +// return the highest chunk level able to hold this size. +// Returns INVALID_CHUNK_LEVEL if no fitting level can be found. +chunklevel_t level_fitting_word_size(size_t word_size); + +// Shorthands to refer to exact sizes +static const chunklevel_t CHUNK_LEVEL_4M = ROOT_CHUNK_LEVEL; +static const chunklevel_t CHUNK_LEVEL_2M = (ROOT_CHUNK_LEVEL + 1); +static const chunklevel_t CHUNK_LEVEL_1M = (ROOT_CHUNK_LEVEL + 2); +static const chunklevel_t CHUNK_LEVEL_512K = (ROOT_CHUNK_LEVEL + 3); +static const chunklevel_t CHUNK_LEVEL_256K = (ROOT_CHUNK_LEVEL + 4); +static const chunklevel_t CHUNK_LEVEL_128K = (ROOT_CHUNK_LEVEL + 5); +static const chunklevel_t CHUNK_LEVEL_64K = (ROOT_CHUNK_LEVEL + 6); +static const chunklevel_t CHUNK_LEVEL_32K = (ROOT_CHUNK_LEVEL + 7); +static const chunklevel_t CHUNK_LEVEL_16K = (ROOT_CHUNK_LEVEL + 8); +static const chunklevel_t CHUNK_LEVEL_8K = (ROOT_CHUNK_LEVEL + 9); +static const chunklevel_t CHUNK_LEVEL_4K = (ROOT_CHUNK_LEVEL + 10); +static const chunklevel_t CHUNK_LEVEL_2K = (ROOT_CHUNK_LEVEL + 11); +static const chunklevel_t CHUNK_LEVEL_1K = (ROOT_CHUNK_LEVEL + 12); + +STATIC_ASSERT(CHUNK_LEVEL_1K == HIGHEST_CHUNK_LEVEL); +STATIC_ASSERT(CHUNK_LEVEL_4M == LOWEST_CHUNK_LEVEL); +STATIC_ASSERT(ROOT_CHUNK_LEVEL == LOWEST_CHUNK_LEVEL); + +///////////////////////////////////////////////////////// +// print helpers +void print_chunk_size(outputStream* st, chunklevel_t lvl); + + +} // namespace chunklevel + +} // namespace metaspace + +#endif // SHARE_MEMORY_METASPACE_CHUNKLEVEL_HPP diff --git a/src/hotspot/share/memory/metaspace/chunkManager.cpp b/src/hotspot/share/memory/metaspace/chunkManager.cpp --- a/src/hotspot/share/memory/metaspace/chunkManager.cpp +++ b/src/hotspot/share/memory/metaspace/chunkManager.cpp @@ -1,5 +1,6 @@ /* - * Copyright (c) 2018, 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2018, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2018, 2020 SAP SE. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -21,622 +22,458 @@ * questions. * */ + #include "precompiled.hpp" + #include "logging/log.hpp" #include "logging/logStream.hpp" -#include "memory/binaryTreeDictionary.inline.hpp" -#include "memory/freeList.inline.hpp" +#include "memory/metaspace/arenaGrowthPolicy.hpp" +#include "memory/metaspace/chunkLevel.hpp" #include "memory/metaspace/chunkManager.hpp" +#include "memory/metaspace/internStat.hpp" #include "memory/metaspace/metachunk.hpp" -#include "memory/metaspace/metaDebug.hpp" #include "memory/metaspace/metaspaceCommon.hpp" +#include "memory/metaspace/metaspaceContext.hpp" #include "memory/metaspace/metaspaceStatistics.hpp" -#include "memory/metaspace/occupancyMap.hpp" +#include "memory/metaspace/settings.hpp" #include "memory/metaspace/virtualSpaceNode.hpp" +#include "memory/metaspace/virtualSpaceList.hpp" #include "runtime/mutexLocker.hpp" #include "utilities/debug.hpp" #include "utilities/globalDefinitions.hpp" -#include "utilities/ostream.hpp" namespace metaspace { -ChunkManager::ChunkManager(bool is_class) - : _is_class(is_class), _free_chunks_total(0), _free_chunks_count(0) { - _free_chunks[SpecializedIndex].set_size(get_size_for_nonhumongous_chunktype(SpecializedIndex, is_class)); - _free_chunks[SmallIndex].set_size(get_size_for_nonhumongous_chunktype(SmallIndex, is_class)); - _free_chunks[MediumIndex].set_size(get_size_for_nonhumongous_chunktype(MediumIndex, is_class)); -} +#define LOGFMT "ChkMgr @" PTR_FORMAT " (%s)" +#define LOGFMT_ARGS p2i(this), this->_name -void ChunkManager::remove_chunk(Metachunk* chunk) { - size_t word_size = chunk->word_size(); - ChunkIndex index = list_index(word_size); - if (index != HumongousIndex) { - free_chunks(index)->remove_chunk(chunk); - } else { - humongous_dictionary()->remove_chunk(chunk); - } - - // Chunk has been removed from the chunks free list, update counters. - account_for_removed_chunk(chunk); -} - -bool ChunkManager::attempt_to_coalesce_around_chunk(Metachunk* chunk, ChunkIndex target_chunk_type) { - assert_lock_strong(MetaspaceExpand_lock); - assert(chunk != NULL, "invalid chunk pointer"); - // Check for valid merge combinations. - assert((chunk->get_chunk_type() == SpecializedIndex && - (target_chunk_type == SmallIndex || target_chunk_type == MediumIndex)) || - (chunk->get_chunk_type() == SmallIndex && target_chunk_type == MediumIndex), - "Invalid chunk merge combination."); - - const size_t target_chunk_word_size = - get_size_for_nonhumongous_chunktype(target_chunk_type, this->is_class()); - - // [ prospective merge region ) - MetaWord* const p_merge_region_start = - (MetaWord*) align_down(chunk, target_chunk_word_size * sizeof(MetaWord)); - MetaWord* const p_merge_region_end = - p_merge_region_start + target_chunk_word_size; - - // We need the VirtualSpaceNode containing this chunk and its occupancy map. - VirtualSpaceNode* const vsn = chunk->container(); - OccupancyMap* const ocmap = vsn->occupancy_map(); - - // The prospective chunk merge range must be completely contained by the - // committed range of the virtual space node. - if (p_merge_region_start < vsn->bottom() || p_merge_region_end > vsn->top()) { - return false; - } - - // Only attempt to merge this range if at its start a chunk starts and at its end - // a chunk ends. If a chunk (can only be humongous) straddles either start or end - // of that range, we cannot merge. - if (!ocmap->chunk_starts_at_address(p_merge_region_start)) { - return false; - } - if (p_merge_region_end < vsn->top() && - !ocmap->chunk_starts_at_address(p_merge_region_end)) { - return false; - } - - // Now check if the prospective merge area contains live chunks. If it does we cannot merge. - if (ocmap->is_region_in_use(p_merge_region_start, target_chunk_word_size)) { - return false; - } - - // Success! Remove all chunks in this region... - log_trace(gc, metaspace, freelist)("%s: coalescing chunks in area [%p-%p)...", - (is_class() ? "class space" : "metaspace"), - p_merge_region_start, p_merge_region_end); - - const int num_chunks_removed = - remove_chunks_in_area(p_merge_region_start, target_chunk_word_size); - - // ... and create a single new bigger chunk. - Metachunk* const p_new_chunk = - ::new (p_merge_region_start) Metachunk(target_chunk_type, is_class(), target_chunk_word_size, vsn); - assert(p_new_chunk == (Metachunk*)p_merge_region_start, "Sanity"); - p_new_chunk->set_origin(origin_merge); - - log_trace(gc, metaspace, freelist)("%s: created coalesced chunk at %p, size " SIZE_FORMAT_HEX ".", - (is_class() ? "class space" : "metaspace"), - p_new_chunk, p_new_chunk->word_size() * sizeof(MetaWord)); - - // Fix occupancy map: remove old start bits of the small chunks and set new start bit. - ocmap->wipe_chunk_start_bits_in_region(p_merge_region_start, target_chunk_word_size); - ocmap->set_chunk_starts_at_address(p_merge_region_start, true); - - // Mark chunk as free. Note: it is not necessary to update the occupancy - // map in-use map, because the old chunks were also free, so nothing - // should have changed. - p_new_chunk->set_is_tagged_free(true); - - // Add new chunk to its freelist. - ChunkList* const list = free_chunks(target_chunk_type); - list->return_chunk_at_head(p_new_chunk); - - // And adjust ChunkManager:: _free_chunks_count (_free_chunks_total - // should not have changed, because the size of the space should be the same) - _free_chunks_count -= num_chunks_removed; - _free_chunks_count ++; - - // VirtualSpaceNode::chunk_count does not have to be modified: - // it means "number of active (non-free) chunks", so merging free chunks - // should not affect that count. - - // At the end of a chunk merge, run verification tests. -#ifdef ASSERT - - EVERY_NTH(VerifyMetaspaceInterval) - locked_verify(true); - vsn->verify(true); - END_EVERY_NTH - - g_internal_statistics.num_chunk_merges ++; - -#endif - - return true; -} - -// Remove all chunks in the given area - the chunks are supposed to be free - -// from their corresponding freelists. Mark them as invalid. -// - This does not correct the occupancy map. -// - This does not adjust the counters in ChunkManager. -// - Does not adjust container count counter in containing VirtualSpaceNode -// Returns number of chunks removed. -int ChunkManager::remove_chunks_in_area(MetaWord* p, size_t word_size) { - assert(p != NULL && word_size > 0, "Invalid range."); - const size_t smallest_chunk_size = get_size_for_nonhumongous_chunktype(SpecializedIndex, is_class()); - assert_is_aligned(word_size, smallest_chunk_size); - - Metachunk* const start = (Metachunk*) p; - const Metachunk* const end = (Metachunk*)(p + word_size); - Metachunk* cur = start; - int num_removed = 0; - while (cur < end) { - Metachunk* next = (Metachunk*)(((MetaWord*)cur) + cur->word_size()); - DEBUG_ONLY(do_verify_chunk(cur)); - assert(cur->get_chunk_type() != HumongousIndex, "Unexpected humongous chunk found at %p.", cur); - assert(cur->is_tagged_free(), "Chunk expected to be free (%p)", cur); - log_trace(gc, metaspace, freelist)("%s: removing chunk %p, size " SIZE_FORMAT_HEX ".", - (is_class() ? "class space" : "metaspace"), - cur, cur->word_size() * sizeof(MetaWord)); - cur->remove_sentinel(); - // Note: cannot call ChunkManager::remove_chunk, because that - // modifies the counters in ChunkManager, which we do not want. So - // we call remove_chunk on the freelist directly (see also the - // splitting function which does the same). - ChunkList* const list = free_chunks(list_index(cur->word_size())); - list->remove_chunk(cur); - num_removed ++; - cur = next; - } - return num_removed; -} - -// Update internal accounting after a chunk was added -void ChunkManager::account_for_added_chunk(const Metachunk* c) { - assert_lock_strong(MetaspaceExpand_lock); - _free_chunks_count ++; - _free_chunks_total += c->word_size(); -} - -// Update internal accounting after a chunk was removed -void ChunkManager::account_for_removed_chunk(const Metachunk* c) { - assert_lock_strong(MetaspaceExpand_lock); - assert(_free_chunks_count >= 1, - "ChunkManager::_free_chunks_count: about to go negative (" SIZE_FORMAT ").", _free_chunks_count); - assert(_free_chunks_total >= c->word_size(), - "ChunkManager::_free_chunks_total: about to go negative" - "(now: " SIZE_FORMAT ", decrement value: " SIZE_FORMAT ").", _free_chunks_total, c->word_size()); - _free_chunks_count --; - _free_chunks_total -= c->word_size(); -} - -ChunkIndex ChunkManager::list_index(size_t size) { - return get_chunk_type_by_size(size, is_class()); -} - -size_t ChunkManager::size_by_index(ChunkIndex index) const { - index_bounds_check(index); - assert(index != HumongousIndex, "Do not call for humongous chunks."); - return get_size_for_nonhumongous_chunktype(index, is_class()); -} - -#ifdef ASSERT -void ChunkManager::verify(bool slow) const { - MutexLocker cl(MetaspaceExpand_lock, - Mutex::_no_safepoint_check_flag); - locked_verify(slow); -} - -void ChunkManager::locked_verify(bool slow) const { - log_trace(gc, metaspace, freelist)("verifying %s chunkmanager (%s).", - (is_class() ? "class space" : "metaspace"), (slow ? "slow" : "quick")); +// Return a single chunk to the freelist and adjust accounting. No merge is attempted. +void ChunkManager::return_chunk_simple_locked(Metachunk* c) { assert_lock_strong(MetaspaceExpand_lock); - size_t chunks_counted = 0; - size_t wordsize_chunks_counted = 0; - for (ChunkIndex i = ZeroIndex; i < NumberOfFreeLists; i = next_chunk_index(i)) { - const ChunkList* list = _free_chunks + i; - if (list != NULL) { - Metachunk* chunk = list->head(); - while (chunk) { - if (slow) { - do_verify_chunk(chunk); - } - assert(chunk->is_tagged_free(), "Chunk should be tagged as free."); - chunks_counted ++; - wordsize_chunks_counted += chunk->size(); - chunk = chunk->next(); + DEBUG_ONLY(c->verify(false)); + + const chunklevel_t lvl = c->level(); + _chunks.add(c); + c->reset_used_words(); + + // Tracing + log_debug(metaspace)("ChunkManager %s: returned chunk " METACHUNK_FORMAT ".", + _name, METACHUNK_FORMAT_ARGS(c)); + +} + +// Creates a chunk manager with a given name (which is for debug purposes only) +// and an associated space list which will be used to request new chunks from +// (see get_chunk()) +ChunkManager::ChunkManager(const char* name, VirtualSpaceList* space_list) + : _vslist(space_list), + _name(name), + _chunks() +{ +} + +// Given a chunk, split it into a target chunk of a smaller size (higher target level) +// and at least one, possible several splinter chunks. +// The original chunk must be outside of the freelist and its state must be free. +// The splinter chunks are added to the freelist. +// The resulting target chunk will be located at the same address as the original +// chunk, but it will of course be smaller (of a higher level). +// The committed areas within the original chunk carry over to the resulting +// chunks. +void ChunkManager::split_chunk_and_add_splinters(Metachunk* c, chunklevel_t target_level) { + + assert_lock_strong(MetaspaceExpand_lock); + + assert(c->is_free(), "chunk to be split must be free."); + assert(c->level() < target_level, "Target level must be higher than current level."); + assert(c->prev() == NULL && c->next() == NULL, "Chunk must be outside of any list."); + + DEBUG_ONLY(chunklevel::check_valid_level(target_level);) + DEBUG_ONLY(c->verify(true);) + + UL2(debug, "splitting chunk " METACHUNK_FORMAT " to " CHKLVL_FORMAT ".", + METACHUNK_FORMAT_ARGS(c), target_level); + + DEBUG_ONLY(size_t committed_words_before = c->committed_words();) + + const chunklevel_t orig_level = c->level(); + c->vsnode()->split(target_level, c, &_chunks); + + // Splitting should never fail. + assert(c->level() == target_level, "Sanity"); + + // The size of the committed portion should not change (subject to the reduced chunk size of course) +#ifdef ASSERT + if (committed_words_before > c->word_size()) { + assert(c->is_fully_committed(), "Sanity"); + } else { + assert(c->committed_words() == committed_words_before, "Sanity"); + } +#endif + + DEBUG_ONLY(c->verify(false)); + + DEBUG_ONLY(verify_locked(true);) + + SOMETIMES(c->vsnode()->verify_locked(true);) + + InternalStats::inc_num_chunk_splits(); + +} + +// On success, returns a chunk of level of , but at most . +// The first first of the chunk are guaranteed to be committed. +// On error, will return NULL. +// +// This function may fail for two reasons: +// - Either we are unable to reserve space for a new chunk (if the underlying VirtualSpaceList +// is non-expandable but needs expanding - aka out of compressed class space). +// - Or, if the necessary space cannot be committed because we hit a commit limit. +// This may be either the GC threshold or MaxMetaspaceSize. +Metachunk* ChunkManager::get_chunk(chunklevel_t preferred_level, chunklevel_t max_level, size_t min_committed_words) { + + assert(preferred_level <= max_level, "Sanity"); + assert(chunklevel::level_fitting_word_size(min_committed_words) >= max_level, "Sanity"); + + MutexLocker fcl(MetaspaceExpand_lock, Mutex::_no_safepoint_check_flag); + + DEBUG_ONLY(verify_locked(false);) + + DEBUG_ONLY(chunklevel::check_valid_level(max_level);) + DEBUG_ONLY(chunklevel::check_valid_level(preferred_level);) + assert(max_level >= preferred_level, "invalid level."); + + UL2(debug, "requested chunk: pref_level: " CHKLVL_FORMAT + ", max_level: " CHKLVL_FORMAT ", min committed size: " SIZE_FORMAT ".", + preferred_level, max_level, min_committed_words); + + // First, optimistically look for a chunk which is already committed far enough to hold min_word_size. + + // 1) Search best or smaller committed chunks (first attempt): + // Start at the preferred chunk size and work your way down (level up). + // But for now, only consider chunks larger than a certain threshold - + // this is to prevent large loaders (eg boot) from unnecessarily gobbling up + // all the tiny splinter chunks lambdas leave around. + Metachunk* c = NULL; + c = _chunks.search_chunk_ascending(preferred_level, MIN2((chunklevel_t)(preferred_level + 2), max_level), min_committed_words); + + // 2) Search larger committed chunks: + // If that did not yield anything, look at larger chunks, which may be committed. We would have to split + // them first, of course. + if (c == NULL) { + c = _chunks.search_chunk_descending(preferred_level, min_committed_words); + } + + // 3) Search best or smaller committed chunks (second attempt): + // Repeat (1) but now consider even the tiniest chunks as long as they are large enough to hold the + // committed min size. + if (c == NULL) { + c = _chunks.search_chunk_ascending(preferred_level, max_level, min_committed_words); + } + + // if we did not get anything yet, there are no free chunks commmitted enough. Repeat search but look for uncommitted chunks too: + + // 4) Search best or smaller chunks, can be uncommitted: + if (c == NULL) { + c = _chunks.search_chunk_ascending(preferred_level, max_level, 0); + } + + // 5) Search a larger uncommitted chunk: + if (c == NULL) { + c = _chunks.search_chunk_descending(preferred_level, 0); + } + + if (c != NULL) { + UL(trace, "taken from freelist."); + } + + // Failing all that, allocate a new root chunk from the connected virtual space. + // This may fail if the underlying vslist cannot be expanded (e.g. compressed class space) + if (c == NULL) { + c = _vslist->allocate_root_chunk(); + if (c == NULL) { + UL(info, "failed to get new root chunk."); + } else { + assert(c->level() == chunklevel::ROOT_CHUNK_LEVEL, "root chunk expected"); + UL(debug, "allocated new root chunk."); + } + } + + if (c == NULL) { + // If we end up here, we found no match in the freelists and were unable to get a new + // root chunk (so we used up all address space, e.g. out of CompressedClassSpace). + UL2(info, "failed to get chunk (preferred level: " CHKLVL_FORMAT + ", max level " CHKLVL_FORMAT ".", preferred_level, max_level); + c = NULL; + } + + if (c != NULL) { + + // Now we have a chunk. + // It may be larger than what the caller wanted, so we may want to split it. This should + // always work. + if (c->level() < preferred_level) { + split_chunk_and_add_splinters(c, preferred_level); + assert(c->level() == preferred_level, "split failed?"); + } + + // Attempt to commit the chunk (depending on settings, we either fully commit it or just + // commit enough to get the caller going). That may fail if we hit a commit limit. In + // that case put the chunk back to the freelist (re-merging it with its neighbors if we + // did split it) and return NULL. + const size_t to_commit = Settings::new_chunks_are_fully_committed() ? c->word_size() : min_committed_words; + if (c->committed_words() < to_commit) { + if (c->ensure_committed_locked(to_commit) == false) { + UL2(info, "failed to commit " SIZE_FORMAT " words on chunk " METACHUNK_FORMAT ".", + to_commit, METACHUNK_FORMAT_ARGS(c)); + c->set_in_use(); // gets asserted in return_chunk(). + return_chunk_locked(c); + c = NULL; + } + } + + if (c != NULL) { + + // Still here? We have now a good chunk, all is well. + assert(c->committed_words() >= min_committed_words, "Sanity"); + + // Any chunk returned from ChunkManager shall be marked as in use. + c->set_in_use(); + + UL2(debug, "handing out chunk " METACHUNK_FORMAT ".", METACHUNK_FORMAT_ARGS(c)); + + InternalStats::inc_num_chunks_taken_from_freelist(); + + SOMETIMES(c->vsnode()->verify_locked(true);) + + } + + } + + DEBUG_ONLY(verify_locked(false);) + + return c; + +} + + +// Return a single chunk to the ChunkManager and adjust accounting. May merge chunk +// with neighbors. +// As a side effect this removes the chunk from whatever list it has been in previously. +// Happens after a Classloader was unloaded and releases its metaspace chunks. +// !! Note: this may invalidate the chunk. Do not access the chunk after +// this function returns !! +void ChunkManager::return_chunk(Metachunk* c) { + MutexLocker fcl(MetaspaceExpand_lock, Mutex::_no_safepoint_check_flag); + return_chunk_locked(c); +} + +// See return_chunk(). +void ChunkManager::return_chunk_locked(Metachunk* c) { + + assert_lock_strong(MetaspaceExpand_lock); + + UL2(debug, ": returning chunk " METACHUNK_FORMAT ".", METACHUNK_FORMAT_ARGS(c)); + + DEBUG_ONLY(c->verify(true);) + + assert(contains_chunk(c) == false, "A chunk to be added to the freelist must not be in the freelist already."); + + assert(c->is_in_use(), "Unexpected chunk state"); + assert(!c->in_list(), "Remove from list first"); + c->set_free(); + c->reset_used_words(); + + const chunklevel_t orig_lvl = c->level(); + + Metachunk* merged = NULL; + if (!c->is_root_chunk()) { + // Only attempt merging if we are not of the lowest level already. + merged = c->vsnode()->merge(c, &_chunks); + } + + if (merged != NULL) { + + InternalStats::inc_num_chunk_merges(); + + DEBUG_ONLY(merged->verify(false)); + + // We did merge our chunk into a different chunk. + + // We did merge chunks and now have a bigger chunk. + assert(merged->level() < orig_lvl, "Sanity"); + + UL2(debug, "merged into chunk " METACHUNK_FORMAT ".", METACHUNK_FORMAT_ARGS(merged)); + + c = merged; + + } + + if (Settings::uncommit_free_chunks() && + c->word_size() >= Settings::commit_granule_words()) + { + UL2(debug, "uncommitting free chunk " METACHUNK_FORMAT ".", METACHUNK_FORMAT_ARGS(c)); + c->uncommit_locked(); + } + + return_chunk_simple_locked(c); + + DEBUG_ONLY(verify_locked(false);) + SOMETIMES(c->vsnode()->verify_locked(true);) + + InternalStats::inc_num_chunks_returned_to_freelist(); + +} + +// Given a chunk c, whose state must be "in-use" and must not be a root chunk, attempt to +// enlarge it in place by claiming its trailing buddy. +// +// This will only work if c is the leader of the buddy pair and the trailing buddy is free. +// +// If successful, the follower chunk will be removed from the freelists, the leader chunk c will +// double in size (level decreased by one). +// +// On success, true is returned, false otherwise. +bool ChunkManager::attempt_enlarge_chunk(Metachunk* c) { + MutexLocker fcl(MetaspaceExpand_lock, Mutex::_no_safepoint_check_flag); + return c->vsnode()->attempt_enlarge_chunk(c, &_chunks); +} + +static void print_word_size_delta(outputStream* st, size_t word_size_1, size_t word_size_2) { + if (word_size_1 == word_size_2) { + print_scaled_words(st, word_size_1); + st->print (" (no change)"); + } else { + print_scaled_words(st, word_size_1); + st->print("->"); + print_scaled_words(st, word_size_2); + st->print(" ("); + if (word_size_2 <= word_size_1) { + st->print("-"); + print_scaled_words(st, word_size_1 - word_size_2); + } else { + st->print("+"); + print_scaled_words(st, word_size_2 - word_size_1); + } + st->print(")"); + } +} + +void ChunkManager::purge() { + + MutexLocker fcl(MetaspaceExpand_lock, Mutex::_no_safepoint_check_flag); + + UL(info, ": reclaiming memory..."); + + const size_t reserved_before = _vslist->reserved_words(); + const size_t committed_before = _vslist->committed_words(); + int num_nodes_purged = 0; + + // 1) purge virtual space list + num_nodes_purged = _vslist->purge(&_chunks); + InternalStats::inc_num_purges(); + + // 2) uncommit free chunks + if (Settings::uncommit_free_chunks()) { + const chunklevel_t max_level = + chunklevel::level_fitting_word_size(Settings::commit_granule_words()); + for (chunklevel_t l = chunklevel::LOWEST_CHUNK_LEVEL; + l <= max_level; + l ++) + { + // Since we uncommit all chunks at this level, we do not break the "committed chunks are + // at the front of the list" condition. + for (Metachunk* c = _chunks.first_at_level(l); c != NULL; c = c->next()) { + c->uncommit_locked(); } } } - chunks_counted += humongous_dictionary()->total_free_blocks(); - wordsize_chunks_counted += humongous_dictionary()->total_size(); + const size_t reserved_after = _vslist->reserved_words(); + const size_t committed_after = _vslist->committed_words(); - assert(chunks_counted == _free_chunks_count && wordsize_chunks_counted == _free_chunks_total, - "freelist accounting mismatch: " - "we think: " SIZE_FORMAT " chunks, total " SIZE_FORMAT " words, " - "reality: " SIZE_FORMAT " chunks, total " SIZE_FORMAT " words.", - _free_chunks_count, _free_chunks_total, - chunks_counted, wordsize_chunks_counted); -} -#endif // ASSERT + // Print a nice report. + if (reserved_after == reserved_before && committed_after == committed_before) { + UL(info, "nothing reclaimed."); + } else { + LogTarget(Info, metaspace) lt; + if (lt.is_enabled()) { + LogStream ls(lt); + ls.print_cr(LOGFMT ": finished reclaiming memory: ", LOGFMT_ARGS); -void ChunkManager::locked_print_free_chunks(outputStream* st) { - assert_lock_strong(MetaspaceExpand_lock); - st->print_cr("Free chunk total " SIZE_FORMAT " count " SIZE_FORMAT, - _free_chunks_total, _free_chunks_count); -} + ls.print("reserved: "); + print_word_size_delta(&ls, reserved_before, reserved_after); + ls.cr(); -ChunkList* ChunkManager::free_chunks(ChunkIndex index) { - assert(index == SpecializedIndex || index == SmallIndex || index == MediumIndex, - "Bad index: %d", (int)index); - return &_free_chunks[index]; -} + ls.print("committed: "); + print_word_size_delta(&ls, committed_before, committed_after); + ls.cr(); -ChunkList* ChunkManager::find_free_chunks_list(size_t word_size) { - ChunkIndex index = list_index(word_size); - assert(index < HumongousIndex, "No humongous list"); - return free_chunks(index); -} - -// Helper for chunk splitting: given a target chunk size and a larger free chunk, -// split up the larger chunk into n smaller chunks, at least one of which should be -// the target chunk of target chunk size. The smaller chunks, including the target -// chunk, are returned to the freelist. The pointer to the target chunk is returned. -// Note that this chunk is supposed to be removed from the freelist right away. -Metachunk* ChunkManager::split_chunk(size_t target_chunk_word_size, Metachunk* larger_chunk) { - assert(larger_chunk->word_size() > target_chunk_word_size, "Sanity"); - - const ChunkIndex larger_chunk_index = larger_chunk->get_chunk_type(); - const ChunkIndex target_chunk_index = get_chunk_type_by_size(target_chunk_word_size, is_class()); - - MetaWord* const region_start = (MetaWord*)larger_chunk; - const size_t region_word_len = larger_chunk->word_size(); - MetaWord* const region_end = region_start + region_word_len; - VirtualSpaceNode* const vsn = larger_chunk->container(); - OccupancyMap* const ocmap = vsn->occupancy_map(); - - // Any larger non-humongous chunk size is a multiple of any smaller chunk size. - // Since non-humongous chunks are aligned to their chunk size, the larger chunk should start - // at an address suitable to place the smaller target chunk. - assert_is_aligned(region_start, target_chunk_word_size); - - // Remove old chunk. - free_chunks(larger_chunk_index)->remove_chunk(larger_chunk); - larger_chunk->remove_sentinel(); - - // Prevent access to the old chunk from here on. - larger_chunk = NULL; - // ... and wipe it. - DEBUG_ONLY(memset(region_start, 0xfe, region_word_len * BytesPerWord)); - - // In its place create first the target chunk... - MetaWord* p = region_start; - Metachunk* target_chunk = ::new (p) Metachunk(target_chunk_index, is_class(), target_chunk_word_size, vsn); - assert(target_chunk == (Metachunk*)p, "Sanity"); - target_chunk->set_origin(origin_split); - - // Note: we do not need to mark its start in the occupancy map - // because it coincides with the old chunk start. - - // Mark chunk as free and return to the freelist. - do_update_in_use_info_for_chunk(target_chunk, false); - free_chunks(target_chunk_index)->return_chunk_at_head(target_chunk); - - // This chunk should now be valid and can be verified. - DEBUG_ONLY(do_verify_chunk(target_chunk)); - - // In the remaining space create the remainder chunks. - p += target_chunk->word_size(); - assert(p < region_end, "Sanity"); - - while (p < region_end) { - - // Find the largest chunk size which fits the alignment requirements at address p. - ChunkIndex this_chunk_index = prev_chunk_index(larger_chunk_index); - size_t this_chunk_word_size = 0; - for(;;) { - this_chunk_word_size = get_size_for_nonhumongous_chunktype(this_chunk_index, is_class()); - if (is_aligned(p, this_chunk_word_size * BytesPerWord)) { - break; - } else { - this_chunk_index = prev_chunk_index(this_chunk_index); - assert(this_chunk_index >= target_chunk_index, "Sanity"); - } - } - - assert(this_chunk_word_size >= target_chunk_word_size, "Sanity"); - assert(is_aligned(p, this_chunk_word_size * BytesPerWord), "Sanity"); - assert(p + this_chunk_word_size <= region_end, "Sanity"); - - // Create splitting chunk. - Metachunk* this_chunk = ::new (p) Metachunk(this_chunk_index, is_class(), this_chunk_word_size, vsn); - assert(this_chunk == (Metachunk*)p, "Sanity"); - this_chunk->set_origin(origin_split); - ocmap->set_chunk_starts_at_address(p, true); - do_update_in_use_info_for_chunk(this_chunk, false); - - // This chunk should be valid and can be verified. - DEBUG_ONLY(do_verify_chunk(this_chunk)); - - // Return this chunk to freelist and correct counter. - free_chunks(this_chunk_index)->return_chunk_at_head(this_chunk); - _free_chunks_count ++; - - log_trace(gc, metaspace, freelist)("Created chunk at " PTR_FORMAT ", word size " - SIZE_FORMAT_HEX " (%s), in split region [" PTR_FORMAT "..." PTR_FORMAT ").", - p2i(this_chunk), this_chunk->word_size(), chunk_size_name(this_chunk_index), - p2i(region_start), p2i(region_end)); - - p += this_chunk_word_size; - - } - - // Note: at this point, the VirtualSpaceNode is invalid since we split a chunk and - // did not yet hand out part of that split; so, vsn->verify_free_chunks_are_ideally_merged() - // would assert. Instead, do all verifications in the caller. - - DEBUG_ONLY(g_internal_statistics.num_chunk_splits ++); - - return target_chunk; -} - -Metachunk* ChunkManager::free_chunks_get(size_t word_size) { - assert_lock_strong(MetaspaceExpand_lock); - - Metachunk* chunk = NULL; - bool we_did_split_a_chunk = false; - - if (list_index(word_size) != HumongousIndex) { - - ChunkList* free_list = find_free_chunks_list(word_size); - assert(free_list != NULL, "Sanity check"); - - chunk = free_list->head(); - - if (chunk == NULL) { - // Split large chunks into smaller chunks if there are no smaller chunks, just large chunks. - // This is the counterpart of the coalescing-upon-chunk-return. - - ChunkIndex target_chunk_index = get_chunk_type_by_size(word_size, is_class()); - - // Is there a larger chunk we could split? - Metachunk* larger_chunk = NULL; - ChunkIndex larger_chunk_index = next_chunk_index(target_chunk_index); - while (larger_chunk == NULL && larger_chunk_index < NumberOfFreeLists) { - larger_chunk = free_chunks(larger_chunk_index)->head(); - if (larger_chunk == NULL) { - larger_chunk_index = next_chunk_index(larger_chunk_index); - } - } - - if (larger_chunk != NULL) { - assert(larger_chunk->word_size() > word_size, "Sanity"); - assert(larger_chunk->get_chunk_type() == larger_chunk_index, "Sanity"); - - // We found a larger chunk. Lets split it up: - // - remove old chunk - // - in its place, create new smaller chunks, with at least one chunk - // being of target size, the others sized as large as possible. This - // is to make sure the resulting chunks are "as coalesced as possible" - // (similar to VirtualSpaceNode::retire()). - // Note: during this operation both ChunkManager and VirtualSpaceNode - // are temporarily invalid, so be careful with asserts. - - log_trace(gc, metaspace, freelist)("%s: splitting chunk " PTR_FORMAT - ", word size " SIZE_FORMAT_HEX " (%s), to get a chunk of word size " SIZE_FORMAT_HEX " (%s)...", - (is_class() ? "class space" : "metaspace"), p2i(larger_chunk), larger_chunk->word_size(), - chunk_size_name(larger_chunk_index), word_size, chunk_size_name(target_chunk_index)); - - chunk = split_chunk(word_size, larger_chunk); - - // This should have worked. - assert(chunk != NULL, "Sanity"); - assert(chunk->word_size() == word_size, "Sanity"); - assert(chunk->is_tagged_free(), "Sanity"); - - we_did_split_a_chunk = true; - - } - } - - if (chunk == NULL) { - return NULL; - } - - // Remove the chunk as the head of the list. - free_list->remove_chunk(chunk); - - log_trace(gc, metaspace, freelist)("ChunkManager::free_chunks_get: free_list: " PTR_FORMAT " chunks left: " SSIZE_FORMAT ".", - p2i(free_list), free_list->count()); - - } else { - chunk = humongous_dictionary()->get_chunk(word_size); - - if (chunk == NULL) { - return NULL; - } - - log_trace(gc, metaspace, alloc)("Free list allocate humongous chunk size " SIZE_FORMAT " for requested size " SIZE_FORMAT " waste " SIZE_FORMAT, - chunk->word_size(), word_size, chunk->word_size() - word_size); - } - - // Chunk has been removed from the chunk manager; update counters. - account_for_removed_chunk(chunk); - do_update_in_use_info_for_chunk(chunk, true); - chunk->container()->inc_container_count(); - chunk->inc_use_count(); - - // Remove it from the links to this freelist - chunk->set_next(NULL); - chunk->set_prev(NULL); - - // Run some verifications (some more if we did a chunk split) -#ifdef ASSERT - - EVERY_NTH(VerifyMetaspaceInterval) - // Be extra verify-y when chunk split happened. - locked_verify(true); - VirtualSpaceNode* const vsn = chunk->container(); - vsn->verify(true); - if (we_did_split_a_chunk) { - vsn->verify_free_chunks_are_ideally_merged(); - } - END_EVERY_NTH - - g_internal_statistics.num_chunks_removed_from_freelist ++; - -#endif - - return chunk; -} - -Metachunk* ChunkManager::chunk_freelist_allocate(size_t word_size) { - assert_lock_strong(MetaspaceExpand_lock); - - // Take from the beginning of the list - Metachunk* chunk = free_chunks_get(word_size); - if (chunk == NULL) { - return NULL; - } - - assert((word_size <= chunk->word_size()) || - (list_index(chunk->word_size()) == HumongousIndex), - "Non-humongous variable sized chunk"); - LogTarget(Trace, gc, metaspace, freelist) lt; - if (lt.is_enabled()) { - size_t list_count; - if (list_index(word_size) < HumongousIndex) { - ChunkList* list = find_free_chunks_list(word_size); - list_count = list->count(); - } else { - list_count = humongous_dictionary()->total_count(); - } - LogStream ls(lt); - ls.print("ChunkManager::chunk_freelist_allocate: " PTR_FORMAT " chunk " PTR_FORMAT " size " SIZE_FORMAT " count " SIZE_FORMAT " ", - p2i(this), p2i(chunk), chunk->word_size(), list_count); - ResourceMark rm; - locked_print_free_chunks(&ls); - } - - return chunk; -} - -void ChunkManager::return_single_chunk(Metachunk* chunk) { - -#ifdef ASSERT - EVERY_NTH(VerifyMetaspaceInterval) - this->locked_verify(false); - do_verify_chunk(chunk); - END_EVERY_NTH -#endif - - const ChunkIndex index = chunk->get_chunk_type(); - assert_lock_strong(MetaspaceExpand_lock); - DEBUG_ONLY(g_internal_statistics.num_chunks_added_to_freelist ++;) - assert(chunk != NULL, "Expected chunk."); - assert(chunk->container() != NULL, "Container should have been set."); - assert(chunk->is_tagged_free() == false, "Chunk should be in use."); - index_bounds_check(index); - - // Note: mangle *before* returning the chunk to the freelist or dictionary. It does not - // matter for the freelist (non-humongous chunks), but the humongous chunk dictionary - // keeps tree node pointers in the chunk payload area which mangle will overwrite. - DEBUG_ONLY(chunk->mangle(badMetaWordVal);) - - // may need node for verification later after chunk may have been merged away. - DEBUG_ONLY(VirtualSpaceNode* vsn = chunk->container(); ) - - if (index != HumongousIndex) { - // Return non-humongous chunk to freelist. - ChunkList* list = free_chunks(index); - assert(list->size() == chunk->word_size(), "Wrong chunk type."); - list->return_chunk_at_head(chunk); - log_trace(gc, metaspace, freelist)("returned one %s chunk at " PTR_FORMAT " to freelist.", - chunk_size_name(index), p2i(chunk)); - } else { - // Return humongous chunk to dictionary. - assert(chunk->word_size() > free_chunks(MediumIndex)->size(), "Wrong chunk type."); - assert(chunk->word_size() % free_chunks(SpecializedIndex)->size() == 0, - "Humongous chunk has wrong alignment."); - _humongous_dictionary.return_chunk(chunk); - log_trace(gc, metaspace, freelist)("returned one %s chunk at " PTR_FORMAT " (word size " SIZE_FORMAT ") to freelist.", - chunk_size_name(index), p2i(chunk), chunk->word_size()); - } - chunk->container()->dec_container_count(); - do_update_in_use_info_for_chunk(chunk, false); - - // Chunk has been added; update counters. - account_for_added_chunk(chunk); - - // Attempt coalesce returned chunks with its neighboring chunks: - // if this chunk is small or special, attempt to coalesce to a medium chunk. - if (index == SmallIndex || index == SpecializedIndex) { - if (!attempt_to_coalesce_around_chunk(chunk, MediumIndex)) { - // This did not work. But if this chunk is special, we still may form a small chunk? - if (index == SpecializedIndex) { - if (!attempt_to_coalesce_around_chunk(chunk, SmallIndex)) { - // give up. - } - } + ls.print_cr("full nodes purged: %d", num_nodes_purged); } } - // From here on do not access chunk anymore, it may have been merged with another chunk. - -#ifdef ASSERT - EVERY_NTH(VerifyMetaspaceInterval) - this->locked_verify(true); - vsn->verify(true); - vsn->verify_free_chunks_are_ideally_merged(); - END_EVERY_NTH -#endif + DEBUG_ONLY(_vslist->verify_locked(true)); + DEBUG_ONLY(verify_locked(true)); } -void ChunkManager::return_chunk_list(Metachunk* chunks) { - if (chunks == NULL) { - return; - } - LogTarget(Trace, gc, metaspace, freelist) log; - if (log.is_enabled()) { // tracing - log.print("returning list of chunks..."); - } - unsigned num_chunks_returned = 0; - size_t size_chunks_returned = 0; - Metachunk* cur = chunks; - while (cur != NULL) { - // Capture the next link before it is changed - // by the call to return_chunk_at_head(); - Metachunk* next = cur->next(); - if (log.is_enabled()) { // tracing - num_chunks_returned ++; - size_chunks_returned += cur->word_size(); - } - return_single_chunk(cur); - cur = next; - } - if (log.is_enabled()) { // tracing - log.print("returned %u chunks to freelist, total word size " SIZE_FORMAT ".", - num_chunks_returned, size_chunks_returned); - } +// Convenience methods to return the global class-space chunkmanager +// and non-class chunkmanager, respectively. +ChunkManager* ChunkManager::chunkmanager_class() { + return MetaspaceContext::context_class() == NULL ? NULL : MetaspaceContext::context_class()->cm(); } -void ChunkManager::collect_statistics(ChunkManagerStatistics* out) const { - MutexLocker cl(MetaspaceExpand_lock, Mutex::_no_safepoint_check_flag); - for (ChunkIndex i = ZeroIndex; i < NumberOfInUseLists; i = next_chunk_index(i)) { - out->chunk_stats(i).add(num_free_chunks(i), size_free_chunks_in_bytes(i) / sizeof(MetaWord)); +ChunkManager* ChunkManager::chunkmanager_nonclass() { + return MetaspaceContext::context_nonclass() == NULL ? NULL : MetaspaceContext::context_nonclass()->cm(); +} + +// Update statistics. +void ChunkManager::add_to_statistics(cm_stats_t* out) const { + + MutexLocker fcl(MetaspaceExpand_lock, Mutex::_no_safepoint_check_flag); + + for (chunklevel_t l = chunklevel::ROOT_CHUNK_LEVEL; l <= chunklevel::HIGHEST_CHUNK_LEVEL; l ++) { + out->num_chunks[l] += _chunks.num_chunks_at_level(l); + out->committed_word_size[l] += _chunks.committed_word_size_at_level(l); } + + DEBUG_ONLY(out->verify();) + +} + +#ifdef ASSERT + +void ChunkManager::verify(bool slow) const { + MutexLocker fcl(MetaspaceExpand_lock, Mutex::_no_safepoint_check_flag); + verify_locked(slow); +} + +void ChunkManager::verify_locked(bool slow) const { + assert_lock_strong(MetaspaceExpand_lock); + assert(_vslist != NULL, "No vslist"); + _chunks.verify(); +} + +bool ChunkManager::contains_chunk(Metachunk* c) const { + return _chunks.contains(c); +} + +#endif // ASSERT + +void ChunkManager::print_on(outputStream* st) const { + MutexLocker fcl(MetaspaceExpand_lock, Mutex::_no_safepoint_check_flag); + print_on_locked(st); +} + +void ChunkManager::print_on_locked(outputStream* st) const { + assert_lock_strong(MetaspaceExpand_lock); + st->print_cr("cm %s: %d chunks, total word size: " SIZE_FORMAT ", committed word size: " SIZE_FORMAT, _name, + total_num_chunks(), total_word_size(), _chunks.committed_word_size()); + _chunks.print_on(st); } } // namespace metaspace - - - diff --git a/src/hotspot/share/memory/metaspace/chunkManager.hpp b/src/hotspot/share/memory/metaspace/chunkManager.hpp --- a/src/hotspot/share/memory/metaspace/chunkManager.hpp +++ b/src/hotspot/share/memory/metaspace/chunkManager.hpp @@ -1,5 +1,6 @@ /* - * Copyright (c) 2018, 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2018, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2018, 2020 SAP SE. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -26,173 +27,166 @@ #define SHARE_MEMORY_METASPACE_CHUNKMANAGER_HPP #include "memory/allocation.hpp" -#include "memory/binaryTreeDictionary.hpp" -#include "memory/freeList.hpp" +#include "memory/metaspace/chunkLevel.hpp" +#include "memory/metaspace/counter.hpp" +#include "memory/metaspace/freeChunkList.hpp" #include "memory/metaspace/metachunk.hpp" -#include "memory/metaspace/metaspaceStatistics.hpp" -#include "memory/metaspaceChunkFreeListSummary.hpp" -#include "utilities/globalDefinitions.hpp" - -class ChunkManagerTestAccessor; namespace metaspace { -typedef class FreeList ChunkList; -typedef BinaryTreeDictionary > ChunkTreeDictionary; +class VirtualSpaceList; +struct cm_stats_t; -// Manages the global free lists of chunks. -class ChunkManager : public CHeapObj { - friend class ::ChunkManagerTestAccessor; +// ChunkManager has a somewhat central role. - // Free list of chunks of different sizes. - // SpecializedChunk - // SmallChunk - // MediumChunk - ChunkList _free_chunks[NumberOfFreeLists]; +// Arenas request chunks from it and, on death, return chunks back to it. +// It keeps freelists for chunks, one per chunk level, sorted by chunk +// commit state. +// To feed the freelists, it allocates root chunks from the associated +// VirtualSpace below it. +// +// ChunkManager directs splitting chunks, if a chunk request cannot be +// fulfilled directly. It also takes care of merging when chunks are +// returned to it, before they are added to the freelist. +// +// The freelists are double linked double headed; fully committed chunks +// are added to the front, others to the back. +// +// Level +// +--------------------+ +--------------------+ +// 0 +----| free root chunk |---| free root chunk |---... +// | +--------------------+ +--------------------+ +// | +// | +----------+ +----------+ +// 1 +----| |---| |---... +// | +----------+ +----------+ +// | +// . +// . +// . +// +// | +-+ +-+ +// 12 +----| |---| |---... +// +-+ +-+ - // Whether or not this is the class chunkmanager. - const bool _is_class; - // Return non-humongous chunk list by its index. - ChunkList* free_chunks(ChunkIndex index); +class ChunkManager : public CHeapObj { - // Returns non-humongous chunk list for the given chunk word size. - ChunkList* find_free_chunks_list(size_t word_size); + // A chunk manager is connected to a virtual space list which is used + // to allocate new root chunks when no free chunks are found. + VirtualSpaceList* const _vslist; - // HumongousChunk - ChunkTreeDictionary _humongous_dictionary; + // Name + const char* const _name; - // Returns the humongous chunk dictionary. - ChunkTreeDictionary* humongous_dictionary() { return &_humongous_dictionary; } - const ChunkTreeDictionary* humongous_dictionary() const { return &_humongous_dictionary; } + // Freelists + FreeChunkListVector _chunks; - // Size, in metaspace words, of all chunks managed by this ChunkManager - size_t _free_chunks_total; - // Number of chunks in this ChunkManager - size_t _free_chunks_count; + // Returns true if this manager contains the given chunk. Slow (walks free lists) and + // only needed for verifications. + DEBUG_ONLY(bool contains_chunk(Metachunk* c) const;) - // Update counters after a chunk was added or removed removed. - void account_for_added_chunk(const Metachunk* c); - void account_for_removed_chunk(const Metachunk* c); + // Given a chunk, split it into a target chunk of a smaller size (target level) + // at least one, possible more splinter chunks. Splinter chunks are added to the + // freelist. + // The original chunk must be outside of the freelist and its state must be free. + // The resulting target chunk will be located at the same address as the original + // chunk, but it will of course be smaller (of a higher level). + // The committed areas within the original chunk carry over to the resulting + // chunks. + void split_chunk_and_add_splinters(Metachunk* c, chunklevel_t target_level); - // Given a pointer to a chunk, attempts to merge it with neighboring - // free chunks to form a bigger chunk. Returns true if successful. - bool attempt_to_coalesce_around_chunk(Metachunk* chunk, ChunkIndex target_chunk_type); + // See get_chunk(s,s,s) + Metachunk* get_chunk_locked(size_t preferred_word_size, size_t min_word_size, size_t min_committed_words); - // Helper for chunk merging: - // Given an address range with 1-n chunks which are all supposed to be - // free and hence currently managed by this ChunkManager, remove them - // from this ChunkManager and mark them as invalid. - // - This does not correct the occupancy map. - // - This does not adjust the counters in ChunkManager. - // - Does not adjust container count counter in containing VirtualSpaceNode. - // Returns number of chunks removed. - int remove_chunks_in_area(MetaWord* p, size_t word_size); + // Uncommit all chunks equal or below the given level. + void uncommit_free_chunks(chunklevel_t max_level); - // Helper for chunk splitting: given a target chunk size and a larger free chunk, - // split up the larger chunk into n smaller chunks, at least one of which should be - // the target chunk of target chunk size. The smaller chunks, including the target - // chunk, are returned to the freelist. The pointer to the target chunk is returned. - // Note that this chunk is supposed to be removed from the freelist right away. - Metachunk* split_chunk(size_t target_chunk_word_size, Metachunk* chunk); + // Return a single chunk to the freelist without doing any merging, and adjust accounting. + void return_chunk_simple_locked(Metachunk* c); - public: + // See return_chunk(). + void return_chunk_locked(Metachunk* c); - ChunkManager(bool is_class); +public: - // Add or delete (return) a chunk to the global freelist. - Metachunk* chunk_freelist_allocate(size_t word_size); + // Creates a chunk manager with a given name (which is for debug purposes only) + // and an associated space list which will be used to request new chunks from + // (see get_chunk()) + ChunkManager(const char* name, VirtualSpaceList* space_list); - // Map a size to a list index assuming that there are lists - // for special, small, medium, and humongous chunks. - ChunkIndex list_index(size_t size); + // On success, returns a chunk of level of , but at most . + // The first first of the chunk are guaranteed to be committed. + // On error, will return NULL. + // + // This function may fail for two reasons: + // - Either we are unable to reserve space for a new chunk (if the underlying VirtualSpaceList + // is non-expandable but needs expanding - aka out of compressed class space). + // - Or, if the necessary space cannot be committed because we hit a commit limit. + // This may be either the GC threshold or MaxMetaspaceSize. + Metachunk* get_chunk(chunklevel_t preferred_level, chunklevel_t max_level, size_t min_committed_words); - // Map a given index to the chunk size. - size_t size_by_index(ChunkIndex index) const; + // Convenience function - get a chunk of a given level, uncommitted. + Metachunk* get_chunk(chunklevel_t lvl) { return get_chunk(lvl, lvl, 0); } - bool is_class() const { return _is_class; } + // Return a single chunk to the ChunkManager and adjust accounting. May merge chunk + // with neighbors. + // Happens after a Classloader was unloaded and releases its metaspace chunks. + // !! Notes: + // 1) After this method returns, c may not be valid anymore. ** Do not access c after this function returns **. + // 2) This function will not remove c from its current chunk list. This has to be done by the caller prior to + // calling this method. + void return_chunk(Metachunk* c); - // Convenience accessors. - size_t medium_chunk_word_size() const { return size_by_index(MediumIndex); } - size_t small_chunk_word_size() const { return size_by_index(SmallIndex); } - size_t specialized_chunk_word_size() const { return size_by_index(SpecializedIndex); } + // Given a chunk c, which must be "in use" and must not be a root chunk, attempt to + // enlarge it in place by claiming its trailing buddy. + // + // This will only work if c is the leader of the buddy pair and the trailing buddy is free. + // + // If successful, the follower chunk will be removed from the freelists, the leader chunk c will + // double in size (level decreased by one). + // + // On success, true is returned, false otherwise. + bool attempt_enlarge_chunk(Metachunk* c); - // Take a chunk from the ChunkManager. The chunk is expected to be in - // the chunk manager (the freelist if non-humongous, the dictionary if - // humongous). - void remove_chunk(Metachunk* chunk); + // Attempt to reclaim free areas in metaspace wholesale: + // - first, attempt to purge nodes of the backing virtual space list: nodes which are completely + // unused get unmapped and deleted completely. + // - second, it will uncommit free chunks depending on commit granule size. + void purge(); - // Return a single chunk of type index to the ChunkManager. - void return_single_chunk(Metachunk* chunk); + // Run verifications. slow=true: verify chunk-internal integrity too. + DEBUG_ONLY(void verify(bool slow) const;) + DEBUG_ONLY(void verify_locked(bool slow) const;) - // Add the simple linked list of chunks to the freelist of chunks - // of type index. - void return_chunk_list(Metachunk* chunk); + // Returns the name of this chunk manager. + const char* name() const { return _name; } - // Total of the space in the free chunks list - size_t free_chunks_total_words() const { return _free_chunks_total; } - size_t free_chunks_total_bytes() const { return free_chunks_total_words() * BytesPerWord; } + // Returns total number of chunks + int total_num_chunks() const { return _chunks.num_chunks(); } - // Number of chunks in the free chunks list - size_t free_chunks_count() const { return _free_chunks_count; } + // Returns number of words in all free chunks (regardless of commit state). + size_t total_word_size() const { return _chunks.word_size(); } - // Remove from a list by size. Selects list based on size of chunk. - Metachunk* free_chunks_get(size_t chunk_word_size); + // Returns number of committed words in all free chunks. + size_t total_committed_word_size() const { return _chunks.committed_word_size(); } -#define index_bounds_check(index) \ - assert(is_valid_chunktype(index), "Bad index: %d", (int) index) + // Update statistics. + void add_to_statistics(cm_stats_t* out) const; - size_t num_free_chunks(ChunkIndex index) const { - index_bounds_check(index); + void print_on(outputStream* st) const; + void print_on_locked(outputStream* st) const; - if (index == HumongousIndex) { - return _humongous_dictionary.total_free_blocks(); - } +public: - ssize_t count = _free_chunks[index].count(); - return count == -1 ? 0 : (size_t) count; - } + // Convenience methods to return the global class-space chunkmanager + // and non-class chunkmanager, respectively. + static ChunkManager* chunkmanager_class(); + static ChunkManager* chunkmanager_nonclass(); - size_t size_free_chunks_in_bytes(ChunkIndex index) const { - index_bounds_check(index); - - size_t word_size = 0; - if (index == HumongousIndex) { - word_size = _humongous_dictionary.total_size(); - } else { - const size_t size_per_chunk_in_words = _free_chunks[index].size(); - word_size = size_per_chunk_in_words * num_free_chunks(index); - } - - return word_size * BytesPerWord; - } - - MetaspaceChunkFreeListSummary chunk_free_list_summary() const { - return MetaspaceChunkFreeListSummary(num_free_chunks(SpecializedIndex), - num_free_chunks(SmallIndex), - num_free_chunks(MediumIndex), - num_free_chunks(HumongousIndex), - size_free_chunks_in_bytes(SpecializedIndex), - size_free_chunks_in_bytes(SmallIndex), - size_free_chunks_in_bytes(MediumIndex), - size_free_chunks_in_bytes(HumongousIndex)); - } - -#ifdef ASSERT - // Debug support - // Verify free list integrity. slow=true: verify chunk-internal integrity too. - void verify(bool slow) const; - void locked_verify(bool slow) const; -#endif - - void locked_print_free_chunks(outputStream* st); - - // Fill in current statistic values to the given statistics object. - void collect_statistics(ChunkManagerStatistics* out) const; }; } // namespace metaspace - #endif // SHARE_MEMORY_METASPACE_CHUNKMANAGER_HPP diff --git a/src/hotspot/share/memory/metaspace/classLoaderMetaspace.cpp b/src/hotspot/share/memory/metaspace/classLoaderMetaspace.cpp new file mode 100644 --- /dev/null +++ b/src/hotspot/share/memory/metaspace/classLoaderMetaspace.cpp @@ -0,0 +1,210 @@ +/* + * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020 SAP SE. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" + +#include "logging/log.hpp" +#include "memory/metaspace/arenaGrowthPolicy.hpp" +#include "memory/metaspace.hpp" +#include "memory/metaspaceTracer.hpp" +#include "memory/metaspace/chunkManager.hpp" +#include "memory/metaspace/internStat.hpp" +#include "memory/metaspace/metaspaceArena.hpp" +#include "memory/metaspace/metaspaceEnums.hpp" +#include "memory/metaspace/metaspaceStatistics.hpp" +#include "memory/metaspace/runningCounters.hpp" +#include "memory/metaspace/settings.hpp" +#include "runtime/atomic.hpp" +#include "utilities/debug.hpp" + +using metaspace::clms_stats_t; +using metaspace::ChunkManager; +using metaspace::MetaspaceArena; +using metaspace::ArenaGrowthPolicy; +using metaspace::RunningCounters; +using metaspace::InternalStats; + +#define LOGFMT "CLMS @" PTR_FORMAT " " +#define LOGFMT_ARGS p2i(this) + +static bool use_class_space(bool is_class) { + if (Metaspace::using_class_space() && is_class) { + return true; + } + return false; +} + +static bool use_class_space(Metaspace::MetadataType mdType) { + return use_class_space(metaspace::is_class(mdType)); +} + +ClassLoaderMetaspace::ClassLoaderMetaspace(Mutex* lock, Metaspace::MetaspaceType space_type) + : _lock(lock) + , _space_type(space_type) + , _non_class_space_arena(NULL) + , _class_space_arena(NULL) +{ + ChunkManager* const non_class_cm = + ChunkManager::chunkmanager_nonclass(); + + // Initialize non-class Arena + _non_class_space_arena = new MetaspaceArena( + non_class_cm, + ArenaGrowthPolicy::policy_for_space_type(space_type, false), + lock, + RunningCounters::used_nonclass_counter(), + "non-class sm"); + + // If needed, initialize class arena + if (Metaspace::using_class_space()) { + ChunkManager* const class_cm = + ChunkManager::chunkmanager_class(); + _class_space_arena = new MetaspaceArena( + class_cm, + ArenaGrowthPolicy::policy_for_space_type(space_type, true), + lock, + RunningCounters::used_class_counter(), + "class sm"); + } + + UL2(debug, "born (SpcMgr nonclass: " PTR_FORMAT ", SpcMgr class: " PTR_FORMAT ".", + p2i(_non_class_space_arena), p2i(_class_space_arena)); +} + +ClassLoaderMetaspace::~ClassLoaderMetaspace() { + Metaspace::assert_not_frozen(); + + UL(debug, "dies."); + + delete _non_class_space_arena; + delete _class_space_arena; + +} + +// Allocate word_size words from Metaspace. +MetaWord* ClassLoaderMetaspace::allocate(size_t word_size, Metaspace::MetadataType mdType) { + Metaspace::assert_not_frozen(); + if (use_class_space(mdType)) { + return class_space_arena()->allocate(word_size); + } else { + return non_class_space_arena()->allocate(word_size); + } +} + +// Attempt to expand the GC threshold to be good for at least another word_size words +// and allocate. Returns NULL if failure. Used during Metaspace GC. +MetaWord* ClassLoaderMetaspace::expand_and_allocate(size_t word_size, Metaspace::MetadataType mdType) { + Metaspace::assert_not_frozen(); + size_t delta_bytes = MetaspaceGC::delta_capacity_until_GC(word_size * BytesPerWord); + assert(delta_bytes > 0, "Must be"); + + size_t before = 0; + size_t after = 0; + bool can_retry = true; + MetaWord* res; + bool incremented; + + // Each thread increments the HWM at most once. Even if the thread fails to increment + // the HWM, an allocation is still attempted. This is because another thread must then + // have incremented the HWM and therefore the allocation might still succeed. + do { + incremented = MetaspaceGC::inc_capacity_until_GC(delta_bytes, &after, &before, &can_retry); + res = allocate(word_size, mdType); + } while (!incremented && res == NULL && can_retry); + + if (incremented) { + Metaspace::tracer()->report_gc_threshold(before, after, + MetaspaceGCThresholdUpdater::ExpandAndAllocate); + // Keeping both for now until I am sure the old variant (gc + metaspace) is not needed anymore + log_trace(gc, metaspace)("Increase capacity to GC from " SIZE_FORMAT " to " SIZE_FORMAT, before, after); + UL2(info, "GC threshold increased: " SIZE_FORMAT "->" SIZE_FORMAT ".", before, after); + } + + return res; +} + +// Prematurely returns a metaspace allocation to the _block_freelists +// because it is not needed anymore. +void ClassLoaderMetaspace::deallocate(MetaWord* ptr, size_t word_size, bool is_class) { + + Metaspace::assert_not_frozen(); + + if (use_class_space(is_class)) { + class_space_arena()->deallocate(ptr, word_size); + } else { + non_class_space_arena()->deallocate(ptr, word_size); + } + + DEBUG_ONLY(InternalStats::inc_num_deallocs();) + +} + +// Update statistics. This walks all in-use chunks. +void ClassLoaderMetaspace::add_to_statistics(clms_stats_t* out) const { + if (non_class_space_arena() != NULL) { + non_class_space_arena()->add_to_statistics(&out->arena_stats_nonclass); + } + if (class_space_arena() != NULL) { + class_space_arena()->add_to_statistics(&out->arena_stats_class); + } +} + +#ifdef ASSERT +void ClassLoaderMetaspace::verify() const { + metaspace::check_valid_spacetype(_space_type); + if (non_class_space_arena() != NULL) { + non_class_space_arena()->verify(false); + } + if (class_space_arena() != NULL) { + class_space_arena()->verify(false); + } +} +#endif // ASSERT + + +// This only exists for JFR and jcmd VM.classloader_stats. We may want to +// change this. Capacity as a stat is of questionable use since it may +// contain committed and uncommitted areas. For now we do this to maintain +// backward compatibility with JFR. +void ClassLoaderMetaspace::calculate_jfr_stats(size_t* p_used_bytes, size_t* p_capacity_bytes) const { + // Implement this using the standard statistics objects. + size_t used_c = 0, cap_c = 0, used_nc = 0, cap_nc = 0; + if (non_class_space_arena() != NULL) { + non_class_space_arena()->usage_numbers(&used_nc, NULL, &cap_nc); + } + if (class_space_arena() != NULL) { + class_space_arena()->usage_numbers(&used_c, NULL, &cap_c); + } + if (p_used_bytes != NULL) { + *p_used_bytes = used_c + used_nc; + } + if (p_capacity_bytes != NULL) { + *p_capacity_bytes = cap_c + cap_nc; + } +} + + + + diff --git a/src/hotspot/share/memory/metaspace/commitLimiter.cpp b/src/hotspot/share/memory/metaspace/commitLimiter.cpp new file mode 100644 --- /dev/null +++ b/src/hotspot/share/memory/metaspace/commitLimiter.cpp @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020 SAP SE. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" + +#include "memory/metaspace/commitLimiter.hpp" +#include "memory/metaspace.hpp" +#include "utilities/debug.hpp" +#include "utilities/globalDefinitions.hpp" + +namespace metaspace { + +// Returns the size, in words, by which we may expand the metaspace committed area without: +// - _cap == 0: hitting GC threshold or the MaxMetaspaceSize +// - _cap > 0: hitting cap (this is just for testing purposes) +size_t CommitLimiter::possible_expansion_words() const { + + if (_cap > 0) { // Testing. + assert(_cnt.get() <= _cap, "Beyond limit?"); + return _cap - _cnt.get(); + } + + assert(_cnt.get() * BytesPerWord <= MaxMetaspaceSize, "Beyond limit?"); + const size_t words_left_below_max = MaxMetaspaceSize / BytesPerWord - _cnt.get(); + + const size_t words_left_below_gc_threshold = MetaspaceGC::allowed_expansion(); + + return MIN2(words_left_below_max, words_left_below_gc_threshold); + +} + +static CommitLimiter g_global_limiter(0); + +// Returns the global metaspace commit counter +CommitLimiter* CommitLimiter::globalLimiter() { + return &g_global_limiter; +} + +} // namespace metaspace + diff --git a/src/hotspot/share/memory/metaspace/commitLimiter.hpp b/src/hotspot/share/memory/metaspace/commitLimiter.hpp new file mode 100644 --- /dev/null +++ b/src/hotspot/share/memory/metaspace/commitLimiter.hpp @@ -0,0 +1,84 @@ +/* + * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020 SAP SE. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_MEMORY_METASPACE_COMMITLIMITER_HPP +#define SHARE_MEMORY_METASPACE_COMMITLIMITER_HPP + +#include "memory/allocation.hpp" +#include "memory/metaspace/counter.hpp" + +namespace metaspace { + +// The CommitLimiter encapsulates a limit we may want to impose on how much +// memory can be committed. This is a matter of separation of concerns: +// +// In metaspace, we have two limits to committing memory: the absolute limit, +// MaxMetaspaceSize; and the GC threshold. In both cases an allocation should +// fail if it would require committing memory and hit one of these limits. +// +// However, the actual Metaspace allocator is a generic one and this +// GC- and classloading specific logic should be kept separate. Therefore +// it is hidden inside this interface. +// +// This allows us to: +// - more easily write tests for metaspace, by providing a different implementation +// of the commit limiter, thus keeping test logic separate from VM state. +// - (potentially) use the metaspace for things other than class metadata, +// where different commit rules would apply. +// +class CommitLimiter : public CHeapObj { + + // Counts total words committed for metaspace + SizeCounter _cnt; + + // Purely for testing purposes: cap, in words. + const size_t _cap; + +public: + + // Create a commit limiter. This is only useful for testing, with a cap != 0, + // since normal code should use the global commit limiter. + // If cap != 0 (word size), the cap replaces the internal logic of limiting. + CommitLimiter(size_t cap = 0) : _cnt(), _cap(cap) {} + + // Returns the size, in words, by which we may expand the metaspace committed area without: + // - _cap == 0: hitting GC threshold or the MaxMetaspaceSize + // - _cap > 0: hitting cap (this is just for testing purposes) + size_t possible_expansion_words() const; + + void increase_committed(size_t word_size) { _cnt.increment_by(word_size); } + void decrease_committed(size_t word_size) { _cnt.decrement_by(word_size); } + + size_t committed_words() const { return _cnt.get(); } + size_t cap() const { return _cap; } + + // Returns the global metaspace commit counter + static CommitLimiter* globalLimiter(); + +}; + +} // namespace metaspace + +#endif // SHARE_MEMORY_METASPACE_COMMITLIMITER_HPP diff --git a/src/hotspot/share/memory/metaspace/commitMask.cpp b/src/hotspot/share/memory/metaspace/commitMask.cpp new file mode 100644 --- /dev/null +++ b/src/hotspot/share/memory/metaspace/commitMask.cpp @@ -0,0 +1,110 @@ +/* + * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020 SAP SE. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" + +#include "memory/metaspace/commitMask.hpp" +#include "memory/metaspace/metaspaceCommon.hpp" +#include "memory/metaspace/settings.hpp" +#include "runtime/stubRoutines.hpp" + +#include "utilities/align.hpp" +#include "utilities/debug.hpp" + +namespace metaspace { + +CommitMask::CommitMask(const MetaWord* start, size_t word_size) + : CHeapBitMap(mask_size(word_size, Settings::commit_granule_words())) + , _base(start) + , _word_size(word_size) + , _words_per_bit(Settings::commit_granule_words()) +{ + assert(_word_size > 0 && _words_per_bit > 0 && + is_aligned(_word_size, _words_per_bit), "Sanity"); +} + +#ifdef ASSERT + +// This is very expensive +static const bool TEST_UNCOMMITTED_REGION = false; + +volatile u1 x; + +static void check_range_is_accessible(const MetaWord* p, size_t word_size) { + const MetaWord* const p_end = p + word_size; + u1 x2 = 0; + for (const MetaWord* q = p; q < p_end; q += os::vm_page_size() / BytesPerWord) { + x2 += *(u1*)q; + } + x = x2; +} + +void CommitMask::verify(bool slow) const { + + // Walk the whole commit mask. + // For each 1 bit, check if the associated granule is accessible. + // For each 0 bit, check if the associated granule is not accessible. Slow mode only. + + assert(_base != NULL && _word_size > 0 && _words_per_bit > 0, "Sanity"); + assert_is_aligned(_base, _words_per_bit * BytesPerWord); + assert_is_aligned(_word_size, _words_per_bit); + + if (slow) { + for (idx_t i = 0; i < size(); i ++) { + const MetaWord* const p = _base + (i * _words_per_bit); + if (at(i)) { + // Should be accessible. Just touch it. + check_range_is_accessible(p, _words_per_bit); + } else { + // Note: results may differ between platforms. On Linux, this should be true since + // we uncommit memory by setting protection to PROT_NONE. We may have to look if + // this works as expected on other platforms. + if (TEST_UNCOMMITTED_REGION && CanUseSafeFetch32()) { + assert(os::is_readable_pointer(p) == false, + "index %u, pointer " PTR_FORMAT ", should not be accessible.", + (unsigned)i, p2i(p)); + } + } + } + } + +} + +#endif // ASSERT + +void CommitMask::print_on(outputStream* st) const { + + st->print("commit mask, base " PTR_FORMAT ":", p2i(base())); + + for (idx_t i = 0; i < size(); i ++) { + st->print("%c", at(i) ? 'X' : '-'); + } + + st->cr(); + +} + +} // namespace metaspace + diff --git a/src/hotspot/share/memory/metaspace/commitMask.hpp b/src/hotspot/share/memory/metaspace/commitMask.hpp new file mode 100644 --- /dev/null +++ b/src/hotspot/share/memory/metaspace/commitMask.hpp @@ -0,0 +1,182 @@ +/* + * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020 SAP SE. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_MEMORY_METASPACE_COMMITMASK_HPP +#define SHARE_MEMORY_METASPACE_COMMITMASK_HPP + +#include "utilities/debug.hpp" +#include "utilities/bitMap.hpp" +#include "utilities/globalDefinitions.hpp" + +class outputStream; + +namespace metaspace { + +// The CommitMask is a bitmask used to store the commit state of commit granules. +// It keeps one bit per granule; 1 means committed, 0 means uncommitted. + +class CommitMask : public CHeapBitMap { + + const MetaWord* const _base; + const size_t _word_size; + const size_t _words_per_bit; + + // Given an offset, in words, into the area, return the number of the bit + // covering it. + static idx_t bitno_for_word_offset(size_t offset, size_t words_per_bit) { + return offset / words_per_bit; + } + + idx_t bitno_for_address(const MetaWord* p) const { + // Note: we allow one-beyond since this is a typical need. + assert(p >= _base && p <= _base + _word_size, "Invalid address"); + const size_t off = p - _base; + return bitno_for_word_offset(off, _words_per_bit); + } + + static idx_t mask_size(size_t word_size, size_t words_per_bit) { + return bitno_for_word_offset(word_size, words_per_bit); + } + + struct BitCounterClosure : public BitMapClosure { + idx_t cnt; + bool do_bit(BitMap::idx_t offset) { cnt ++; return true; } + }; + +#ifdef ASSERT + // Given a pointer, check if it points into the range this bitmap covers. + bool is_pointer_valid(const MetaWord* p) const { + return p >= _base && p < _base + _word_size; + } + + // Given a pointer, check if it points into the range this bitmap covers. + void check_pointer(const MetaWord* p) const { + assert(is_pointer_valid(p), + "Pointer " PTR_FORMAT " not in range of this bitmap [" PTR_FORMAT ", " PTR_FORMAT ").", + p2i(p), p2i(_base), p2i(_base + _word_size)); + } + // Given a pointer, check if it points into the range this bitmap covers, + // and if it is aligned to commit granule border. + void check_pointer_aligned(const MetaWord* p) const { + check_pointer(p); + assert(is_aligned(p, _words_per_bit * BytesPerWord), + "Pointer " PTR_FORMAT " should be aligned to commit granule size " SIZE_FORMAT ".", + p2i(p), _words_per_bit * BytesPerWord); + } + // Given a range, check if it points into the range this bitmap covers, + // and if its borders are aligned to commit granule border. + void check_range(const MetaWord* start, size_t word_size) const { + check_pointer_aligned(start); + assert(is_aligned(word_size, _words_per_bit), + "Range " SIZE_FORMAT " should be aligned to commit granule size " SIZE_FORMAT ".", + word_size, _words_per_bit); + check_pointer(start + word_size - 1); + } +#endif + + // Marks a single commit granule as committed (value == true) + // or uncomitted (value == false) and returns + // its prior state. + bool mark_granule(idx_t bitno, bool value) { + bool b = at(bitno); + at_put(bitno, value); + return b; + } + +public: + + // Create a commit mask covering a range [start, start + word_size). + CommitMask(const MetaWord* start, size_t word_size); + + const MetaWord* base() const { return _base; } + size_t word_size() const { return _word_size; } + const MetaWord* end() const { return _base + word_size(); } + + // Given an address, returns true if the address is committed, false if not. + bool is_committed_address(const MetaWord* p) const { + DEBUG_ONLY(check_pointer(p)); + const idx_t bitno = bitno_for_address(p); + return at(bitno); + } + + // Given an address range, return size, in number of words, of committed area within that range. + size_t get_committed_size_in_range(const MetaWord* start, size_t word_size) const { + DEBUG_ONLY(check_range(start, word_size)); + assert(word_size > 0, "zero range"); + const idx_t b1 = bitno_for_address(start); + const idx_t b2 = bitno_for_address(start + word_size); + const idx_t num_bits = count_one_bits(b1, b2); + return num_bits * _words_per_bit; + } + + // Return total committed size, in number of words. + size_t get_committed_size() const { + return count_one_bits() * _words_per_bit; + } + + // Mark a whole address range [start, end) as committed. + // Return the number of words which had already been committed before this operation. + size_t mark_range_as_committed(const MetaWord* start, size_t word_size) { + DEBUG_ONLY(check_range(start, word_size)); + assert(word_size > 0, "zero range"); + const idx_t b1 = bitno_for_address(start); + const idx_t b2 = bitno_for_address(start + word_size); + if (b1 == b2) { // Simple case, 1 granule + bool was_committed = mark_granule(b1, true); + return was_committed ? _words_per_bit : 0; + } + const idx_t one_bits_in_range_before = count_one_bits(b1, b2); + set_range(b1, b2); + return one_bits_in_range_before * _words_per_bit; + } + + // Mark a whole address range [start, end) as uncommitted. + // Return the number of words which had already been uncommitted before this operation. + size_t mark_range_as_uncommitted(const MetaWord* start, size_t word_size) { + DEBUG_ONLY(check_range(start, word_size)); + assert(word_size > 0, "zero range"); + const idx_t b1 = bitno_for_address(start); + const idx_t b2 = bitno_for_address(start + word_size); + if (b1 == b2) { // Simple case, 1 granule + bool was_committed = mark_granule(b1, false); + return was_committed ? 0 : _words_per_bit; + } + const idx_t zero_bits_in_range_before = + (b2 - b1) - count_one_bits(b1, b2); + clear_range(b1, b2); + return zero_bits_in_range_before * _words_per_bit; + } + + + //// Debug stuff //// + DEBUG_ONLY(void verify(bool slow) const;) + + void print_on(outputStream* st) const; + +}; + +} // namespace metaspace + +#endif // SHARE_MEMORY_METASPACE_COMMITMASK_HPP diff --git a/src/hotspot/share/memory/metaspace/counter.hpp b/src/hotspot/share/memory/metaspace/counter.hpp new file mode 100644 --- /dev/null +++ b/src/hotspot/share/memory/metaspace/counter.hpp @@ -0,0 +1,186 @@ +/* + * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020 SAP SE. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_MEMORY_METASPACE_COUNTER_HPP +#define SHARE_MEMORY_METASPACE_COUNTER_HPP + +#include "metaprogramming/isSigned.hpp" +#include "runtime/atomic.hpp" +#include "utilities/debug.hpp" +#include "utilities/globalDefinitions.hpp" + + +namespace metaspace { + +// We seem to be counting a lot of things which makes it worthwhile to +// make helper classes for all that boilerplate coding. + +// AbstractCounter counts something and asserts overflow and underflow. +template +class AbstractCounter { + + T _c; + + // Only allow unsigned values for now + STATIC_ASSERT(IsSigned::value == false); + +public: + + AbstractCounter() : _c(0) {} + + T get() const { return _c; } + + void increment() { increment_by(1); } + void decrement() { decrement_by(1); } + + void increment_by(T v) { +#ifdef ASSERT + T old = _c; + assert(old + v >= old, + "overflow (" UINT64_FORMAT "+" UINT64_FORMAT ")", (uint64_t)old, (uint64_t)v); +#endif + _c += v; + } + + void decrement_by(T v) { + assert(_c >= v, + "underflow (" UINT64_FORMAT "-" UINT64_FORMAT ")", + (uint64_t)_c, (uint64_t)v); + _c -= v; + } + + void reset() { _c = 0; } + +#ifdef ASSERT + void check(T expected) const { + assert(_c == expected, "Counter mismatch: %d, expected: %d.", + (int)_c, (int)expected); + } +#endif + +}; + +// Atomic variant of AbstractCounter. +template +class AbstractAtomicCounter { + + volatile T _c; + + // Only allow unsigned values for now + STATIC_ASSERT(IsSigned::value == false); + +public: + + AbstractAtomicCounter() : _c(0) {} + + T get() const { return _c; } + + void increment() { + Atomic::inc(&_c); + } + + void decrement() { +#ifdef ASSERT + T old = Atomic::load_acquire(&_c); + assert(old >= 1, + "underflow (" UINT64_FORMAT "-1)", (uint64_t)old); +#endif + Atomic::dec(&_c); + } + + void increment_by(T v) { + Atomic::add(&_c, v); + } + + void decrement_by(T v) { +#ifdef ASSERT + T old = Atomic::load_acquire(&_c); + assert(old >= v, + "underflow (" UINT64_FORMAT "+" UINT64_FORMAT ")", (uint64_t)old, (uint64_t)v); +#endif + Atomic::sub(&_c, v); + } + +#ifdef ASSERT + void check(T expected) const { + assert(_c == expected, "Counter mismatch: %d, expected: %d.", + (int)_c, (int)expected); + } +#endif + +}; + +typedef AbstractCounter SizeCounter; +typedef AbstractCounter IntCounter; + +typedef AbstractAtomicCounter SizeAtomicCounter; + + +// We often count memory ranges (blocks, chunks etc). +// Make a helper class for that. +template +class AbstractMemoryRangeCounter { + + AbstractCounter _count; + AbstractCounter _total_size; + +public: + + void add(T_size s) { + if(s > 0) { + _count.increment(); + _total_size.increment_by(s); + } + } + + void sub(T_size s) { + if(s > 0) { + _count.decrement(); + _total_size.decrement_by(s); + } + } + + T_num count() const { return _count.get(); } + T_size total_size() const { return _total_size.get(); } + + +#ifdef ASSERT + void check(T_num expected_count, T_size expected_size) const { + _count.check(expected_count); + _total_size.check(expected_size); + } + void check(const AbstractMemoryRangeCounter& other) const { + check(other.count(), other.total_size()); + } +#endif + +}; + +typedef AbstractMemoryRangeCounter MemRangeCounter; + +} // namespace metaspace + +#endif // SHARE_MEMORY_METASPACE_WORDSIZECOUNTER_HPP + diff --git a/src/hotspot/share/memory/metaspace/freeBlocks.cpp b/src/hotspot/share/memory/metaspace/freeBlocks.cpp new file mode 100644 --- /dev/null +++ b/src/hotspot/share/memory/metaspace/freeBlocks.cpp @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020 SAP SE. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "memory/metaspace/freeBlocks.hpp" +#include "utilities/globalDefinitions.hpp" +#include "utilities/debug.hpp" + +namespace metaspace { + +void FreeBlocks::add_block(MetaWord* p, size_t word_size) { + assert(word_size >= minimal_word_size, "sanity (" SIZE_FORMAT ")", word_size); + if (word_size >= _small_blocks.maximal_word_size) { + _tree.add_block(p, word_size); + } else { + _small_blocks.add_block(p, word_size); + } +} + +MetaWord* FreeBlocks::get_block(size_t requested_word_size) { + assert(requested_word_size >= minimal_word_size, + "requested_word_size too small (" SIZE_FORMAT ")", requested_word_size); + size_t real_size = 0; + MetaWord* p = NULL; + if (requested_word_size >= _small_blocks.maximal_word_size) { + p = _tree.get_block(requested_word_size, &real_size); + } else { + p = _small_blocks.get_block(requested_word_size, &real_size); + } + if (p != NULL) { + // Blocks which are larger than a certain threshold are split and + // the remainder is handed back to the manager. + const size_t waste = real_size - requested_word_size; + if (waste > MAX2(minimal_word_size, splinter_threshold)) { + add_block(p + requested_word_size, waste); + } + } + return p; +} + +} // namespace metaspace + diff --git a/src/hotspot/share/memory/metaspace/freeBlocks.hpp b/src/hotspot/share/memory/metaspace/freeBlocks.hpp new file mode 100644 --- /dev/null +++ b/src/hotspot/share/memory/metaspace/freeBlocks.hpp @@ -0,0 +1,117 @@ +/* + * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020 SAP SE. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_MEMORY_METASPACE_LEFTOVERBINS_HPP +#define SHARE_MEMORY_METASPACE_LEFTOVERBINS_HPP + +#include "memory/allocation.hpp" + +#include "memory/metaspace/counter.hpp" +#include "memory/metaspace/binlist.hpp" +#include "memory/metaspace/blocktree.hpp" + +#include "utilities/debug.hpp" +#include "utilities/globalDefinitions.hpp" + + +class outputStream; + +namespace metaspace { + +// Class FreeBlocks manages deallocated blocks in Metaspace. +// +// In Metaspace, allocated memory blocks may be release prematurely. This is +// uncommon (otherwise an arena-based allocation scheme would not make sense). +// It can happen e.g. when class loading fails or when bytecode gets rewritten. +// +// All these released blocks should be reused, so they are collected. Since these +// blocks are embedded into chunks which are still in use by a live arena, +// we cannot just give these blocks to anyone; only the owner of this arena can +// reuse these blocks. Therefore these blocks are kept at arena-level. +// +// The structure to manage these released blocks at arena level is class FreeBlocks. +// +// FreeBlocks is optimized toward the typical size and number of deallocated +// blocks. The vast majority of them (about 90%) are below 16 words in size, +// but there is a significant portion of memory blocks much larger than that, +// leftover space from retired chunks, see MetaspaceArena::retire_current_chunk(). +// +// Since the vast majority of blocks are small or very small, FreeBlocks consists +// internally of two separate structures to keep very small blocks and other blocks. +// Very small blocks are kept in a bin list (see binlist.hpp) and larger blocks in +// a BST (see blocktree.hpp). + +class FreeBlocks : public CHeapObj { + + typedef BinList32 SmallBlocksType; + + // _small_blocks takes care of small to very small blocks. + SmallBlocksType _small_blocks; + + // A BST for larger blocks. + BlockTree _tree; + + static const size_t splinter_threshold = 0;// 0x100; + +public: + + const static size_t minimal_word_size = SmallBlocksType::minimal_word_size; + + // Add a block to the deallocation management. + void add_block(MetaWord* p, size_t word_size); + + // Retrieve a block of at least requested_word_size. + MetaWord* get_block(size_t requested_word_size); + +#ifdef ASSERT + void verify() const { + _tree.verify(); + _small_blocks.verify(); + }; +#endif + + // Returns number of blocks. + int count() const { + return _small_blocks.count() + _tree.count(); + } + + // Returns total size, in words, of all elements. + size_t total_size() const { + return _small_blocks.total_size() + _tree.total_size(); + } + + // Returns true if empty. + bool is_empty() const { + return _small_blocks.is_empty() && _tree.is_empty(); + } + +}; + + + + +} // namespace metaspace + +#endif // SHARE_MEMORY_METASPACE_CHUNKMANAGER_HPP diff --git a/src/hotspot/share/memory/metaspace/freeChunkList.cpp b/src/hotspot/share/memory/metaspace/freeChunkList.cpp new file mode 100644 --- /dev/null +++ b/src/hotspot/share/memory/metaspace/freeChunkList.cpp @@ -0,0 +1,183 @@ +/* + * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020 SAP SE. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "memory/metaspace/chunkLevel.hpp" +#include "memory/metaspace/freeChunkList.hpp" +#include "utilities/globalDefinitions.hpp" +#include "utilities/debug.hpp" +#include "utilities/ostream.hpp" + + + +namespace metaspace { + +void FreeChunkList::print_on(outputStream* st) const { + + if (_num_chunks.get() > 0) { + for (const Metachunk* c = _first; c != NULL; c = c->next()) { + st->print(" - <"); + c->print_on(st); + st->print(">"); + } + st->print(" - total : %d chunks.", _num_chunks.get()); + } else { + st->print("empty"); + } + +} + +#ifdef ASSERT + +bool FreeChunkList::contains(const Metachunk* c) const { + for (Metachunk* c2 = _first; c2 != NULL; c2 = c2->next()) { + if (c2 == c) { + return true; + } + } + return false; +} + +void FreeChunkList::verify() const { + + if (_first == NULL) { + assert(_last == NULL, "Sanity"); + } else { + assert(_last != NULL, "Sanity"); + size_t committed = 0; + int num = 0; + bool uncommitted = (_first->committed_words() == 0); + for (Metachunk* c = _first; c != NULL; c = c->next()) { + assert(c->is_free(), "Chunks in freelist should be free"); + assert(c->used_words() == 0, "Chunk in freelist should have not used words."); + assert(c->level() == _first->level(), "wrong level"); + assert(c->next() == NULL || c->next()->prev() == c, "front link broken"); + assert(c->prev() == NULL || c->prev()->next() == c, "back link broken"); + assert(c != c->prev() && c != c->next(), "circle"); + c->verify(false); + committed += c->committed_words(); + num ++; + } + _num_chunks.check(num); + _committed_word_size.check(committed); + } + +} + +#endif // ASSERT + + +// Returns total size in all lists (regardless of commit state of underlying memory) +size_t FreeChunkListVector::word_size() const { + size_t sum = 0; + for (chunklevel_t l = chunklevel::LOWEST_CHUNK_LEVEL; l <= chunklevel::HIGHEST_CHUNK_LEVEL; l ++) { + sum += list_for_level(l)->num_chunks() * chunklevel::word_size_for_level(l); + } + return sum; +} + +// Returns total committed size in all lists +size_t FreeChunkListVector::committed_word_size() const { + size_t sum = 0; + for (chunklevel_t l = chunklevel::LOWEST_CHUNK_LEVEL; l <= chunklevel::HIGHEST_CHUNK_LEVEL; l ++) { + sum += list_for_level(l)->committed_word_size(); + } + return sum; +} + +// Returns total committed size in all lists +int FreeChunkListVector::num_chunks() const { + int n = 0; + for (chunklevel_t l = chunklevel::LOWEST_CHUNK_LEVEL; l <= chunklevel::HIGHEST_CHUNK_LEVEL; l ++) { + n += list_for_level(l)->num_chunks(); + } + return n; +} + + +// Look for a chunk: starting at level, up to and including max_level, +// return the first chunk whose committed words >= min_committed_words. +// Return NULL if no such chunk was found. +Metachunk* FreeChunkListVector::search_chunk_ascending(chunklevel_t level, chunklevel_t max_level, size_t min_committed_words) { + assert(min_committed_words <= chunklevel::word_size_for_level(max_level), + "min chunk size too small to hold min_committed_words"); + for (chunklevel_t l = level; l <= max_level; l ++) { + Metachunk* c = list_for_level(l)->first(); + if (c != NULL && c->committed_words() >= min_committed_words) { + list_for_level(l)->remove(c); + return c; + } + } + return NULL; +} + +// Look for a chunk: starting at level, down to (including) the root chunk level, +// return the first chunk whose committed words >= min_committed_words. +// Return NULL if no such chunk was found. +Metachunk* FreeChunkListVector::search_chunk_descending(chunklevel_t level, size_t min_committed_words) { + for (chunklevel_t l = level; l >= chunklevel::LOWEST_CHUNK_LEVEL; l --) { + Metachunk* c = list_for_level(l)->first(); + if (c != NULL && c->committed_words() >= min_committed_words) { + list_for_level(l)->remove(c); + return c; + } + } + return NULL; +} + +void FreeChunkListVector::print_on(outputStream* st) const { + for (chunklevel_t l = chunklevel::LOWEST_CHUNK_LEVEL; l <= chunklevel::HIGHEST_CHUNK_LEVEL; l ++) { + st->print("-- List[" CHKLVL_FORMAT "]: ", l); + list_for_level(l)->print_on(st); + st->cr(); + } + st->print_cr("total chunks: %d, total word size: " SIZE_FORMAT ", committed word size: " SIZE_FORMAT ".", + num_chunks(), word_size(), committed_word_size()); +} + + +#ifdef ASSERT + +void FreeChunkListVector::verify() const { + for (chunklevel_t l = chunklevel::LOWEST_CHUNK_LEVEL; l <= chunklevel::HIGHEST_CHUNK_LEVEL; l ++) { + list_for_level(l)->verify(); + } +} + +bool FreeChunkListVector::contains(const Metachunk* c) const { + for (chunklevel_t l = chunklevel::LOWEST_CHUNK_LEVEL; l <= chunklevel::HIGHEST_CHUNK_LEVEL; l ++) { + if (list_for_level(l)->contains(c)) { + return true; + } + } + return false; +} + +#endif // ASSERT + + + +} // namespace metaspace + diff --git a/src/hotspot/share/memory/metaspace/freeChunkList.hpp b/src/hotspot/share/memory/metaspace/freeChunkList.hpp new file mode 100644 --- /dev/null +++ b/src/hotspot/share/memory/metaspace/freeChunkList.hpp @@ -0,0 +1,275 @@ +/* + * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020 SAP SE. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_MEMORY_METASPACE_FREECHUNKLIST_HPP +#define SHARE_MEMORY_METASPACE_FREECHUNKLIST_HPP + +#include "memory/allocation.hpp" +#include "memory/metaspace/chunkLevel.hpp" +#include "memory/metaspace/counter.hpp" +#include "memory/metaspace/metachunk.hpp" +#include "memory/metaspace/metachunkList.hpp" + +class outputStream; + +namespace metaspace { + + +// This is the free list underlying the ChunkManager. +// +// Chunks are kept in a vector of double-linked double-headed lists +// (using Metachunk::prev/next). One list per chunk level exists. +// +// Chunks in these lists are roughly ordered: uncommitted chunks +// are added to the back of the list, fully or partially committed +// chunks to the front. +// +// (Small caveat: commit state of a chunk may change as a result of +// actions on neighboring chunks, if the chunk is smaller than a commit +// granule and therefore shares its granule with neighbors. So it may change +// after the chunk has been added to the list. +// It will never involuntarily uncommit: only chunks >= granule size are uncommitted. +// But it may get involuntarily committed if an in-granule neighbor is committed and +// causes committing of the whole granule. +// In practice this is not a big deal; it has very little consequence.) +// +// Beyond adding at either front or at back, we do not sort on insert, since the +// insert path is used during Metaspace reclamation which may happen at GC pause. +// +// During retrieval (at class loading), we search the list for a chunk of at least +// n committed words to satisfy the caller requested committed word size. We stop +// searching at the first fully uncommitted chunk. +// +// Note that even though this is an O(n) search, in practice this is not a problem: +// - in all likelihood the requested commit word size is way smaller than even a single +// commit granule, so 99% of all searches would end at the first chunk (which is either +// uncommitted or committed to at least one commit granule size). +// - in all likelihood chunks, when added to this list, are either fully committed +// or fully uncommitted (see Settings::uncommit_on_return_min_word_size()). +// +// Should we ever encounter situations where the O(n) search is a bottleneck, this +// structure can easily be optimized (e.g. a BST). But for now lets keep this simple. + +class FreeChunkList { + + Metachunk* _first; + Metachunk* _last; + + IntCounter _num_chunks; + SizeCounter _committed_word_size; + + void add_front(Metachunk* c) { + if (_first == NULL) { + assert(_last == NULL, "Sanity"); + _first = _last = c; + c->set_prev(NULL); + c->set_next(NULL); + } else { + assert(_last != NULL, "Sanity"); + c->set_next(_first); + c->set_prev(NULL); + _first->set_prev(c); + _first = c; + } + } + + // Add chunk to the back of the list. + void add_back(Metachunk* c) { + if (_last == NULL) { + assert(_first == NULL, "Sanity"); + _last = _first = c; + c->set_prev(NULL); + c->set_next(NULL); + } else { + assert(_first != NULL, "Sanity"); + c->set_next(NULL); + c->set_prev(_last); + _last->set_next(c); + _last = c; + } + } + +public: + + FreeChunkList() : + _first(NULL), + _last(NULL) + {} + + // Remove given chunk from anywhere in the list. + Metachunk* remove(Metachunk* c) { + assert(contains(c), "Must be contained here"); + Metachunk* pred = c->prev(); + Metachunk* succ = c->next(); + if (pred) { + pred->set_next(succ); + } + if (succ) { + succ->set_prev(pred); + } + if (_first == c) { + _first = succ; + } + if (_last == c) { + _last = pred; + } + c->set_next(NULL); + c->set_prev(NULL); + _committed_word_size.decrement_by(c->committed_words()); + _num_chunks.decrement(); + return c; + } + + void add(Metachunk* c) { + assert(contains(c) == false, "Chunk already in freelist"); + assert(_first == NULL || _first->level() == c->level(), "wrong level"); + // Uncomitted chunks go to the back, fully or partially committed to the front. + if (c->committed_words() == 0) { + add_back(c); + } else { + add_front(c); + } + _committed_word_size.increment_by(c->committed_words()); + _num_chunks.increment(); + } + + // Removes the first chunk from the list and returns it. Returns NULL if list is empty. + Metachunk* remove_first() { + Metachunk* c = _first; + if (c != NULL) { + remove(c); + } + return c; + } + + // Find and removes a chunk in this list which has at least min_committed_words committed words. + // Returns NULL if not found. + Metachunk* find_matching(size_t min_committed_words) { + Metachunk* c = _first; + while (c != NULL && c->committed_words() > 0) { + if (c->committed_words() <= min_committed_words) { + remove(c); + return c; + } + c = c->next(); + } + return NULL; + } + + // Returns reference to the first chunk in the list, or NULL + Metachunk* first() const { return _first; } + +#ifdef ASSERT + bool contains(const Metachunk* c) const; + void verify() const; +#endif + + // Returns number of chunks + int num_chunks() const { return _num_chunks.get(); } + + // Returns total committed word size + size_t committed_word_size() const { return _committed_word_size.get(); } + + void print_on(outputStream* st) const; + +}; + + +// A vector of free chunk lists, one per chunk level +class FreeChunkListVector { + + FreeChunkList _lists[chunklevel::NUM_CHUNK_LEVELS]; + + const FreeChunkList* list_for_level(chunklevel_t lvl) const { DEBUG_ONLY(chunklevel::check_valid_level(lvl)); return _lists + lvl; } + FreeChunkList* list_for_level(chunklevel_t lvl) { DEBUG_ONLY(chunklevel::check_valid_level(lvl)); return _lists + lvl; } + + const FreeChunkList* list_for_chunk(const Metachunk* c) const { return list_for_level(c->level()); } + FreeChunkList* list_for_chunk(const Metachunk* c) { return list_for_level(c->level()); } + +public: + + // Remove given chunk from its list. List must contain that chunk. + void remove(Metachunk* c) { + list_for_chunk(c)->remove(c); + } + + // Remove first node unless empty. Returns node or NULL. + Metachunk* remove_first(chunklevel_t lvl) { + Metachunk* c = list_for_level(lvl)->remove_first(); + return c; + } + + void add(Metachunk* c) { + list_for_chunk(c)->add(c); + } + + // Returns number of chunks for a given level. + int num_chunks_at_level(chunklevel_t lvl) const { + return list_for_level(lvl)->num_chunks(); + } + + // Returns number of chunks for a given level. + size_t committed_word_size_at_level(chunklevel_t lvl) const { + return list_for_level(lvl)->committed_word_size(); + } + + // Returns reference to first chunk at this level, or NULL if sublist is empty. + Metachunk* first_at_level(chunklevel_t lvl) const { + return list_for_level(lvl)->first(); + } + + // Look for a chunk: starting at level, up to and including max_level, + // return the first chunk whose committed words >= min_committed_words. + // Return NULL if no such chunk was found. + Metachunk* search_chunk_ascending(chunklevel_t level, chunklevel_t max_level, + size_t min_committed_words); + + // Look for a chunk: starting at level, down to (including) the root chunk level, + // return the first chunk whose committed words >= min_committed_words. + // Return NULL if no such chunk was found. + Metachunk* search_chunk_descending(chunklevel_t level, size_t min_committed_words); + + // Returns total size in all lists (regardless of commit state of underlying memory) + size_t word_size() const; + + // Returns total committed size in all lists + size_t committed_word_size() const; + + // Returns number of chunks in all lists + int num_chunks() const; + +#ifdef ASSERT + bool contains(const Metachunk* c) const; + void verify() const; +#endif + + void print_on(outputStream* st) const; + +}; + + +} // namespace metaspace + +#endif // SHARE_MEMORY_METASPACE_FREECHUNKLIST_HPP diff --git a/src/hotspot/share/memory/metaspace/internStat.cpp b/src/hotspot/share/memory/metaspace/internStat.cpp new file mode 100644 --- /dev/null +++ b/src/hotspot/share/memory/metaspace/internStat.cpp @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020 SAP SE. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "memory/metaspace/internStat.hpp" +#include "utilities/globalDefinitions.hpp" +#include "utilities/ostream.hpp" + +namespace metaspace { + +#define MATERIALIZE_COUNTER(name) uint64_t InternalStats::_##name; +#define MATERIALIZE_ATOMIC_COUNTER(name) volatile uint64_t InternalStats::_##name; + ALL_MY_COUNTERS(MATERIALIZE_COUNTER, MATERIALIZE_ATOMIC_COUNTER) +#undef MATERIALIZE_COUNTER +#undef MATERIALIZE_ATOMIC_COUNTER + + +void InternalStats::print_on(outputStream* st) { + +#define xstr(s) str(s) +#define str(s) #s + +#define PRINT_COUNTER(name) st->print_cr("%s: " UINT64_FORMAT ".", xstr(name), _##name); + ALL_MY_COUNTERS(PRINT_COUNTER, PRINT_COUNTER) +#undef PRINT_COUNTER + +} + +} // namespace metaspace + diff --git a/src/hotspot/share/memory/metaspace/internStat.hpp b/src/hotspot/share/memory/metaspace/internStat.hpp new file mode 100644 --- /dev/null +++ b/src/hotspot/share/memory/metaspace/internStat.hpp @@ -0,0 +1,128 @@ +/* + * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020 SAP SE. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_MEMORY_METASPACE_INTERNSTAT_HPP +#define SHARE_MEMORY_METASPACE_INTERNSTAT_HPP + +#include "memory/allocation.hpp" +#include "runtime/atomic.hpp" +#include "utilities/globalDefinitions.hpp" + +class outputStream; + +namespace metaspace { + + +// These are some counters useful for debugging and analyzing Metaspace problems. +// They get printed as part of the Metaspace report (e.g. via jcmd VM.metaspace) + +class InternalStats : public AllStatic { + + // Note: all counters which are modified on the classloader local allocation path + // (not under ExpandLock protection) have to be atomic. + +#define ALL_MY_COUNTERS(x, x_atomic) \ + \ + /* Number of allocations. */ \ + DEBUG_ONLY(x_atomic(num_allocs)) \ + \ + /* Number of external deallocations */ \ + /* (excluding retired chunk remains) */ \ + DEBUG_ONLY(x_atomic(num_deallocs)) \ + \ + /* Number of times an allocation was satisfied */ \ + /* from deallocated blocks. */ \ + DEBUG_ONLY(x_atomic(num_allocs_from_deallocated_blocks)) \ + \ + /* Number of times an arena retired a chunk */ \ + DEBUG_ONLY(x_atomic(num_chunks_retired)) \ + \ + /* Number of times an allocation failed */ \ + /* because we hit a limit. */ \ + x_atomic(num_allocs_failed_limit) \ + \ + /* Number of times an arena was born ... */ \ + x_atomic(num_arena_births) \ + /* ... and died. */ \ + x_atomic(num_arena_deaths) \ + \ + /* Number of times VirtualSpaceNode were */ \ + /* born... */ \ + x(num_vsnodes_births) \ + /* ... and died. */ \ + x(num_vsnodes_deaths) \ + \ + /* Number of times we committed space. */ \ + x(num_space_committed) \ + /* Number of times we uncommitted space. */ \ + x(num_space_uncommitted) \ + \ + /* Number of times a chunk was returned to the */ \ + /* freelist (external only). */ \ + x(num_chunks_returned_to_freelist) \ + /* Number of times a chunk was taken from */ \ + /* freelist (external only) */ \ + x(num_chunks_taken_from_freelist) \ + \ + /* Number of successful chunk merges */ \ + x(num_chunk_merges) \ + /* Number of chunk splits */ \ + x(num_chunk_splits) \ + /* Number of chunk in place enlargements */ \ + x(num_chunks_enlarged) \ + \ + /* Number of times we did a purge */ \ + x(num_purges) \ + + +#define DEFINE_COUNTER(name) static uint64_t _##name; +#define DEFINE_ATOMIC_COUNTER(name) static volatile uint64_t _##name; + ALL_MY_COUNTERS(DEFINE_COUNTER, DEFINE_ATOMIC_COUNTER) +#undef DEFINE_COUNTER +#undef DEFINE_ATOMIC_COUNTER + +public: + +// incrementors +#define INCREMENTOR(name) static void inc_##name() { _##name ++; } +#define INCREMENTOR_ATOMIC(name) static void inc_##name() { Atomic::inc(&_##name); } + ALL_MY_COUNTERS(INCREMENTOR, INCREMENTOR_ATOMIC) +#undef INCREMENTOR +#undef INCREMENTOR_ATOMIC + +// getters +#define GETTER(name) static uint64_t name() { return _##name; } + ALL_MY_COUNTERS(GETTER, GETTER) +#undef GETTER + + + static void print_on(outputStream* st); + + +}; + +} // namespace metaspace + +#endif // SHARE_MEMORY_METASPACE_INTERNSTAT_HPP diff --git a/src/hotspot/share/memory/metaspace/metaDebug.cpp b/src/hotspot/share/memory/metaspace/metaDebug.cpp deleted file mode 100644 --- a/src/hotspot/share/memory/metaspace/metaDebug.cpp +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "logging/log.hpp" -#include "memory/metaspace/metaDebug.hpp" -#include "runtime/os.hpp" -#include "runtime/thread.hpp" -#include "utilities/debug.hpp" -#include "utilities/globalDefinitions.hpp" - -namespace metaspace { - -int Metadebug::_allocation_fail_alot_count = 0; - -void Metadebug::init_allocation_fail_alot_count() { - if (MetadataAllocationFailALot) { - _allocation_fail_alot_count = - 1+(long)((double)MetadataAllocationFailALotInterval*os::random()/(max_jint+1.0)); - } -} - -#ifdef ASSERT -bool Metadebug::test_metadata_failure() { - if (MetadataAllocationFailALot && - Threads::is_vm_complete()) { - if (_allocation_fail_alot_count > 0) { - _allocation_fail_alot_count--; - } else { - log_trace(gc, metaspace, freelist)("Metadata allocation failing for MetadataAllocationFailALot"); - init_allocation_fail_alot_count(); - return true; - } - } - return false; -} -#endif - - -} // namespace metaspace - - diff --git a/src/hotspot/share/memory/metaspace/metaDebug.hpp b/src/hotspot/share/memory/metaspace/metaDebug.hpp deleted file mode 100644 --- a/src/hotspot/share/memory/metaspace/metaDebug.hpp +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Copyright (c) 2018, 2019, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_MEMORY_METASPACE_METADEBUG_HPP -#define SHARE_MEMORY_METASPACE_METADEBUG_HPP - -#include "memory/allocation.hpp" - -namespace metaspace { - -class Metadebug : AllStatic { - // Debugging support for Metaspaces - static int _allocation_fail_alot_count; - - public: - - static void init_allocation_fail_alot_count(); -#ifdef ASSERT - static bool test_metadata_failure(); -#endif -}; - -#ifdef ASSERT -#define EVERY_NTH(n) \ -{ static int counter_ = 0; \ - if (n > 0) { \ - counter_ ++; \ - if (counter_ > n) { \ - counter_ = 0; \ - -#define END_EVERY_NTH } } } -#endif // ASSERT - -} // namespace metaspace - -#endif // SHARE_MEMORY_METASPACE_METADEBUG_HPP diff --git a/src/hotspot/share/memory/metaspace/metabase.hpp b/src/hotspot/share/memory/metaspace/metabase.hpp deleted file mode 100644 --- a/src/hotspot/share/memory/metaspace/metabase.hpp +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Copyright (c) 2012, 2020, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_MEMORY_METASPACE_METABASE_HPP -#define SHARE_MEMORY_METASPACE_METABASE_HPP - -#include "utilities/globalDefinitions.hpp" - -namespace metaspace { - -// Super class of Metablock and Metachunk to allow them to -// be put on the FreeList and in the BinaryTreeDictionary. -template -class Metabase { - size_t _word_size; - T* _next; - T* _prev; - - protected: - Metabase(size_t word_size) : _word_size(word_size), _next(NULL), _prev(NULL) {} - - public: - T* next() const { return _next; } - T* prev() const { return _prev; } - void set_next(T* v) { _next = v; assert(v != this, "Boom");} - void set_prev(T* v) { _prev = v; assert(v != this, "Boom");} - void clear_next() { set_next(NULL); } - void clear_prev() { set_prev(NULL); } - - size_t size() const { return _word_size; } - - void link_next(T* ptr) { set_next(ptr); } - void link_prev(T* ptr) { set_prev(ptr); } - void link_after(T* ptr) { - link_next(ptr); - if (ptr != NULL) ptr->link_prev((T*)this); - } - - uintptr_t* end() const { return ((uintptr_t*) this) + size(); } - - bool cantCoalesce() const { return false; } - - // Debug support -#ifdef ASSERT - void* prev_addr() const { return (void*)&_prev; } - void* next_addr() const { return (void*)&_next; } - void* size_addr() const { return (void*)&_word_size; } -#endif - bool verify_chunk_in_free_list(T* tc) const { return true; } - bool verify_par_locked() { return true; } - - void assert_is_mangled() const {/* Don't check "\*/} - - bool is_free() { return true; } -}; - -} // namespace metaspace - -#endif // SHARE_MEMORY_METASPACE_METABASE_HPP diff --git a/src/hotspot/share/memory/metaspace/metablock.hpp b/src/hotspot/share/memory/metaspace/metablock.hpp deleted file mode 100644 --- a/src/hotspot/share/memory/metaspace/metablock.hpp +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright (c) 2012, 2019, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_MEMORY_METASPACE_METABLOCK_HPP -#define SHARE_MEMORY_METASPACE_METABLOCK_HPP - -#include "memory/metaspace/metabase.hpp" -#include "utilities/globalDefinitions.hpp" - -namespace metaspace { - -// Metablock is the unit of allocation from a Chunk. -// -// A Metablock may be reused by its SpaceManager but are never moved between -// SpaceManagers. There is no explicit link to the Metachunk -// from which it was allocated. Metablock may be deallocated and -// put on a freelist but the space is never freed, rather -// the Metachunk it is a part of will be deallocated when it's -// associated class loader is collected. - -class Metablock : public Metabase { - friend class VMStructs; - public: - Metablock(size_t word_size) : Metabase(word_size) {} -}; - -} // namespace metaspace - -#endif // SHARE_MEMORY_METASPACE_METABLOCK_HPP diff --git a/src/hotspot/share/memory/metaspace/metachunk.cpp b/src/hotspot/share/memory/metaspace/metachunk.cpp --- a/src/hotspot/share/memory/metaspace/metachunk.cpp +++ b/src/hotspot/share/memory/metaspace/metachunk.cpp @@ -1,5 +1,6 @@ /* - * Copyright (c) 2012, 2016, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2012, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2017, 2020 SAP SE. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -23,149 +24,483 @@ */ #include "precompiled.hpp" -#include "memory/allocation.hpp" + +#include "logging/log.hpp" +#include "memory/metaspace/chunkLevel.hpp" #include "memory/metaspace/metachunk.hpp" -#include "memory/metaspace/occupancyMap.hpp" +#include "memory/metaspace/metaspaceCommon.hpp" +#include "memory/metaspace/settings.hpp" #include "memory/metaspace/virtualSpaceNode.hpp" +#include "runtime/mutexLocker.hpp" + #include "utilities/align.hpp" #include "utilities/copy.hpp" #include "utilities/debug.hpp" namespace metaspace { -size_t Metachunk::object_alignment() { - // Must align pointers and sizes to 8, - // so that 64 bit types get correctly aligned. - const size_t alignment = 8; - - // Make sure that the Klass alignment also agree. - STATIC_ASSERT(alignment == (size_t)KlassAlignmentInBytes); - - return alignment; +// Return a single char presentation of the state ('f', 'u', 'd') +char Metachunk::get_state_char() const { + switch (_state) { + case state_free: return 'f'; + case state_in_use: return 'u'; + case state_dead: return 'd'; + } + return '?'; } -size_t Metachunk::overhead() { - return align_up(sizeof(Metachunk), object_alignment()) / BytesPerWord; +#ifdef ASSERT +void Metachunk::assert_have_expand_lock() { + assert_lock_strong(MetaspaceExpand_lock); +} +#endif + +// Commit uncommitted section of the chunk. +// Fails if we hit a commit limit. +bool Metachunk::commit_up_to(size_t new_committed_words) { + + // Please note: + // + // VirtualSpaceNode::ensure_range_is_committed(), when called over a range containing both committed and uncommitted parts, + // will replace the whole range with a new mapping, thus erasing the existing content in the committed parts. Therefore + // we must make sure never to call VirtualSpaceNode::ensure_range_is_committed() over a range containing live data. + // + // Luckily, this cannot happen by design. We have two cases: + // + // 1) chunks equal or larger than a commit granule. + // In this case, due to chunk geometry, the chunk should cover whole commit granules (in other words, a chunk equal or larger than + // a commit granule will never share a granule with a neighbor). That means whatever we commit or uncommit here does not affect + // neighboring chunks. We only have to take care not to re-commit used parts of ourself. We do this by moving the committed_words + // limit in multiple of commit granules. + // + // 2) chunks smaller than a commit granule. + // In this case, a chunk shares a single commit granule with its neighbors. But this never can be a problem: + // - Either the commit granule is already committed (and maybe the neighbors contain live data). In that case calling + // ensure_range_is_committed() will do nothing. + // - Or the commit granule is not committed, but in this case, the neighbors are uncommitted too and cannot contain live data. + +#ifdef ASSERT + if (word_size() >= Settings::commit_granule_words()) { + // case (1) + assert(is_aligned(base(), Settings::commit_granule_bytes()) && + is_aligned(end(), Settings::commit_granule_bytes()), + "Chunks larger than a commit granule must cover whole granules."); + assert(is_aligned(_committed_words, Settings::commit_granule_words()), + "The commit boundary must be aligned to commit granule size"); + assert(_used_words <= _committed_words, "Sanity"); + } else { + // case (2) + assert(_committed_words == 0 || _committed_words == word_size(), "Sanity"); + } +#endif + + // We should hold the expand lock at this point. + assert_lock_strong(MetaspaceExpand_lock); + + const size_t commit_from = _committed_words; + const size_t commit_to = MIN2(align_up(new_committed_words, Settings::commit_granule_words()), word_size()); + + assert(commit_from >= used_words(), "Sanity"); + assert(commit_to <= word_size(), "Sanity"); + + if (commit_to > commit_from) { + log_debug(metaspace)("Chunk " METACHUNK_FORMAT ": attempting to move commit line to " + SIZE_FORMAT " words.", METACHUNK_FORMAT_ARGS(this), commit_to); + + if (!_vsnode->ensure_range_is_committed(base() + commit_from, commit_to - commit_from)) { + DEBUG_ONLY(verify(true);) + return false; + } + } + + // Remember how far we have committed. + _committed_words = commit_to; + + DEBUG_ONLY(verify(true);) + + return true; + } -// Metachunk methods -Metachunk::Metachunk(ChunkIndex chunktype, bool is_class, size_t word_size, - VirtualSpaceNode* container) - : Metabase(word_size), - _container(container), - _top(NULL), - _sentinel(CHUNK_SENTINEL), - _chunk_type(chunktype), - _is_class(is_class), - _origin(origin_normal), - _use_count(0) -{ - _top = initial_top(); - set_is_tagged_free(false); -#ifdef ASSERT - mangle(uninitMetaWordVal); - verify(); -#endif +// Ensure that chunk is committed up to at least new_committed_words words. +// Fails if we hit a commit limit. +bool Metachunk::ensure_committed(size_t new_committed_words) { + + bool rc = true; + + if (new_committed_words > committed_words()) { + MutexLocker cl(MetaspaceExpand_lock, Mutex::_no_safepoint_check_flag); + rc = commit_up_to(new_committed_words); + } + + return rc; + } -MetaWord* Metachunk::allocate(size_t word_size) { - MetaWord* result = NULL; - // If available, bump the pointer to allocate. - if (free_word_size() >= word_size) { - result = _top; - _top = _top + word_size; +bool Metachunk::ensure_committed_locked(size_t new_committed_words) { + + // the .._locked() variant should be called if we own the lock already. + assert_lock_strong(MetaspaceExpand_lock); + + bool rc = true; + + if (new_committed_words > committed_words()) { + rc = commit_up_to(new_committed_words); } - return result; + + return rc; + } -// _bottom points to the start of the chunk including the overhead. -size_t Metachunk::used_word_size() const { - return pointer_delta(_top, bottom(), sizeof(MetaWord)); +// Uncommit chunk area. The area must be a common multiple of the +// commit granule size (in other words, we cannot uncommit chunks smaller than +// a commit granule size). +void Metachunk::uncommit() { + MutexLocker cl(MetaspaceExpand_lock, Mutex::_no_safepoint_check_flag); + uncommit_locked(); } -size_t Metachunk::free_word_size() const { - return pointer_delta(end(), _top, sizeof(MetaWord)); +void Metachunk::uncommit_locked() { + // Only uncommit chunks which are free, have no used words set (extra precaution) and are equal or larger in size than a single commit granule. + assert_lock_strong(MetaspaceExpand_lock); + assert(_state == state_free && _used_words == 0 && word_size() >= Settings::commit_granule_words(), + "Only free chunks equal or larger than commit granule size can be uncommitted " + "(chunk " METACHUNK_FULL_FORMAT ").", METACHUNK_FULL_FORMAT_ARGS(this)); + if (word_size() >= Settings::commit_granule_words()) { + _vsnode->uncommit_range(base(), word_size()); + _committed_words = 0; + } +} +void Metachunk::set_committed_words(size_t v) { + // Set committed words. Since we know that we only commit whole commit granules, we can round up v here. + v = MIN2(align_up(v, Settings::commit_granule_words()), word_size()); + _committed_words = v; } -void Metachunk::print_on(outputStream* st) const { - st->print_cr("Metachunk:" - " bottom " PTR_FORMAT " top " PTR_FORMAT - " end " PTR_FORMAT " size " SIZE_FORMAT " (%s)", - p2i(bottom()), p2i(_top), p2i(end()), word_size(), - chunk_size_name(get_chunk_type())); - if (Verbose) { - st->print_cr(" used " SIZE_FORMAT " free " SIZE_FORMAT, - used_word_size(), free_word_size()); + +// Allocate word_size words from this chunk (word_size must be aligned to +// allocation_alignment_words). +// +// Caller must make sure the chunk is both large enough and committed far enough +// to hold the allocation. Will always work. +// +MetaWord* Metachunk::allocate(size_t request_word_size) { + + log_trace(metaspace)("Chunk " METACHUNK_FULL_FORMAT ": allocating " SIZE_FORMAT " words.", + METACHUNK_FULL_FORMAT_ARGS(this), request_word_size); + + // Caller must have made sure this works + assert(free_words() >= request_word_size, "Chunk too small."); + assert(free_below_committed_words() >= request_word_size, "Chunk not committed."); + + MetaWord* const p = top(); + + _used_words += request_word_size; + + SOMETIMES(verify(false);) + + return p; + +} + +#ifdef ASSERT + +// Zap this structure. +void Metachunk::zap_header(uint8_t c) { + memset(this, c, sizeof(Metachunk)); +} + +void Metachunk::fill_with_pattern(MetaWord pattern, size_t word_size) { + assert(word_size <= committed_words(), "Sanity"); + for (size_t l = 0; l < word_size; l ++) { + _base[l] = pattern; } } -#ifdef ASSERT -void Metachunk::mangle(juint word_value) { - // Overwrite the payload of the chunk and not the links that - // maintain list of chunks. - HeapWord* start = (HeapWord*)initial_top(); - size_t size = word_size() - overhead(); - Copy::fill_to_words(start, size, word_value); -} +void Metachunk::check_pattern(MetaWord pattern, size_t word_size) { + assert(word_size <= committed_words(), "Sanity"); + for (size_t l = 0; l < word_size; l ++) { + assert(_base[l] == pattern, + "chunk " METACHUNK_FULL_FORMAT ": pattern change at " PTR_FORMAT ": expected " UINTX_FORMAT " but got " UINTX_FORMAT ".", + METACHUNK_FULL_FORMAT_ARGS(this), p2i(_base + l), (uintx)pattern, (uintx)_base[l]); -void Metachunk::verify() const { - assert(is_valid_sentinel(), "Chunk " PTR_FORMAT ": sentinel invalid", p2i(this)); - const ChunkIndex chunk_type = get_chunk_type(); - assert(is_valid_chunktype(chunk_type), "Chunk " PTR_FORMAT ": Invalid chunk type.", p2i(this)); - if (chunk_type != HumongousIndex) { - assert(word_size() == get_size_for_nonhumongous_chunktype(chunk_type, is_class()), - "Chunk " PTR_FORMAT ": wordsize " SIZE_FORMAT " does not fit chunk type %s.", - p2i(this), word_size(), chunk_size_name(chunk_type)); - } - assert(is_valid_chunkorigin(get_origin()), "Chunk " PTR_FORMAT ": Invalid chunk origin.", p2i(this)); - assert(bottom() <= _top && _top <= (MetaWord*)end(), - "Chunk " PTR_FORMAT ": Chunk top out of chunk bounds.", p2i(this)); + //////////////////////////////////////////// + // A double-headed list of Metachunks. - // For non-humongous chunks, starting address shall be aligned - // to its chunk size. Humongous chunks start address is - // aligned to specialized chunk size. - const size_t required_alignment = - (chunk_type != HumongousIndex ? word_size() : get_size_for_nonhumongous_chunktype(SpecializedIndex, is_class())) * sizeof(MetaWord); - assert(is_aligned((address)this, required_alignment), - "Chunk " PTR_FORMAT ": (size " SIZE_FORMAT ") not aligned to " SIZE_FORMAT ".", - p2i(this), word_size() * sizeof(MetaWord), required_alignment); -} + class AbstractMetachunkList { -#endif // ASSERT + Metachunk* _first; + Metachunk* _last; -// Helper, returns a descriptive name for the given index. -const char* chunk_size_name(ChunkIndex index) { - switch (index) { - case SpecializedIndex: - return "specialized"; - case SmallIndex: - return "small"; - case MediumIndex: - return "medium"; - case HumongousIndex: - return "humongous"; - default: - return "Invalid index"; + // Number of chunks + IntCounter _num; + + protected: + + AbstractMetachunkList() : _first(NULL), _last(NULL), _num() {} + + Metachunk* first() const { return _first; } + int count() const { return _num.get(); } + + // Add chunk to the front of the list. + void add_front(Metachunk* c) { + if (_first == NULL) { + assert(_last == NULL && _num.get() == 0, "Sanity"); + _first = _last = c; + c->set_prev(NULL); + c->set_next(NULL); + } else { + assert(_last != NULL && _num.get() > 0, "Sanity"); + c->set_next(_first); + c->set_prev(NULL); + _first->set_prev(c); + _first = c; + } + _num.increment(); + } + + // Add chunk to the back of the list. + void add_back(Metachunk* c) { + if (_last == NULL) { + assert(_first == NULL && _num.get() == 0, "Sanity"); + _last = _first = c; + c->set_prev(NULL); + c->set_next(NULL); + } else { + assert(_first != NULL && _num.get() > 0, "Sanity"); + c->set_next(NULL); + c->set_prev(_last); + _last->set_next(c); + _last = c; + } + _num.increment(); + } + + // Remove chunk from the front of the list. Returns NULL if list is empty. + Metachunk* remove_front() { + Metachunk* c = NULL; + if (_first == NULL) { + assert(_last == NULL && _num.get() == 0, "Sanity"); + } else { + c = _first; + assert(c->prev() == NULL, "Sanity"); + if (_first == _last) { + assert(_num.get() == 1, "Sanity"); + _first = _last = NULL; + } else { + assert(_num.get() > 1, "Sanity"); + _first = _first->next(); + _first->set_prev(NULL); + } + _num.decrement(); + c->set_next(NULL); + } + return c; + } + + // Remove chunk from the back of the list. Returns NULL if list is empty. + Metachunk* remove_back() { + Metachunk* c = NULL; + if (_last == NULL) { + assert(_first == NULL && _num.get() == 0, "Sanity"); + } else { + c = _last; + assert(c->next() == NULL, "Sanity"); + if (_first == _last) { + assert(_num.get() == 1, "Sanity"); + _first = _last = NULL; + } else { + assert(_num.get() > 1, "Sanity"); + _last = _last->prev(); + _last->set_next(NULL); + } + _num.decrement(); + c->set_prev(NULL); + } + return c; + } + + public: + + #ifdef ASSERT + bool contains(const Metachunk* c) const; + void verify() const; + #endif + + // Returns size, in words, of committed space of all chunks in this list. + // Note: walks list. + size_t committed_word_size() const { + size_t l = 0; + for (const Metachunk* c = _first; c != NULL; c = c->next()) { + l += c->committed_words(); + } + return l; + } + + void print_on(outputStream* st) const; + + }; + + class UnsortedMetachunkList : public AbstractMetachunkList { + public: + + + + + + }; + + } } -#ifdef ASSERT -void do_verify_chunk(Metachunk* chunk) { - guarantee(chunk != NULL, "Sanity"); - // Verify chunk itself; then verify that it is consistent with the - // occupany map of its containing node. - chunk->verify(); - VirtualSpaceNode* const vsn = chunk->container(); - OccupancyMap* const ocmap = vsn->occupancy_map(); - ocmap->verify_for_chunk(chunk); + +// Verifies linking with neighbors in virtual space. +// Can only be done under expand lock protection. +void Metachunk::verify_neighborhood() const { + + assert_lock_strong(MetaspaceExpand_lock); + assert(!is_dead(), "Do not call on dead chunks."); + + if (is_root_chunk()) { + + // Root chunks are all alone in the world. + assert(next_in_vs() == NULL || prev_in_vs() == NULL, "Root chunks should have no neighbors"); + + } else { + + // Non-root chunks have neighbors, at least one, possibly two. + + assert(next_in_vs() != NULL || prev_in_vs() != NULL, + "A non-root chunk should have neighbors (chunk @" PTR_FORMAT + ", base " PTR_FORMAT ", level " CHKLVL_FORMAT ".", + p2i(this), p2i(base()), level()); + + if (prev_in_vs() != NULL) { + assert(prev_in_vs()->end() == base(), + "Chunk " METACHUNK_FULL_FORMAT ": should be adjacent to predecessor: " METACHUNK_FULL_FORMAT ".", + METACHUNK_FULL_FORMAT_ARGS(this), METACHUNK_FULL_FORMAT_ARGS(prev_in_vs())); + assert(prev_in_vs()->next_in_vs() == this, + "Chunk " METACHUNK_FULL_FORMAT ": broken link to left neighbor: " METACHUNK_FULL_FORMAT " (" PTR_FORMAT ").", + METACHUNK_FULL_FORMAT_ARGS(this), METACHUNK_FULL_FORMAT_ARGS(prev_in_vs()), p2i(prev_in_vs()->next_in_vs())); + } + + if (next_in_vs() != NULL) { + assert(end() == next_in_vs()->base(), + "Chunk " METACHUNK_FULL_FORMAT ": should be adjacent to successor: " METACHUNK_FULL_FORMAT ".", + METACHUNK_FULL_FORMAT_ARGS(this), METACHUNK_FULL_FORMAT_ARGS(next_in_vs())); + assert(next_in_vs()->prev_in_vs() == this, + "Chunk " METACHUNK_FULL_FORMAT ": broken link to right neighbor: " METACHUNK_FULL_FORMAT " (" PTR_FORMAT ").", + METACHUNK_FULL_FORMAT_ARGS(this), METACHUNK_FULL_FORMAT_ARGS(next_in_vs()), p2i(next_in_vs()->prev_in_vs())); + } + + // One of the neighbors must be the buddy. It can be whole or splintered. + + // The chunk following us or preceeding us may be our buddy or a splintered part of it. + Metachunk* buddy = is_leader() ? next_in_vs() : prev_in_vs(); + + assert(buddy != NULL, "Missing neighbor."); + assert(!buddy->is_dead(), "Invalid buddy state."); + + // This neighbor is either or buddy (same level) or a splinter of our buddy - hence + // the level can never be smaller (aka the chunk size cannot be larger). + assert(buddy->level() >= level(), "Wrong level."); + + if (buddy->level() == level()) { + + // If the buddy is of the same size as us, it is unsplintered. + assert(buddy->is_leader() == !is_leader(), + "Only one chunk can be leader in a pair"); + + // When direct buddies are neighbors, one or both should be in use, otherwise they should + // have been merged. + + // But since we call this verification function from internal functions where we are about to merge or just did split, + // do not test this. We have RootChunkArea::verify_area_is_ideally_merged() for testing that. + + // assert(buddy->is_in_use() || is_in_use(), "incomplete merging?"); + + if (is_leader()) { + assert(buddy->base() == end(), "Sanity"); + assert(is_aligned(base(), word_size() * 2 * BytesPerWord), "Sanity"); + } else { + assert(buddy->end() == base(), "Sanity"); + assert(is_aligned(buddy->base(), word_size() * 2 * BytesPerWord), "Sanity"); + } + + } else { + + // Buddy, but splintered, and this is a part of it. + if (is_leader()) { + assert(buddy->base() == end(), "Sanity"); + } else { + assert(buddy->end() > (base() - word_size()), "Sanity"); + } + + } + } } -#endif -void do_update_in_use_info_for_chunk(Metachunk* chunk, bool inuse) { - chunk->set_is_tagged_free(!inuse); - OccupancyMap* const ocmap = chunk->container()->occupancy_map(); - ocmap->set_region_in_use((MetaWord*)chunk, chunk->word_size(), inuse); +volatile MetaWord dummy = 0; + +void Metachunk::verify(bool slow) const { + + // Note. This should be called under CLD lock protection. + + // We can verify everything except the _prev_in_vs/_next_in_vs pair. + // This is because neighbor chunks may be added concurrently, so we cannot rely + // on the content of _next_in_vs/_prev_in_vs unless we have the expand lock. + + assert(!is_dead(), "Do not call on dead chunks."); + + if (is_free()) { + assert(used_words() == 0, "free chunks are not used."); + } + + // Note: only call this on a life Metachunk. + chunklevel::check_valid_level(level()); + + assert(base() != NULL, "No base ptr"); + + assert(committed_words() >= used_words(), + "mismatch: committed: " SIZE_FORMAT ", used: " SIZE_FORMAT ".", + committed_words(), used_words()); + + assert(word_size() >= committed_words(), + "mismatch: word_size: " SIZE_FORMAT ", committed: " SIZE_FORMAT ".", + word_size(), committed_words()); + + // Test base pointer + assert(base() != NULL, "Base pointer NULL"); + assert(vsnode() != NULL, "No space"); + vsnode()->check_pointer(base()); + + // Starting address shall be aligned to chunk size. + const size_t required_alignment = word_size() * sizeof(MetaWord); + assert_is_aligned(base(), required_alignment); + + // If slow, test the committed area + if (slow && _committed_words > 0) { + for (const MetaWord* p = _base; p < _base + _committed_words; p += os::vm_page_size()) { + dummy = *p; + } + dummy = *(_base + _committed_words - 1); + } + +} +#endif // ASSERT + +void Metachunk::print_on(outputStream* st) const { + + // Note: must also work with invalid/random data. (e.g. do not call word_size()) + st->print("Chunk @" PTR_FORMAT ", state %c, base " PTR_FORMAT ", " + "level " CHKLVL_FORMAT " (" SIZE_FORMAT " words), " + "used " SIZE_FORMAT " words, committed " SIZE_FORMAT " words.", + p2i(this), get_state_char(), p2i(base()), level(), + (chunklevel::is_valid_level(level()) ? chunklevel::word_size_for_level(level()) : (size_t)-1), + used_words(), committed_words()); + } } // namespace metaspace diff --git a/src/hotspot/share/memory/metaspace/metachunk.hpp b/src/hotspot/share/memory/metaspace/metachunk.hpp --- a/src/hotspot/share/memory/metaspace/metachunk.hpp +++ b/src/hotspot/share/memory/metaspace/metachunk.hpp @@ -1,5 +1,6 @@ /* - * Copyright (c) 2012, 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2012, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2017, 2020 SAP SE. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -21,152 +22,299 @@ * questions. * */ + #ifndef SHARE_MEMORY_METASPACE_METACHUNK_HPP #define SHARE_MEMORY_METASPACE_METACHUNK_HPP -#include "memory/metaspace/metabase.hpp" -#include "memory/metaspace/metaspaceCommon.hpp" + +#include "memory/metaspace/counter.hpp" +#include "memory/metaspace/chunkLevel.hpp" #include "utilities/debug.hpp" #include "utilities/globalDefinitions.hpp" -class MetachunkTest; + +class outputStream; namespace metaspace { class VirtualSpaceNode; -// Metachunk - Quantum of allocation from a Virtualspace -// Metachunks are reused (when freed are put on a global freelist) and -// have no permanent association to a SpaceManager. +// A Metachunk is a contiguous metaspace memory region. It is part of +// a MetaspaceArena, which keeps a list of MetaChunk and allocates via +// pointer bump from the top element in the list. +// +// The Metachunk object itself (the "chunk header") is separated from +// the memory region (the chunk payload) it describes. It also can have +// no payload (a "dead" chunk). In itself it lives in C-heap, managed +// as part of a pool of Metachunk headers (ChunkHeaderPool). +// +// -- Metachunk state -- +// +// A Metachunk is "in-use" if it is part of a MetaspaceArena. That means +// its memory is used - or will be used shortly - to hold VM metadata +// on behalf of a class loader. +// +// A Metachunk is "free" if its payload is currently unused. In that +// case it is managed by a chunk freelist (the ChunkManager). +// +// A Metachunk is "dead" if it does not have a corresponding payload. +// In that case it lives as part of a freelist-of-dead-chunk-headers +// in the ChunkHeaderPool. +// +// -- Level -- +// +// Metachunks are managed as part of a buddy style allocation scheme. +// Sized always in steps of power-of-2, ranging from the smallest chunk size +// (1Kb) to the largest (4Mb) (see chunklevel.hpp). +// Its size is encoded as level, with level 0 being the largest chunk +// size ("root chunk"). +// +// -- Payload commit state -- +// +// A Metachunk payload may be committed, partly committed or completely +// uncommitted. Technically, a payload may be committed "checkered" - +// i.e. committed and uncommitted parts may interleave - but the +// important part is how much contiguous space is committed starting +// at the base of the payload (since that's where we allocate). +// +// The Metachunk keeps track of how much space is committed starting +// at the base of the payload - which is a performace optimization - +// while underlying layers (VirtualSpaceNode->commitmask) keep track +// of the "real" commit state, aka which granules are committed, +// independent on what chunks reside above those granules. -// +--------------+ <- end --+ --+ -// | | | | -// | | | free | -// | | | | -// | | | | size | capacity -// | | | | -// | | <- top -- + | -// | | | | -// | | | used | -// | | | | -// | | | | -// +--------------+ <- bottom --+ --+ -enum ChunkOrigin { - // Chunk normally born (via take_from_committed) - origin_normal = 1, - // Chunk was born as padding chunk - origin_pad = 2, - // Chunk was born as leftover chunk in VirtualSpaceNode::retire - origin_leftover = 3, - // Chunk was born as result of a merge of smaller chunks - origin_merge = 4, - // Chunk was born as result of a split of a larger chunk - origin_split = 5, +// +--------------+ <- end -----------+ ----------+ +// | | | | +// | | | | +// | | | | +// | | | | +// | | | | +// | ----------- | <- committed_top -- + | +// | | | | +// | | | "free" | +// | | | | size +// | | "free_below_ | | +// | | committed" | | +// | | | | +// | | | | +// | ----------- | <- top --------- + -------- | +// | | | | +// | | "used" | | +// | | | | +// +--------------+ <- start ----------+ ----------+ - origin_minimum = origin_normal, - origin_maximum = origin_split, - origins_count = origin_maximum + 1 -}; +// Note: this is a chunk **descriptor**. The real Payload area lives in metaspace, +// this class lives somewhere else. +class Metachunk { -inline bool is_valid_chunkorigin(ChunkOrigin origin) { - return origin == origin_normal || - origin == origin_pad || - origin == origin_leftover || - origin == origin_merge || - origin == origin_split; -} + // start of chunk memory; NULL if dead. + MetaWord* _base; -class Metachunk : public Metabase { + // Used words. + size_t _used_words; - friend class ::MetachunkTest; + // Size of the region, starting from base, which is guaranteed to be committed. In words. + // The actual size of committed regions may actually be larger. + // + // (This is a performance optimization. The underlying VirtualSpaceNode knows + // which granules are committed; but we want to avoid having to ask.) + size_t _committed_words; - // The VirtualSpaceNode containing this chunk. - VirtualSpaceNode* const _container; + chunklevel_t _level; // aka size. - // Current allocation top. - MetaWord* _top; + // state_free: free, owned by a ChunkManager + // state_in_use: in-use, owned by a MetaspaceArena + // dead: just a hollow chunk header without associated memory, owned + // by chunk header pool. + enum state_t { + state_free = 0, + state_in_use = 1, + state_dead = 2 + }; + state_t _state; - // A 32bit sentinel for debugging purposes. - enum { CHUNK_SENTINEL = 0x4d4554EF, // "MET" - CHUNK_SENTINEL_INVALID = 0xFEEEEEEF - }; + // We need unfortunately a back link to the virtual space node + // for splitting and merging nodes. + VirtualSpaceNode* _vsnode; - uint32_t _sentinel; - const ChunkIndex _chunk_type; - const bool _is_class; - // Whether the chunk is free (in freelist) or in use by some class loader. - bool _is_tagged_free; + // A chunk header is kept in a list: + // 1 in the list of used chunks inside a MetaspaceArena, if it is in use + // 2 in the list of free chunks inside a ChunkManager, if it is free + // 3 in the freelist of unused headers inside the ChunkHeaderPool, + // if it is unused (e.g. result of chunk merging) and has no associated + // memory area. + Metachunk* _prev; + Metachunk* _next; - ChunkOrigin _origin; - int _use_count; + // Furthermore, we keep, per chunk, information about the neighboring chunks. + // This is needed to split and merge chunks. + // + // Note: These members can be modified concurrently while a chunk is alive and in use. + // This can happen if a neighboring chunk is added or removed. + // This means only read or modify these members under expand lock protection. + Metachunk* _prev_in_vs; + Metachunk* _next_in_vs; - MetaWord* initial_top() const { return (MetaWord*)this + overhead(); } - MetaWord* top() const { return _top; } + // Commit uncommitted section of the chunk. + // Fails if we hit a commit limit. + bool commit_up_to(size_t new_committed_words); - public: - // Metachunks are allocated out of a MetadataVirtualSpace and - // and use some of its space to describe itself (plus alignment - // considerations). Metadata is allocated in the rest of the chunk. - // This size is the overhead of maintaining the Metachunk within - // the space. + DEBUG_ONLY(static void assert_have_expand_lock();) - // Alignment of each allocation in the chunks. - static size_t object_alignment(); +public: - // Size of the Metachunk header, in words, including alignment. - static size_t overhead(); + Metachunk() + : _base(NULL), + _used_words(0), + _committed_words(0), + _level(chunklevel::ROOT_CHUNK_LEVEL), + _state(state_free), + _vsnode(NULL), + _prev(NULL), _next(NULL), + _prev_in_vs(NULL), _next_in_vs(NULL) + {} - Metachunk(ChunkIndex chunktype, bool is_class, size_t word_size, VirtualSpaceNode* container); + void clear() { + _base = NULL; + _used_words = 0; + _committed_words = 0; + _level = chunklevel::ROOT_CHUNK_LEVEL; + _state = state_free; + _vsnode = NULL; + _prev = NULL; + _next = NULL; + _prev_in_vs = NULL; + _next_in_vs = NULL; + } - MetaWord* allocate(size_t word_size); - VirtualSpaceNode* container() const { return _container; } + size_t word_size() const { return chunklevel::word_size_for_level(_level); } - MetaWord* bottom() const { return (MetaWord*) this; } + MetaWord* base() const { return _base; } + MetaWord* top() const { return base() + _used_words; } + MetaWord* committed_top() const { return base() + _committed_words; } + MetaWord* end() const { return base() + word_size(); } - // Reset top to bottom so chunk can be reused. - void reset_empty() { _top = initial_top(); clear_next(); clear_prev(); } - bool is_empty() { return _top == initial_top(); } + // Chunk list wiring + void set_prev(Metachunk* c) { _prev = c; } + Metachunk* prev() const { return _prev; } + void set_next(Metachunk* c) { _next = c; } + Metachunk* next() const { return _next; } - // used (has been allocated) - // free (available for future allocations) - size_t word_size() const { return size(); } - size_t used_word_size() const; - size_t free_word_size() const; + DEBUG_ONLY(bool in_list() const { return _prev != NULL || _next != NULL; }) - bool is_tagged_free() { return _is_tagged_free; } - void set_is_tagged_free(bool v) { _is_tagged_free = v; } + // Physical neighbors wiring + void set_prev_in_vs(Metachunk* c) { DEBUG_ONLY(assert_have_expand_lock()); _prev_in_vs = c; } + Metachunk* prev_in_vs() const { DEBUG_ONLY(assert_have_expand_lock()); return _prev_in_vs; } + void set_next_in_vs(Metachunk* c) { DEBUG_ONLY(assert_have_expand_lock()); _next_in_vs = c; } + Metachunk* next_in_vs() const { DEBUG_ONLY(assert_have_expand_lock()); return _next_in_vs; } - bool contains(const void* ptr) { return bottom() <= ptr && ptr < _top; } + bool is_free() const { return _state == state_free; } + bool is_in_use() const { return _state == state_in_use; } + bool is_dead() const { return _state == state_dead; } + void set_free() { _state = state_free; } + void set_in_use() { _state = state_in_use; } + void set_dead() { _state = state_dead; } + + // Return a single char presentation of the state ('f', 'u', 'd') + char get_state_char() const; + + void inc_level() { _level ++; DEBUG_ONLY(chunklevel::is_valid_level(_level);) } + void dec_level() { _level --; DEBUG_ONLY(chunklevel::is_valid_level(_level);) } + chunklevel_t level() const { return _level; } + + // Convenience functions for extreme levels. + bool is_root_chunk() const { return chunklevel::ROOT_CHUNK_LEVEL == _level; } + bool is_leaf_chunk() const { return chunklevel::HIGHEST_CHUNK_LEVEL == _level; } + + VirtualSpaceNode* vsnode() const { return _vsnode; } + + size_t used_words() const { return _used_words; } + size_t free_words() const { return word_size() - used_words(); } + size_t free_below_committed_words() const { return committed_words() - used_words(); } + void reset_used_words() { _used_words = 0; } + + size_t committed_words() const { return _committed_words; } + void set_committed_words(size_t v); + bool is_fully_committed() const { return committed_words() == word_size(); } + bool is_fully_uncommitted() const { return committed_words() == 0; } + + // Ensure that chunk is committed up to at least new_committed_words words. + // Fails if we hit a commit limit. + bool ensure_committed(size_t new_committed_words); + bool ensure_committed_locked(size_t new_committed_words); + + bool ensure_fully_committed() { return ensure_committed(word_size()); } + bool ensure_fully_committed_locked() { return ensure_committed_locked(word_size()); } + + // Ensure that the chunk is committed far enough to serve an additional allocation of word_size. + bool ensure_committed_additional(size_t additional_word_size) { + return ensure_committed(used_words() + additional_word_size); + } + + // Uncommit chunk area. The area must be a common multiple of the + // commit granule size (in other words, we cannot uncommit chunks smaller than + // a commit granule size). + void uncommit(); + void uncommit_locked(); + + // Allocation from a chunk + + // Allocate word_size words from this chunk (word_size must be aligned to + // allocation_alignment_words). + // + // Caller must make sure the chunk is both large enough and committed far enough + // to hold the allocation. Will always work. + // + MetaWord* allocate(size_t request_word_size); + + // Initialize structure for reuse. + void initialize(VirtualSpaceNode* node, MetaWord* base, chunklevel_t lvl) { + _vsnode = node; _base = base; _level = lvl; + _used_words = _committed_words = 0; _state = state_free; + _next = _prev = _next_in_vs = _prev_in_vs = NULL; + } + + // Returns true if this chunk is the leader in its buddy pair, false if not. + // Do not call for root chunks. + bool is_leader() const { + assert(!is_root_chunk(), "Root chunks have no buddy."); // Bit harsh? + return is_aligned(base(), chunklevel::word_size_for_level(level() - 1) * BytesPerWord); + } + + //// Debug stuff //// +#ifdef ASSERT + void verify(bool slow) const; + // Verifies linking with neighbors in virtual space. Needs expand lock protection. + void verify_neighborhood() const; + void zap_header(uint8_t c = 0x17); + void fill_with_pattern(MetaWord pattern, size_t word_size); + void check_pattern(MetaWord pattern, size_t word_size); + + // Returns true if given pointer points into the payload area of this chunk. + bool is_valid_pointer(const MetaWord* p) const { + return base() <= p && p < top(); + } + + // Returns true if given pointer points into the commmitted payload area of this chunk. + bool is_valid_committed_pointer(const MetaWord* p) const { + return base() <= p && p < committed_top(); + } + +#endif // ASSERT void print_on(outputStream* st) const; - bool is_valid_sentinel() const { return _sentinel == CHUNK_SENTINEL; } - void remove_sentinel() { _sentinel = CHUNK_SENTINEL_INVALID; } - - int get_use_count() const { return _use_count; } - void inc_use_count() { _use_count ++; } - - ChunkOrigin get_origin() const { return _origin; } - void set_origin(ChunkOrigin orig) { _origin = orig; } - - ChunkIndex get_chunk_type() const { return _chunk_type; } - bool is_class() const { return _is_class; } - - DEBUG_ONLY(void mangle(juint word_value);) - DEBUG_ONLY(void verify() const;) - }; +// Little print helpers: since we often print out chunks, here some convenience macros +#define METACHUNK_FORMAT "@" PTR_FORMAT ", %c, base " PTR_FORMAT ", level " CHKLVL_FORMAT +#define METACHUNK_FORMAT_ARGS(chunk) p2i(chunk), chunk->get_state_char(), p2i(chunk->base()), chunk->level() -// Helper function that does a bunch of checks for a chunk. -DEBUG_ONLY(void do_verify_chunk(Metachunk* chunk);) - -// Given a Metachunk, update its in-use information (both in the -// chunk and the occupancy map). -void do_update_in_use_info_for_chunk(Metachunk* chunk, bool inuse); +#define METACHUNK_FULL_FORMAT "@" PTR_FORMAT ", %c, base " PTR_FORMAT ", level " CHKLVL_FORMAT " (" SIZE_FORMAT "), used: " SIZE_FORMAT ", committed: " SIZE_FORMAT ", committed-free: " SIZE_FORMAT +#define METACHUNK_FULL_FORMAT_ARGS(chunk) p2i(chunk), chunk->get_state_char(), p2i(chunk->base()), chunk->level(), chunk->word_size(), chunk->used_words(), chunk->committed_words(), chunk->free_below_committed_words() } // namespace metaspace diff --git a/src/hotspot/share/memory/metaspace/metachunkList.cpp b/src/hotspot/share/memory/metaspace/metachunkList.cpp new file mode 100644 --- /dev/null +++ b/src/hotspot/share/memory/metaspace/metachunkList.cpp @@ -0,0 +1,113 @@ +/* + * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020 SAP SE. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" + +#include "memory/metaspace/metachunkList.hpp" +#include "utilities/debug.hpp" +#include "utilities/globalDefinitions.hpp" +#include "utilities/ostream.hpp" + + +namespace metaspace { + +#ifdef ASSERT + +bool MetachunkList::contains(const Metachunk* c) const { + for (Metachunk* c2 = _first; c2 != NULL; c2 = c2->next()) { + if (c == c2) { + return true; + } + } + return false; +} + +void MetachunkList::verify() const { + int num = 0; + const Metachunk* last_c = NULL; + for (const Metachunk* c = _first; c != NULL; c = c->next()) { + num ++; + assert(c->prev() != c && c->next() != c, "circularity"); + assert(c->prev() == last_c, + "Broken link to predecessor. Chunk " METACHUNK_FULL_FORMAT ".", + METACHUNK_FULL_FORMAT_ARGS(c)); + c->verify(false); + last_c = c; + } + _num_chunks.check(num); +} + +#endif // ASSERT + + +size_t MetachunkList::calc_committed_word_size() const { + + if (_first != NULL && _first->is_dead()) { + // list used for chunk header pool; dead chunks have no size. + return 0; + } + + size_t s = 0; + for (Metachunk* c = _first; c != NULL; c = c->next()) { + assert(c->is_dead() == false, "Sanity"); + s += c->committed_words(); + } + return s; +} + +size_t MetachunkList::calc_word_size() const { + + if (_first != NULL && _first->is_dead()) { + // list used for chunk header pool; dead chunks have no size. + return 0; + } + + size_t s = 0; + for (Metachunk* c = _first; c != NULL; c = c->next()) { + assert(c->is_dead() == false, "Sanity"); + s += c->committed_words(); + } + return s; + +} + +void MetachunkList::print_on(outputStream* st) const { + + if (_num_chunks.get() > 0) { + for (const Metachunk* c = _first; c != NULL; c = c->next()) { + st->print(" - <"); + c->print_on(st); + st->print(">"); + } + st->print(" - total : %d chunks.", _num_chunks.get()); + } else { + st->print("empty"); + } + +} + + +} // namespace metaspace + diff --git a/src/hotspot/share/memory/metaspace/metachunkList.hpp b/src/hotspot/share/memory/metaspace/metachunkList.hpp new file mode 100644 --- /dev/null +++ b/src/hotspot/share/memory/metaspace/metachunkList.hpp @@ -0,0 +1,105 @@ +/* + * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020 SAP SE. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_MEMORY_METASPACE_METACHUNKLIST_HPP +#define SHARE_MEMORY_METASPACE_METACHUNKLIST_HPP + + +#include "memory/metaspace/counter.hpp" +#include "memory/metaspace/metachunk.hpp" +#include "memory/metaspace/metaspaceCommon.hpp" +#include "utilities/debug.hpp" +#include "utilities/globalDefinitions.hpp" + + +class outputStream; + +namespace metaspace { + +// A simple single-linked list of chunks, used in MetaspaceArena to keep +// a list of retired chunks, as well as in the ChunkHeaderPool to keep +// a cache of unused chunk headers. + +class MetachunkList { + + Metachunk* _first; + IntCounter _num_chunks; + + // Note: The chunks inside this list may be dead (->chunk header pool). + // So, do not call c->word size on them or anything else which may not + // work with dead chunks. + +public: + + MetachunkList() : _first(NULL), _num_chunks() {} + + int count() const { return _num_chunks.get(); } + + void add(Metachunk* c) { + // Note: contains is expensive (linear search). + ASSERT_SOMETIMES(contains(c) == false, "Chunk already in this list"); + c->set_next(_first); + if (_first) { + _first->set_prev(c); + } + _first = c; + _num_chunks.increment(); + } + + Metachunk* remove_first() { + if (_first) { + Metachunk* c = _first; + _first = _first->next(); + if (_first) { + _first->set_prev(NULL); + } + _num_chunks.decrement(); + c->set_prev(NULL); + c->set_next(NULL); + return c; + } + return NULL; + } + + Metachunk* first() { return _first; } + const Metachunk* first() const { return _first; } + +#ifdef ASSERT + // Note: linear search + bool contains(const Metachunk* c) const; + void verify() const; +#endif + + size_t calc_committed_word_size() const; + size_t calc_word_size() const; + + void print_on(outputStream* st) const; + +}; + + +} // namespace metaspace + +#endif // SHARE_MEMORY_METASPACE_METACHUNKLIST_HPP diff --git a/src/hotspot/share/memory/metaspace/metaspaceArena.cpp b/src/hotspot/share/memory/metaspace/metaspaceArena.cpp new file mode 100644 --- /dev/null +++ b/src/hotspot/share/memory/metaspace/metaspaceArena.cpp @@ -0,0 +1,556 @@ +/* + * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020 SAP SE. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" + +#include "logging/log.hpp" +#include "logging/logStream.hpp" +#include "memory/metaspace/allocationGuard.hpp" +#include "memory/metaspace/arenaGrowthPolicy.hpp" +#include "memory/metaspace/freeBlocks.hpp" +#include "memory/metaspace/chunkManager.hpp" +#include "memory/metaspace/internStat.hpp" +#include "memory/metaspace/metachunk.hpp" +#include "memory/metaspace/metaspaceArena.hpp" +#include "memory/metaspace/metaspaceCommon.hpp" +#include "memory/metaspace/metaspaceStatistics.hpp" +#include "memory/metaspace/virtualSpaceList.hpp" +#include "runtime/atomic.hpp" +#include "runtime/init.hpp" +#include "services/memoryService.hpp" +#include "utilities/align.hpp" +#include "utilities/debug.hpp" +#include "utilities/globalDefinitions.hpp" + +namespace metaspace { + +#define LOGFMT "Arena @" PTR_FORMAT " (%s)" +#define LOGFMT_ARGS p2i(this), this->_name + +// Given a net allocation word size, return the raw word size we actually allocate. +// Note: externally visible for gtests. +//static +size_t get_raw_allocation_word_size(size_t net_word_size) { + + size_t byte_size = net_word_size * BytesPerWord; + + // Deallocated metablocks are kept in a binlist which limits their minimal + // size to at least the size of a binlist item (2 words). + byte_size = MAX2(byte_size, FreeBlocks::minimal_word_size * BytesPerWord); + + // Metaspace allocations are aligned to word size. + byte_size = align_up(byte_size, allocation_alignment_bytes); + + // If we guard allocations, we need additional space for a prefix. +#ifdef ASSERT + if (Settings::use_allocation_guard()) { + byte_size += align_up(prefix_size(), allocation_alignment_bytes); + } +#endif + + size_t word_size = byte_size / BytesPerWord; + + assert(word_size * BytesPerWord == byte_size, "Sanity"); + + return word_size; + +} + +// Returns the level of the next chunk to be added, acc to growth policy. +chunklevel_t MetaspaceArena::next_chunk_level() const { + const int growth_step = _chunks.count(); + return _growth_policy->get_level_at_step(growth_step); +} + +// Given a chunk, add its remaining free committed space to the free block list. +void MetaspaceArena::salvage_chunk(Metachunk* c) { + + if (Settings::handle_deallocations() == false) { + return; + } + + assert_lock_strong(lock()); + + // If the chunk is completely empty, just return it to the chunk manager. + if (c->used_words() == 0) { + UL2(trace, "salvage: returning empty chunk " METACHUNK_FORMAT ".", METACHUNK_FORMAT_ARGS(c)); + _chunk_manager->return_chunk(c); + return; + } + + size_t remaining_words = c->free_below_committed_words(); + + if (remaining_words > FreeBlocks::minimal_word_size) { + + UL2(trace, "salvaging chunk " METACHUNK_FULL_FORMAT ".", METACHUNK_FULL_FORMAT_ARGS(c)); + + MetaWord* ptr = c->allocate(remaining_words); + assert(ptr != NULL, "Should have worked"); + _total_used_words_counter->increment_by(remaining_words); + + add_allocation_to_fbl(ptr, remaining_words); + + // After this operation: the chunk should have no free committed space left. + assert(c->free_below_committed_words() == 0, + "Salvaging chunk failed (chunk " METACHUNK_FULL_FORMAT ").", + METACHUNK_FULL_FORMAT_ARGS(c)); + + } + +} + +// Allocate a new chunk from the underlying chunk manager able to hold at least +// requested word size. +Metachunk* MetaspaceArena::allocate_new_chunk(size_t requested_word_size) { + + assert_lock_strong(lock()); + + // Should this ever happen, we need to increase the maximum possible chunk size. + guarantee(requested_word_size <= chunklevel::MAX_CHUNK_WORD_SIZE, + "Requested size too large (" SIZE_FORMAT ") - max allowed size per allocation is " SIZE_FORMAT ".", + requested_word_size, chunklevel::MAX_CHUNK_WORD_SIZE); + + const int growth_step = _chunks.count(); + const chunklevel_t max_level = chunklevel::level_fitting_word_size(requested_word_size); + const chunklevel_t preferred_level = MIN2(max_level, next_chunk_level()); + + Metachunk* c = _chunk_manager->get_chunk(preferred_level, max_level, requested_word_size); + if (c == NULL) { + return NULL; + } + + assert(c->is_in_use(), "Wrong chunk state."); + assert(c->free_below_committed_words() >= requested_word_size, "Chunk not committed"); + + return c; + +} + +void MetaspaceArena::add_allocation_to_fbl(MetaWord* p, size_t word_size) { + assert(Settings::handle_deallocations(), "Sanity"); + if (_fbl == NULL) { + _fbl = new FreeBlocks(); // Create only on demand + } + _fbl->add_block(p, word_size); +} + +MetaspaceArena::MetaspaceArena(ChunkManager* chunk_manager, + const ArenaGrowthPolicy* growth_policy, + Mutex* lock, + SizeAtomicCounter* total_used_words_counter, + const char* name) +: _lock(lock), + _chunk_manager(chunk_manager), + _growth_policy(growth_policy), + _chunks(), + _fbl(NULL), + _total_used_words_counter(total_used_words_counter), + _name(name) +{ + UL(debug, ": born."); + + // Update statistics + InternalStats::inc_num_arena_births(); +} + +MetaspaceArena::~MetaspaceArena() { + + DEBUG_ONLY(verify(true);) + + MutexLocker fcl(lock(), Mutex::_no_safepoint_check_flag); + + MemRangeCounter return_counter; + + Metachunk* c = _chunks.first(); + Metachunk* c2 = NULL; + + while(c) { + c2 = c->next(); + return_counter.add(c->used_words()); + DEBUG_ONLY(c->set_prev(NULL);) + DEBUG_ONLY(c->set_next(NULL);) + UL2(debug, "return chunk: " METACHUNK_FORMAT ".", METACHUNK_FORMAT_ARGS(c)); + _chunk_manager->return_chunk(c); + // c may be invalid after return_chunk(c) was called. Don't access anymore. + c = c2; + } + + UL2(info, "returned %d chunks, total capacity " SIZE_FORMAT " words.", + return_counter.count(), return_counter.total_size()); + + _total_used_words_counter->decrement_by(return_counter.total_size()); + + DEBUG_ONLY(chunk_manager()->verify(true);) + + delete _fbl; + + UL(debug, ": dies."); + + // Update statistics + InternalStats::inc_num_arena_deaths(); + +} + +// Attempt to enlarge the current chunk to make it large enough to hold at least +// requested_word_size additional words. +// +// On success, true is returned, false otherwise. +bool MetaspaceArena::attempt_enlarge_current_chunk(size_t requested_word_size) { + + assert_lock_strong(lock()); + + Metachunk* c = current_chunk(); + assert(c->free_words() < requested_word_size, "Sanity"); + + // Not if chunk enlargment is switched off... + if (Settings::enlarge_chunks_in_place() == false) { + return false; + } + + // ... we also disallow it for very large chunks... + if (c->word_size() > Settings::enlarge_chunks_in_place_max_word_size()) { + return false; + } + + // ... nor if we are already a root chunk ... + if (c->is_root_chunk()) { + return false; + } + + // ... nor if the combined size of chunk content and new content would bring us above the size of a root chunk ... + if ((c->used_words() + requested_word_size) > metaspace::chunklevel::MAX_CHUNK_WORD_SIZE) { + return false; + } + + const chunklevel_t new_level = + chunklevel::level_fitting_word_size(c->used_words() + requested_word_size); + assert(new_level < c->level(), "Sanity"); + + // Atm we only enlarge by one level (so, doubling the chunk in size). So, if the requested enlargement + // would require the chunk to more than double in size, we bail. But this covers about 99% of all cases, + // so this is good enough. + if (new_level < c->level() - 1) { + return false; + } + + // This only works if chunk is the leader of its buddy pair (and also if buddy + // is free and unsplit, but that we cannot check outside of metaspace lock). + if (!c->is_leader()) { + return false; + } + + // If the size added to the chunk would be larger than allowed for the next growth step + // dont enlarge. + if (next_chunk_level() > c->level()) { + return false; + } + + bool success = _chunk_manager->attempt_enlarge_chunk(c); + + assert(success == false || c->free_words() >= requested_word_size, "Sanity"); + + return success; + +} + +// Allocate memory from Metaspace. +// 1) Attempt to allocate from the free block list. +// 2) Attempt to allocate from the current chunk. +// 3) Attempt to enlarge the current chunk in place if it is too small. +// 4) Attempt to get a new chunk and allocate from that chunk. +// At any point, if we hit a commit limit, we return NULL. +MetaWord* MetaspaceArena::allocate(size_t requested_word_size) { + + MutexLocker cl(lock(), Mutex::_no_safepoint_check_flag); + + UL2(trace, "requested " SIZE_FORMAT " words.", requested_word_size); + + MetaWord* p = NULL; + + const size_t raw_word_size = get_raw_word_size_for_requested_word_size(requested_word_size); + + // 1) Attempt to allocate from the free blocks list + if (Settings::handle_deallocations() && _fbl != NULL && !_fbl->is_empty()) { + p = _fbl->get_block(raw_word_size); + if (p != NULL) { + DEBUG_ONLY(InternalStats::inc_num_allocs_from_deallocated_blocks();) + UL2(trace, "taken from fbl (now: %d, " SIZE_FORMAT ").", + _fbl->count(), _fbl->total_size()); + // Note: Space in the freeblock dictionary counts as already used (see retire_current_chunk()) - + // that means that we do not modify any counters and therefore can skip the epilog. + return p; + } + } + + bool current_chunk_too_small = false; + bool commit_failure = false; + + if (current_chunk() != NULL) { + + // 2) Attempt to satisfy the allocation from the current chunk. + + // If the current chunk is too small to hold the requested size, attempt to enlarge it. + // If that fails, retire the chunk. + if (current_chunk()->free_words() < raw_word_size) { + if (!attempt_enlarge_current_chunk(raw_word_size)) { + current_chunk_too_small = true; + } else { + DEBUG_ONLY(InternalStats::inc_num_chunks_enlarged();) + UL(debug, "enlarged chunk."); + } + } + + // Commit the chunk far enough to hold the requested word size. If that fails, we + // hit a limit (either GC threshold or MaxMetaspaceSize). In that case retire the + // chunk. + if (!current_chunk_too_small) { + if (!current_chunk()->ensure_committed_additional(raw_word_size)) { + UL2(info, "commit failure (requested size: " SIZE_FORMAT ")", raw_word_size); + commit_failure = true; + } + } + + // Allocate from the current chunk. This should work now. + if (!current_chunk_too_small && !commit_failure) { + p = current_chunk()->allocate(raw_word_size); + assert(p != NULL, "Allocation from chunk failed."); + } + + } + + if (p == NULL) { + + // If we are here, we either had no current chunk to begin with or it was deemed insufficient. + assert(current_chunk() == NULL || + current_chunk_too_small || commit_failure, "Sanity"); + + Metachunk* new_chunk = allocate_new_chunk(raw_word_size); + + if (new_chunk != NULL) { + + UL2(debug, "allocated new chunk " METACHUNK_FORMAT " for requested word size " SIZE_FORMAT ".", + METACHUNK_FORMAT_ARGS(new_chunk), requested_word_size); + + assert(new_chunk->free_below_committed_words() >= raw_word_size, "Sanity"); + + // We have a new chunk. Before making it the current chunk, retire the old one. + if (current_chunk() != NULL) { + salvage_chunk(current_chunk()); + DEBUG_ONLY(InternalStats::inc_num_chunks_retired();) + } + + _chunks.add(new_chunk); + + // Now, allocate from that chunk. That should work. + p = current_chunk()->allocate(raw_word_size); + assert(p != NULL, "Allocation from chunk failed."); + + } else { + UL2(info, "failed to allocate new chunk for requested word size " SIZE_FORMAT ".", requested_word_size); + } + + } + +#ifdef ASSERT + // When using allocation guards, establish a prefix. + if (p != NULL && Settings::use_allocation_guard()) { + p = establish_prefix(p, raw_word_size); + } +#endif + + if (p == NULL) { + InternalStats::inc_num_allocs_failed_limit(); + } else { + DEBUG_ONLY(InternalStats::inc_num_allocs();) + _total_used_words_counter->increment_by(raw_word_size); + } + + SOMETIMES(verify_locked(true);) + + if (p == NULL) { + UL(info, "allocation failed, returned NULL."); + } else { + UL2(trace, "returned " PTR_FORMAT ".", p2i(p)); + } + + return p; + +} + +// Prematurely returns a metaspace allocation to the _block_freelists +// because it is not needed anymore (requires CLD lock to be active). +void MetaspaceArena::deallocate_locked(MetaWord* p, size_t word_size) { + + if (Settings::handle_deallocations() == false) { + return; + } + + assert_lock_strong(lock()); + + // At this point a current chunk must exist since we only deallocate if we did allocate before. + assert(current_chunk() != NULL, "stray deallocation?"); + + assert(is_valid_area(p, word_size), + "Pointer range not part of this Arena and cannot be deallocated: (" PTR_FORMAT ".." PTR_FORMAT ").", + p2i(p), p2i(p + word_size)); + + UL2(trace, "deallocating " PTR_FORMAT ", word size: " SIZE_FORMAT ".", + p2i(p), word_size); + + size_t raw_word_size = get_raw_word_size_for_requested_word_size(word_size); + add_allocation_to_fbl(p, raw_word_size); + + DEBUG_ONLY(verify_locked(false);) + +} + +// Prematurely returns a metaspace allocation to the _block_freelists because it is not +// needed anymore. +void MetaspaceArena::deallocate(MetaWord* p, size_t word_size) { + MutexLocker cl(lock(), Mutex::_no_safepoint_check_flag); + deallocate_locked(p, word_size); +} + +// Update statistics. This walks all in-use chunks. +void MetaspaceArena::add_to_statistics(arena_stats_t* out) const { + + MutexLocker cl(lock(), Mutex::_no_safepoint_check_flag); + + for (const Metachunk* c = _chunks.first(); c != NULL; c = c->next()) { + in_use_chunk_stats_t& ucs = out->stats[c->level()]; + ucs.num ++; + ucs.word_size += c->word_size(); + ucs.committed_words += c->committed_words(); + ucs.used_words += c->used_words(); + // Note: for free and waste, we only count what's committed. + if (c == current_chunk()) { + ucs.free_words += c->free_below_committed_words(); + } else { + ucs.waste_words += c->free_below_committed_words(); + } + } + + if (_fbl != NULL) { + out->free_blocks_num += _fbl->count(); + out->free_blocks_word_size += _fbl->total_size(); + } + + SOMETIMES(out->verify();) + +} + +// Convenience method to get the most important usage statistics. +// For deeper analysis use add_to_statistics(). +void MetaspaceArena::usage_numbers(size_t* p_used_words, size_t* p_committed_words, size_t* p_capacity_words) const { + MutexLocker cl(lock(), Mutex::_no_safepoint_check_flag); + size_t used = 0, comm = 0, cap = 0; + for (const Metachunk* c = _chunks.first(); c != NULL; c = c->next()) { + used += c->used_words(); + comm += c->committed_words(); + cap += c->word_size(); + } + if (p_used_words != NULL) { + *p_used_words = used; + } + if (p_committed_words != NULL) { + *p_committed_words = comm; + } + if (p_capacity_words != NULL) { + *p_capacity_words = cap; + } +} + + +#ifdef ASSERT + +void MetaspaceArena::verify_locked(bool slow) const { + + assert_lock_strong(lock()); + + assert(_growth_policy != NULL && _chunk_manager != NULL, "Sanity"); + + _chunks.verify(); + + if (_fbl != NULL) { + _fbl->verify(); + } + + // In slow mode, verify guard zones of all allocations + if (slow && Settings::use_allocation_guard()) { + for (const Metachunk* c = _chunks.first(); c != NULL; c = c->next()) { + const MetaWord* p = c->base(); + while (p < c->top()) { + const prefix_t* pp = (const prefix_t*)p; + check_prefix(pp); + p += pp->word_size; + } + } + } + +} + +void MetaspaceArena::verify(bool slow) const { + + MutexLocker cl(lock(), Mutex::_no_safepoint_check_flag); + verify_locked(slow); + +} + +// Returns true if the area indicated by pointer and size have actually been allocated +// from this arena. +bool MetaspaceArena::is_valid_area(MetaWord* p, size_t word_size) const { + assert(p != NULL && word_size > 0, "Sanity"); + bool found = false; + if (!found) { + for (const Metachunk* c = _chunks.first(); c != NULL && !found; c = c->next()) { + assert(c->is_valid_committed_pointer(p) == + c->is_valid_committed_pointer(p + word_size - 1), "range intersects"); + found = c->is_valid_committed_pointer(p); + } + } + return found; +} + +#endif // ASSERT + +void MetaspaceArena::print_on(outputStream* st) const { + MutexLocker fcl(_lock, Mutex::_no_safepoint_check_flag); + print_on_locked(st); +} + +void MetaspaceArena::print_on_locked(outputStream* st) const { + assert_lock_strong(_lock); + st->print_cr("sm %s: %d chunks, total word size: " SIZE_FORMAT ", committed word size: " SIZE_FORMAT, _name, + _chunks.count(), _chunks.calc_word_size(), _chunks.calc_committed_word_size()); + _chunks.print_on(st); + st->cr(); + st->print_cr("growth-policy " PTR_FORMAT ", lock " PTR_FORMAT ", cm " PTR_FORMAT ", fbl " PTR_FORMAT, + p2i(_growth_policy), p2i(_lock), p2i(_chunk_manager), p2i(_fbl)); +} + + + +} // namespace metaspace + diff --git a/src/hotspot/share/memory/metaspace/metaspaceArena.hpp b/src/hotspot/share/memory/metaspace/metaspaceArena.hpp new file mode 100644 --- /dev/null +++ b/src/hotspot/share/memory/metaspace/metaspaceArena.hpp @@ -0,0 +1,164 @@ +/* + * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020 SAP SE. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_MEMORY_METASPACE_METASPACEARENA_HPP +#define SHARE_MEMORY_METASPACE_METASPACEARENA_HPP + +#include "memory/allocation.hpp" +#include "memory/metaspace.hpp" +#include "memory/metaspace/chunkManager.hpp" +#include "memory/metaspace/metachunk.hpp" +#include "memory/metaspace/metachunkList.hpp" +#include "memory/metaspace/metaspaceCommon.hpp" + + +class outputStream; +class Mutex; + +namespace metaspace { + + +class ArenaGrowthPolicy; +class FreeBlocks; + +struct arena_stats_t; + + +// The MetaspaceArena is a growable metaspace memory pool belonging to a CLD; +// internally it consists of a list of metaspace chunks, of which the head chunk +// is the current chunk from which we allocate via pointer bump. +// +// When the current chunk is used up, MetaspaceArena requestes a new chunk from +// the associated ChunkManager. +// +// MetaspaceArena also keeps a FreeBlocks structure to manage memory blocks which +// had been deallocated prematurely. +// + +class MetaspaceArena : public CHeapObj { + + // Reference to an outside lock to use for synchronizing access to this arena. + // This lock is normally owned by the CLD which owns the ClassLoaderMetaspace which + // owns this arena. + // Todo: This should be changed. Either the CLD should synchronize access to the + // CLMS and its arenas itself, or the arena should have an own lock. The latter + // would allow for more fine granular locking since it would allow access to + // both class- and non-class arena in the CLMS independently. + Mutex* const _lock; + + // Reference to the chunk manager to allocate chunks from. + ChunkManager* const _chunk_manager; + + // Reference to the growth policy to use. + const ArenaGrowthPolicy* const _growth_policy; + + // List of chunks. Head of the list is the current chunk. + MetachunkList _chunks; + + // Structure to take care of leftover/deallocated space in used chunks. + // Owned by the Arena. Gets allocated on demand only. + FreeBlocks* _fbl; + + Metachunk* current_chunk() { return _chunks.first(); } + const Metachunk* current_chunk() const { return _chunks.first(); } + + // Reference to an outside counter to keep track of used space. + SizeAtomicCounter* const _total_used_words_counter; + + // A name for purely debugging/logging purposes. + const char* const _name; + + Mutex* lock() const { return _lock; } + ChunkManager* chunk_manager() const { return _chunk_manager; } + + // free block list + FreeBlocks* fbl() const { return _fbl; } + void add_allocation_to_fbl(MetaWord* p, size_t word_size); + + // Given a chunk, add its remaining free committed space to the free block list. + void salvage_chunk(Metachunk* c); + + // Allocate a new chunk from the underlying chunk manager able to hold at least + // requested word size. + Metachunk* allocate_new_chunk(size_t requested_word_size); + + // Returns the level of the next chunk to be added, acc to growth policy. + chunklevel_t next_chunk_level() const; + + // Attempt to enlarge the current chunk to make it large enough to hold at least + // requested_word_size additional words. + // + // On success, true is returned, false otherwise. + bool attempt_enlarge_current_chunk(size_t requested_word_size); + + // Prematurely returns a metaspace allocation to the _block_freelists + // because it is not needed anymore (requires CLD lock to be active). + void deallocate_locked(MetaWord* p, size_t word_size); + + // Returns true if the area indicated by pointer and size have actually been allocated + // from this arena. + DEBUG_ONLY(bool is_valid_area(MetaWord* p, size_t word_size) const;) + +public: + + MetaspaceArena(ChunkManager* chunk_manager, + const ArenaGrowthPolicy* growth_policy, + Mutex* lock, + SizeAtomicCounter* total_used_words_counter, + const char* name); + + ~MetaspaceArena(); + + // Allocate memory from Metaspace. + // 1) Attempt to allocate from the dictionary of deallocated blocks. + // 2) Attempt to allocate from the current chunk. + // 3) Attempt to enlarge the current chunk in place if it is too small. + // 4) Attempt to get a new chunk and allocate from that chunk. + // At any point, if we hit a commit limit, we return NULL. + MetaWord* allocate(size_t word_size); + + // Prematurely returns a metaspace allocation to the _block_freelists because it is not + // needed anymore. + void deallocate(MetaWord* p, size_t word_size); + + // Update statistics. This walks all in-use chunks. + void add_to_statistics(arena_stats_t* out) const; + + // Convenience method to get the most important usage statistics. + // For deeper analysis use add_to_statistics(). + void usage_numbers(size_t* p_used_words, size_t* p_committed_words, size_t* p_capacity_words) const; + + DEBUG_ONLY(void verify(bool slow) const;) + DEBUG_ONLY(void verify_locked(bool slow) const;) + + void print_on(outputStream* st) const; + void print_on_locked(outputStream* st) const; + +}; + +} // namespace metaspace + +#endif // SHARE_MEMORY_METASPACE_METASPACEARENA_HPP + diff --git a/src/hotspot/share/memory/metaspace/metaspaceCommon.cpp b/src/hotspot/share/memory/metaspace/metaspaceCommon.cpp --- a/src/hotspot/share/memory/metaspace/metaspaceCommon.cpp +++ b/src/hotspot/share/memory/metaspace/metaspaceCommon.cpp @@ -1,5 +1,6 @@ /* - * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2018, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2018, 2020 SAP SE. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -24,15 +25,18 @@ #include "precompiled.hpp" +#include "memory/metaspace/allocationGuard.hpp" +#include "memory/metaspace/freeBlocks.hpp" #include "memory/metaspace/metaspaceCommon.hpp" #include "memory/metaspace/virtualSpaceNode.hpp" + +#include "utilities/align.hpp" #include "utilities/debug.hpp" #include "utilities/globalDefinitions.hpp" #include "utilities/ostream.hpp" namespace metaspace { -DEBUG_ONLY(internal_statistics_t g_internal_statistics;) // Print a size, in words, scaled. void print_scaled_words(outputStream* st, size_t word_size, size_t scale, int width) { @@ -47,6 +51,19 @@ st->print(")"); } +static const char* display_unit_for_scale(size_t scale) { + const char* s = NULL; + switch(scale) { + case 1: s = "bytes"; break; + case BytesPerWord: s = "words"; break; + case K: s = "KB"; break; + case M: s = "MB"; break; + case G: s = "GB"; break; + default: + ShouldNotReachHere(); + } + return s; +} // Print a human readable size. // byte_size: size, in bytes, to be printed. @@ -74,36 +91,45 @@ } #ifdef ASSERT - assert(scale == 1 || scale == BytesPerWord || scale == K || scale == M || scale == G, "Invalid scale"); + assert(scale == 1 || scale == BytesPerWord || + scale == K || scale == M || scale == G, "Invalid scale"); // Special case: printing wordsize should only be done with word-sized values if (scale == BytesPerWord) { assert(byte_size % BytesPerWord == 0, "not word sized"); } #endif - if (scale == 1) { - st->print("%*" PRIuPTR " bytes", width, byte_size); - } else if (scale == BytesPerWord) { - st->print("%*" PRIuPTR " words", width, byte_size / BytesPerWord); + if (width == -1) { + if (scale == 1) { + st->print(SIZE_FORMAT " bytes", byte_size); + } else if (scale == BytesPerWord) { + st->print(SIZE_FORMAT " words", byte_size / BytesPerWord); + } else { + const char* display_unit = display_unit_for_scale(scale); + float display_value = (float) byte_size / scale; + // Prevent very small but non-null values showing up as 0.00. + if (byte_size > 0 && display_value < 0.01f) { + st->print("<0.01 %s", display_unit); + } else { + st->print("%.2f %s", display_value, display_unit); + } + } } else { - const char* display_unit = ""; - switch(scale) { - case 1: display_unit = "bytes"; break; - case BytesPerWord: display_unit = "words"; break; - case K: display_unit = "KB"; break; - case M: display_unit = "MB"; break; - case G: display_unit = "GB"; break; - default: - ShouldNotReachHere(); - } - float display_value = (float) byte_size / scale; - // Since we use width to display a number with two trailing digits, increase it a bit. - width += 3; - // Prevent very small but non-null values showing up as 0.00. - if (byte_size > 0 && display_value < 0.01f) { - st->print("%*s %s", width, "<0.01", display_unit); + if (scale == 1) { + st->print("%*" PRIuPTR " bytes", width, byte_size); + } else if (scale == BytesPerWord) { + st->print("%*" PRIuPTR " words", width, byte_size / BytesPerWord); } else { - st->print("%*.2f %s", width, display_value, display_unit); + const char* display_unit = display_unit_for_scale(scale); + float display_value = (float) byte_size / scale; + // Since we use width to display a number with two trailing digits, increase it a bit. + width += 3; + // Prevent very small but non-null values showing up as 0.00. + if (byte_size > 0 && display_value < 0.01f) { + st->print("%*s %s", width, "<0.01", display_unit); + } else { + st->print("%*.2f %s", width, display_value, display_unit); + } } } } @@ -130,70 +156,6 @@ } } -// Returns size of this chunk type. -size_t get_size_for_nonhumongous_chunktype(ChunkIndex chunktype, bool is_class) { - assert(is_valid_nonhumongous_chunktype(chunktype), "invalid chunk type."); - size_t size = 0; - if (is_class) { - switch(chunktype) { - case SpecializedIndex: size = ClassSpecializedChunk; break; - case SmallIndex: size = ClassSmallChunk; break; - case MediumIndex: size = ClassMediumChunk; break; - default: - ShouldNotReachHere(); - } - } else { - switch(chunktype) { - case SpecializedIndex: size = SpecializedChunk; break; - case SmallIndex: size = SmallChunk; break; - case MediumIndex: size = MediumChunk; break; - default: - ShouldNotReachHere(); - } - } - return size; -} - -ChunkIndex get_chunk_type_by_size(size_t size, bool is_class) { - if (is_class) { - if (size == ClassSpecializedChunk) { - return SpecializedIndex; - } else if (size == ClassSmallChunk) { - return SmallIndex; - } else if (size == ClassMediumChunk) { - return MediumIndex; - } else if (size > ClassMediumChunk) { - // A valid humongous chunk size is a multiple of the smallest chunk size. - assert(is_aligned(size, ClassSpecializedChunk), "Invalid chunk size"); - return HumongousIndex; - } - } else { - if (size == SpecializedChunk) { - return SpecializedIndex; - } else if (size == SmallChunk) { - return SmallIndex; - } else if (size == MediumChunk) { - return MediumIndex; - } else if (size > MediumChunk) { - // A valid humongous chunk size is a multiple of the smallest chunk size. - assert(is_aligned(size, SpecializedChunk), "Invalid chunk size"); - return HumongousIndex; - } - } - ShouldNotReachHere(); - return (ChunkIndex)-1; -} - -ChunkIndex next_chunk_index(ChunkIndex i) { - assert(i < NumberOfInUseLists, "Out of bound"); - return (ChunkIndex) (i+1); -} - -ChunkIndex prev_chunk_index(ChunkIndex i) { - assert(i > ZeroIndex, "Out of bound"); - return (ChunkIndex) (i-1); -} - const char* loaders_plural(uintx num) { return num == 1 ? "loader" : "loaders"; } @@ -209,5 +171,34 @@ } } +// Given a net allocation word size, return the raw word size we actually allocate. +// Note: externally visible for gtests. +//static +size_t get_raw_word_size_for_requested_word_size(size_t word_size) { + + size_t byte_size = word_size * BytesPerWord; + + // Deallocated metablocks are kept in a binlist which limits their minimal + // size to at least the size of a binlist item (2 words). + byte_size = MAX2(byte_size, FreeBlocks::minimal_word_size * BytesPerWord); + + // Metaspace allocations are aligned to word size. + byte_size = align_up(byte_size, allocation_alignment_bytes); + + // If we guard allocations, we need additional space for a prefix. +#ifdef ASSERT + if (Settings::use_allocation_guard()) { + byte_size += align_up(prefix_size(), allocation_alignment_bytes); + } +#endif + + size_t raw_word_size = byte_size / BytesPerWord; + + assert(raw_word_size * BytesPerWord == byte_size, "Sanity"); + + return raw_word_size; + +} + } // namespace metaspace diff --git a/src/hotspot/share/memory/metaspace/metaspaceCommon.hpp b/src/hotspot/share/memory/metaspace/metaspaceCommon.hpp --- a/src/hotspot/share/memory/metaspace/metaspaceCommon.hpp +++ b/src/hotspot/share/memory/metaspace/metaspaceCommon.hpp @@ -1,5 +1,6 @@ /* - * Copyright (c) 2018, 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2018, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2018, 2020 SAP SE. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -25,6 +26,7 @@ #ifndef SHARE_MEMORY_METASPACE_METASPACECOMMON_HPP #define SHARE_MEMORY_METASPACE_METASPACECOMMON_HPP +#include "runtime/globals.hpp" #include "utilities/align.hpp" #include "utilities/debug.hpp" #include "utilities/globalDefinitions.hpp" @@ -33,14 +35,30 @@ namespace metaspace { -enum ChunkSizes { // in words. - ClassSpecializedChunk = 128, - SpecializedChunk = 128, - ClassSmallChunk = 256, - SmallChunk = 512, - ClassMediumChunk = 4 * K, - MediumChunk = 8 * K -}; + +// Metaspace allocation alignment: + +// 1) Metaspace allocations have to be aligned such that 64bit values are aligned +// correctly. +// +// 2) Klass* structures allocated from Metaspace have to be aligned to KlassAlignmentInBytes. +// +// At the moment LogKlassAlignmentInBytes is 3, so KlassAlignmentInBytes == 8, +// so (1) and (2) can both be fulfilled with an alignment of 8. Should we increase +// KlassAlignmentInBytes at any time this will increase the necessary alignment as well. In +// that case we may think about introducing a separate alignment just for the class space +// since that alignment would only be needed for Klass structures. + +static const size_t allocation_alignment_bytes = 8; +STATIC_ASSERT(allocation_alignment_bytes == (size_t)KlassAlignmentInBytes); + +static const size_t allocation_alignment_words = allocation_alignment_bytes / BytesPerWord; + +// Returns the raw word size allocated for a given net allocation +size_t get_raw_word_size_for_requested_word_size(size_t word_size); + + +// Utility functions // Print a size, in words, scaled. void print_scaled_words(outputStream* st, size_t word_size, size_t scale = 0, int width = -1); @@ -60,91 +78,74 @@ void print_percentage(outputStream* st, size_t total, size_t part); +#ifdef ASSERT #define assert_is_aligned(value, alignment) \ assert(is_aligned((value), (alignment)), \ SIZE_FORMAT_HEX " is not aligned to " \ - SIZE_FORMAT, (size_t)(uintptr_t)value, (alignment)) - -// Internal statistics. -#ifdef ASSERT -struct internal_statistics_t { - // Number of allocations. - uintx num_allocs; - // Number of times a ClassLoaderMetaspace was born... - uintx num_metaspace_births; - // ... and died. - uintx num_metaspace_deaths; - // Number of times VirtualSpaceListNodes were created... - uintx num_vsnodes_created; - // ... and purged. - uintx num_vsnodes_purged; - // Number of times we expanded the committed section of the space. - uintx num_committed_space_expanded; - // Number of deallocations - uintx num_deallocs; - // Number of deallocations triggered from outside ("real" deallocations). - uintx num_external_deallocs; - // Number of times an allocation was satisfied from deallocated blocks. - uintx num_allocs_from_deallocated_blocks; - // Number of times a chunk was added to the freelist - uintx num_chunks_added_to_freelist; - // Number of times a chunk was removed from the freelist - uintx num_chunks_removed_from_freelist; - // Number of chunk merges - uintx num_chunk_merges; - // Number of chunk splits - uintx num_chunk_splits; -}; -extern internal_statistics_t g_internal_statistics; + SIZE_FORMAT_HEX, (size_t)(uintptr_t)value, (size_t)(alignment)) +#else +#define assert_is_aligned(value, alignment) #endif -// ChunkIndex defines the type of chunk. -// Chunk types differ by size: specialized < small < medium, chunks -// larger than medium are humongous chunks of varying size. -enum ChunkIndex { - ZeroIndex = 0, - SpecializedIndex = ZeroIndex, - SmallIndex = SpecializedIndex + 1, - MediumIndex = SmallIndex + 1, - HumongousIndex = MediumIndex + 1, - NumberOfFreeLists = 3, - NumberOfInUseLists = 4 -}; - -// Utility functions. -size_t get_size_for_nonhumongous_chunktype(ChunkIndex chunk_type, bool is_class); -ChunkIndex get_chunk_type_by_size(size_t size, bool is_class); - -ChunkIndex next_chunk_index(ChunkIndex i); -ChunkIndex prev_chunk_index(ChunkIndex i); -// Returns a descriptive name for a chunk type. -const char* chunk_size_name(ChunkIndex index); - -// Verify chunk sizes. -inline bool is_valid_chunksize(bool is_class, size_t size) { - const size_t reasonable_maximum_humongous_chunk_size = 1 * G; - return is_aligned(size, sizeof(MetaWord)) && - size < reasonable_maximum_humongous_chunk_size && - is_class ? - (size == ClassSpecializedChunk || size == ClassSmallChunk || size >= ClassMediumChunk) : - (size == SpecializedChunk || size == SmallChunk || size >= MediumChunk); -} - -// Verify chunk type. -inline bool is_valid_chunktype(ChunkIndex index) { - return index == SpecializedIndex || index == SmallIndex || - index == MediumIndex || index == HumongousIndex; -} - -inline bool is_valid_nonhumongous_chunktype(ChunkIndex index) { - return is_valid_chunktype(index) && index != HumongousIndex; -} // Pretty printing helpers const char* classes_plural(uintx num); const char* loaders_plural(uintx num); void print_number_of_classes(outputStream* out, uintx classes, uintx classes_shared); + +// Since Metaspace verifications are expensive, we want to do them at a reduced rate, +// but not completely avoiding them. +// For that we introduce the macros SOMETIMES() and ASSERT_SOMETIMES() which will +// execute code or assert at intervals controlled via VerifyMetaspaceInterval. +#ifdef ASSERT + +#define EVERY_NTH(n) \ +{ static int counter_ = 0; \ + if (n > 0) { \ + counter_ ++; \ + if (counter_ > n) { \ + counter_ = 0; \ + +#define END_EVERY_NTH } } } + +#define SOMETIMES(code) \ + EVERY_NTH(VerifyMetaspaceInterval) \ + { code } \ + END_EVERY_NTH + +#define ASSERT_SOMETIMES(condition, ...) \ + EVERY_NTH(VerifyMetaspaceInterval) \ + assert( (condition), __VA_ARGS__); \ + END_EVERY_NTH + +#else + +#define SOMETIMES(code) +#define ASSERT_SOMETIMES(condition, ...) + +#endif // ASSERT + +///////// Logging ////////////// + +// What we log at which levels: + +// "info" : metaspace failed allocation, commit failure, reserve failure, metaspace oom, metaspace gc threshold changed, Arena created, destroyed, metaspace purged + +// "debug" : "info" + vslist extended, memory committed/uncommitted, chunk created/split/merged/enlarged, chunk returned + +// "trace" : "debug" + every single allocation and deallocation, internals + +#define HAVE_UL + +#ifdef HAVE_UL +#define UL(level, message) log_##level(metaspace)(LOGFMT ": " message, LOGFMT_ARGS); +#define UL2(level, message, ...) log_##level(metaspace)(LOGFMT ": " message, LOGFMT_ARGS, __VA_ARGS__); +#else +#define UL(level, ...) +#define UL2(level, ...) +#endif + } // namespace metaspace #endif // SHARE_MEMORY_METASPACE_METASPACECOMMON_HPP diff --git a/src/hotspot/share/memory/metaspace/metaspaceContext.cpp b/src/hotspot/share/memory/metaspace/metaspaceContext.cpp new file mode 100644 --- /dev/null +++ b/src/hotspot/share/memory/metaspace/metaspaceContext.cpp @@ -0,0 +1,89 @@ +/* + * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020 SAP SE. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" + +#include "memory/metaspace/chunkManager.hpp" +#include "memory/metaspace/metaspaceContext.hpp" +#include "memory/metaspace/virtualSpaceList.hpp" +#include "utilities/debug.hpp" +#include "utilities/globalDefinitions.hpp" +#include "utilities/ostream.hpp" + + +namespace metaspace { + +#define LOGFMT "SpcMgr @" PTR_FORMAT " (%s)" +#define LOGFMT_ARGS p2i(this), this->_name + + +MetaspaceContext* MetaspaceContext::_class_space_context = NULL; +MetaspaceContext* MetaspaceContext::_nonclass_space_context = NULL; + +// Destroys the context: deletes chunkmanager and virtualspacelist. +// If this is a non-expandable context over an existing space, that space remains +// untouched, otherwise all memory is unmapped. +MetaspaceContext::~MetaspaceContext() { + delete _cm; + delete _vslist; +} + +// Create a new, empty, expandable metaspace context. +MetaspaceContext* MetaspaceContext::create_expandable_context(const char* name, CommitLimiter* commit_limiter) { + VirtualSpaceList* vsl = new VirtualSpaceList(name, commit_limiter); + ChunkManager* cm = new ChunkManager(name, vsl); + return new MetaspaceContext(name, vsl, cm); +} + +// Create a new, empty, non-expandable metaspace context atop of an externally provided space. +MetaspaceContext* MetaspaceContext::create_nonexpandable_context(const char* name, ReservedSpace rs, CommitLimiter* commit_limiter) { + VirtualSpaceList* vsl = new VirtualSpaceList(name, rs, commit_limiter); + ChunkManager* cm = new ChunkManager(name, vsl); + return new MetaspaceContext(name, vsl, cm); +} + + +void MetaspaceContext::initialize_class_space_context(ReservedSpace rs) { + _class_space_context = create_nonexpandable_context("class-space", rs, CommitLimiter::globalLimiter()); +} + +void MetaspaceContext::initialize_nonclass_space_context() { + _nonclass_space_context = create_expandable_context("non-class-space", CommitLimiter::globalLimiter()); +} + +void MetaspaceContext::print_on(outputStream* st) const { + _vslist->print_on(st); + _cm->print_on(st); +} + +#ifdef ASSERT +void MetaspaceContext::verify(bool slow) const { + _vslist->verify(slow); + _cm->verify(slow); +} +#endif // ASSERT + +} // namespace metaspace + diff --git a/src/hotspot/share/memory/metaspace/metaspaceContext.hpp b/src/hotspot/share/memory/metaspace/metaspaceContext.hpp new file mode 100644 --- /dev/null +++ b/src/hotspot/share/memory/metaspace/metaspaceContext.hpp @@ -0,0 +1,108 @@ +/* + * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020 SAP SE. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_MEMORY_METASPACE_METASPACECONTEXT_HPP +#define SHARE_MEMORY_METASPACE_METASPACECONTEXT_HPP + +#include "memory/allocation.hpp" +#include "memory/virtualspace.hpp" +#include "utilities/debug.hpp" + +class outputStream; + +namespace metaspace { + +class ChunkManager; +class VirtualSpaceList; +class CommitLimiter; + +// MetaspaceContext is a convenience bracket around: +// +// - a VirtualSpaceList managing a memory area used for Metaspace +// - a ChunkManager sitting atop of that which manages chunk freelists +// +// In a normal VM only one or two of these contexts ever exist: one for the metaspace, and +// optionally another one for the compressed class space. +// +// For tests more contexts may be created, and this would also be a way to use Metaspace +// for things other than class metadata. We would have to work on the naming then. +// +// - (Future TODO): Context should own a lock to guard it. Currently this stuff is guarded +// by one global lock, the slightly misnamed Metaspace_expandlock, but that one +// should be split into one per context. +// - (Future TODO): Context can/should have its own allocation alignment. That way we +// can have different alignment between class space and non-class metaspace. That could +// help optimize compressed class pointer encoding, see discussion for JDK-8244943). + +class MetaspaceContext : public CHeapObj { + + const char* const _name; + VirtualSpaceList* const _vslist; + ChunkManager* const _cm; + + MetaspaceContext(const char* name, VirtualSpaceList* vslist, ChunkManager* cm) + : _name(name), _vslist(vslist), _cm(cm) {} + + static MetaspaceContext* _nonclass_space_context; + static MetaspaceContext* _class_space_context; + +public: + + // Destroys the context: deletes chunkmanager and virtualspacelist. + // If this is a non-expandable context over an existing space, that space remains + // untouched, otherwise all memory is unmapped. + ~MetaspaceContext(); + + VirtualSpaceList* vslist() { return _vslist; } + ChunkManager* cm() { return _cm; } + + // Create a new, empty, expandable metaspace context. + static MetaspaceContext* create_expandable_context(const char* name, CommitLimiter* commit_limiter); + + // Create a new, empty, non-expandable metaspace context atop of an externally provided space. + static MetaspaceContext* create_nonexpandable_context(const char* name, ReservedSpace rs, CommitLimiter* commit_limiter); + + void print_on(outputStream* st) const; + + DEBUG_ONLY(void verify(bool slow) const;) + + static void initialize_class_space_context(ReservedSpace rs); + static void initialize_nonclass_space_context(); + + // Returns pointer to the global metaspace context. + // If compressed class space is active, this contains the non-class-space allocations. + // If compressed class space is inactive, this contains all metaspace allocations. + static MetaspaceContext* context_nonclass() { return _nonclass_space_context; } + + // Returns pointer to the global class space context, if compressed class space is active, + // NULL otherwise. + static MetaspaceContext* context_class() { return _class_space_context; } + +}; + +} // end namespace + +#endif // SHARE_MEMORY_METASPACE_METASPACECONTEXT_HPP + diff --git a/src/hotspot/share/memory/metaspace/metaspaceEnums.cpp b/src/hotspot/share/memory/metaspace/metaspaceEnums.cpp new file mode 100644 --- /dev/null +++ b/src/hotspot/share/memory/metaspace/metaspaceEnums.cpp @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020 SAP SE. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "memory/metaspace/metaspaceEnums.hpp" +#include "utilities/debug.hpp" + +namespace metaspace { + +const char* describe_spacetype(Metaspace::MetaspaceType st) { + const char* s = NULL; + switch (st) { + case Metaspace::StandardMetaspaceType: s = "Standard"; break; + case Metaspace::BootMetaspaceType: s = "Boot"; break; + case Metaspace::ClassMirrorHolderMetaspaceType: s = "ClassMirrorHolder"; break; + case Metaspace::ReflectionMetaspaceType: s = "Reflection"; break; + default: ShouldNotReachHere(); + } + return s; +} + +const char* describe_mdtype(Metaspace::MetadataType md) { + const char* s = NULL; + switch (md) { + case Metaspace::NonClassType: s = "nonclass"; break; + case Metaspace::ClassType: s = "class"; break; + default: ShouldNotReachHere(); + } + return s; +} + +} // namespace metaspace + diff --git a/src/hotspot/share/memory/metaspace/metaspaceEnums.hpp b/src/hotspot/share/memory/metaspace/metaspaceEnums.hpp new file mode 100644 --- /dev/null +++ b/src/hotspot/share/memory/metaspace/metaspaceEnums.hpp @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020 SAP SE. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_MEMORY_METASPACEENUMS_HPP +#define SHARE_MEMORY_METASPACEENUMS_HPP + +#include "memory/metaspace.hpp" +#include "utilities/debug.hpp" + + +namespace metaspace { + +// A bunch of convenience functions around MetadataType and MetaspaceType + +/////////////////////// + +inline bool is_class(Metaspace::MetadataType md) { return md == Metaspace::ClassType; } + +const char* describe_mdtype(Metaspace::MetadataType md); + +#ifdef ASSERT +inline bool is_valid_mdtype(Metaspace::MetadataType md) { + return (int)md >= 0 && (int)md < Metaspace::MetadataTypeCount; +} +inline void check_valid_mdtype(Metaspace::MetadataType md) { + assert(is_valid_mdtype(md), "Wrong value for MetadataType: %d", (int) md); +} +#endif // ASSERT + +/////////////////////// + +const char* describe_spacetype(Metaspace::MetaspaceType st); + +#ifdef ASSERT +inline bool is_valid_spacetype(Metaspace::MetaspaceType st) { + return (int)st >= 0 && (int)st < Metaspace::MetaspaceTypeCount; +} +inline void check_valid_spacetype(Metaspace::MetaspaceType st) { + assert(is_valid_spacetype(st), "Wrong value for MetaspaceType: %d", (int) st); +} +#endif + +} // namespace metaspace + +#endif // SHARE_MEMORY_METASPACEENUMS_HPP diff --git a/src/hotspot/share/memory/metaspace/metaspaceReport.cpp b/src/hotspot/share/memory/metaspace/metaspaceReport.cpp new file mode 100644 --- /dev/null +++ b/src/hotspot/share/memory/metaspace/metaspaceReport.cpp @@ -0,0 +1,379 @@ +/* + * Copyright (c) 2018, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2018, 2020 SAP SE. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "classfile/classLoaderData.hpp" +#include "classfile/classLoaderDataGraph.hpp" +#include "memory/metaspace/chunkHeaderPool.hpp" +#include "memory/metaspace/chunkManager.hpp" +#include "memory/metaspace/internStat.hpp" +#include "memory/metaspace/metaspaceCommon.hpp" +#include "memory/metaspace/metaspaceEnums.hpp" +#include "memory/metaspace/metaspaceReport.hpp" +#include "memory/metaspace/metaspaceStatistics.hpp" +#include "memory/metaspace/printCLDMetaspaceInfoClosure.hpp" +#include "memory/metaspace/runningCounters.hpp" +#include "memory/metaspace/virtualSpaceList.hpp" +#include "memory/metaspace.hpp" +#include "runtime/os.hpp" + +namespace metaspace { + +static void print_vs(outputStream* out, size_t scale) { + + const size_t reserved_nc = RunningCounters::reserved_words_nonclass(); + const size_t committed_nc = RunningCounters::committed_words_nonclass(); + const int num_nodes_nc = VirtualSpaceList::vslist_nonclass()->num_nodes(); + + if (Metaspace::using_class_space()) { + + const size_t reserved_c = RunningCounters::reserved_words_class(); + const size_t committed_c = RunningCounters::committed_words_class(); + const int num_nodes_c = VirtualSpaceList::vslist_class()->num_nodes(); + + out->print(" Non-class space: "); + print_scaled_words(out, reserved_nc, scale, 7); + out->print(" reserved, "); + print_scaled_words_and_percentage(out, committed_nc, reserved_nc, scale, 7); + out->print(" committed, "); + out->print(" %d nodes.", num_nodes_nc); + out->cr(); + out->print(" Class space: "); + print_scaled_words(out, reserved_c, scale, 7); + out->print(" reserved, "); + print_scaled_words_and_percentage(out, committed_c, reserved_c, scale, 7); + out->print(" committed, "); + out->print(" %d nodes.", num_nodes_c); + out->cr(); + out->print(" Both: "); + print_scaled_words(out, reserved_c + reserved_nc, scale, 7); + out->print(" reserved, "); + print_scaled_words_and_percentage(out, committed_c + committed_nc, reserved_c + reserved_nc, scale, 7); + out->print(" committed. "); + out->cr(); + + } else { + print_scaled_words(out, reserved_nc, scale, 7); + out->print(" reserved, "); + print_scaled_words_and_percentage(out, committed_nc, reserved_nc, scale, 7); + out->print(" committed, "); + out->print(" %d nodes.", num_nodes_nc); + out->cr(); + } +} + +static void print_settings(outputStream* out, size_t scale) { + out->print("MaxMetaspaceSize: "); + if (MaxMetaspaceSize >= (max_uintx) - (2 * os::vm_page_size())) { + // aka "very big". Default is max_uintx, but due to rounding in arg parsing the real + // value is smaller. + out->print("unlimited"); + } else { + print_human_readable_size(out, MaxMetaspaceSize, scale); + } + out->cr(); + if (Metaspace::using_class_space()) { + out->print("CompressedClassSpaceSize: "); + print_human_readable_size(out, CompressedClassSpaceSize, scale); + } + out->cr(); + Settings::print_on(out); +} + +// This will print out a basic metaspace usage report but +// unlike print_report() is guaranteed not to lock or to walk the CLDG. +void MetaspaceReporter::print_basic_report(outputStream* out, size_t scale) { + + if (!Metaspace::initialized()) { + out->print_cr("Metaspace not yet initialized."); + return; + } + + out->cr(); + out->print_cr("Usage:"); + + if (Metaspace::using_class_space()) { + out->print(" Non-class: "); + } + + // Note: since we want to purely rely on counters, without any locking or walking the CLDG, + // for Usage stats (statistics over in-use chunks) all we can print is the + // used words. We cannot print committed areas, or free/waste areas, of in-use chunks require + // walking. + const size_t used_nc = MetaspaceUtils::used_words(Metaspace::NonClassType); + + print_scaled_words(out, used_nc, scale, 5); + out->print(" used."); + out->cr(); + + if (Metaspace::using_class_space()) { + const size_t used_c = MetaspaceUtils::used_words(Metaspace::ClassType); + out->print(" Class: "); + print_scaled_words(out, used_c, scale, 5); + out->print(" used."); + out->cr(); + + out->print(" Both: "); + const size_t used = used_nc + used_c; + print_scaled_words(out, used, scale, 5); + out->print(" used."); + out->cr(); + } + + out->cr(); + out->print_cr("Virtual space:"); + + print_vs(out, scale); + + out->cr(); + out->print_cr("Chunk freelists:"); + + if (Metaspace::using_class_space()) { + out->print(" Non-Class: "); + } + print_scaled_words(out, ChunkManager::chunkmanager_nonclass()->total_word_size(), scale); + out->cr(); + if (Metaspace::using_class_space()) { + out->print(" Class: "); + print_scaled_words(out, ChunkManager::chunkmanager_class()->total_word_size(), scale); + out->cr(); + out->print(" Both: "); + print_scaled_words(out, ChunkManager::chunkmanager_nonclass()->total_word_size() + + ChunkManager::chunkmanager_class()->total_word_size(), scale); + out->cr(); + } + + out->cr(); + + // Print basic settings + print_settings(out, scale); + + out->cr(); + + out->cr(); + out->print_cr("Internal statistics:"); + out->cr(); + InternalStats::print_on(out); + out->cr(); + +} + +void MetaspaceReporter::print_report(outputStream* out, size_t scale, int flags) { + + if (!Metaspace::initialized()) { + out->print_cr("Metaspace not yet initialized."); + return; + } + + const bool print_loaders = (flags & rf_show_loaders) > 0; + const bool print_classes = (flags & rf_show_classes) > 0; + const bool print_by_chunktype = (flags & rf_break_down_by_chunktype) > 0; + const bool print_by_spacetype = (flags & rf_break_down_by_spacetype) > 0; + + // Some report options require walking the class loader data graph. + metaspace::PrintCLDMetaspaceInfoClosure cl(out, scale, print_loaders, print_classes, print_by_chunktype); + if (print_loaders) { + out->cr(); + out->print_cr("Usage per loader:"); + out->cr(); + } + + ClassLoaderDataGraph::loaded_cld_do(&cl); // collect data and optionally print + + // Print totals, broken up by space type. + if (print_by_spacetype) { + out->cr(); + out->print_cr("Usage per space type:"); + out->cr(); + for (int space_type = (int)Metaspace::ZeroMetaspaceType; + space_type < (int)Metaspace::MetaspaceTypeCount; space_type ++) + { + uintx num_loaders = cl._num_loaders_by_spacetype[space_type]; + uintx num_classes = cl._num_classes_by_spacetype[space_type]; + out->print("%s - " UINTX_FORMAT " %s", + describe_spacetype((Metaspace::MetaspaceType)space_type), + num_loaders, loaders_plural(num_loaders)); + if (num_classes > 0) { + out->print(", "); + + print_number_of_classes(out, num_classes, cl._num_classes_shared_by_spacetype[space_type]); + out->print(":"); + cl._stats_by_spacetype[space_type].print_on(out, scale, print_by_chunktype); + } else { + out->print("."); + out->cr(); + } + out->cr(); + } + } + + // Print totals for in-use data: + out->cr(); + { + uintx num_loaders = cl._num_loaders; + out->print("Total Usage - " UINTX_FORMAT " %s, ", + num_loaders, loaders_plural(num_loaders)); + print_number_of_classes(out, cl._num_classes, cl._num_classes_shared); + out->print(":"); + cl._stats_total.print_on(out, scale, print_by_chunktype); + out->cr(); + } + + ///////////////////////////////////////////////// + // -- Print Virtual space. + out->cr(); + out->print_cr("Virtual space:"); + + print_vs(out, scale); + + // -- Print VirtualSpaceList details. + if ((flags & rf_show_vslist) > 0) { + out->cr(); + out->print_cr("Virtual space list%s:", Metaspace::using_class_space() ? "s" : ""); + + if (Metaspace::using_class_space()) { + out->print_cr(" Non-Class:"); + } + VirtualSpaceList::vslist_nonclass()->print_on(out); + out->cr(); + if (Metaspace::using_class_space()) { + out->print_cr(" Class:"); + VirtualSpaceList::vslist_class()->print_on(out); + out->cr(); + } + } + out->cr(); + + //////////// Freelists (ChunkManager) section /////////////////////////// + + out->cr(); + out->print_cr("Chunk freelist%s:", Metaspace::using_class_space() ? "s" : ""); + + cm_stats_t non_class_cm_stat; + cm_stats_t class_cm_stat; + cm_stats_t total_cm_stat; + + ChunkManager::chunkmanager_nonclass()->add_to_statistics(&non_class_cm_stat); + if (Metaspace::using_class_space()) { + ChunkManager::chunkmanager_nonclass()->add_to_statistics(&non_class_cm_stat); + ChunkManager::chunkmanager_class()->add_to_statistics(&class_cm_stat); + total_cm_stat.add(non_class_cm_stat); + total_cm_stat.add(class_cm_stat); + + out->print_cr(" Non-Class:"); + non_class_cm_stat.print_on(out, scale); + out->cr(); + out->print_cr(" Class:"); + class_cm_stat.print_on(out, scale); + out->cr(); + out->print_cr(" Both:"); + total_cm_stat.print_on(out, scale); + out->cr(); + } else { + ChunkManager::chunkmanager_nonclass()->add_to_statistics(&non_class_cm_stat); + non_class_cm_stat.print_on(out, scale); + out->cr(); + } + + //////////// Waste section /////////////////////////// + // As a convenience, print a summary of common waste. + out->cr(); + out->print("Waste (unused committed space):"); + // For all wastages, print percentages from total. As total use the total size of memory committed for metaspace. + const size_t committed_words = RunningCounters::committed_words(); + + out->print("(percentages refer to total committed size "); + print_scaled_words(out, committed_words, scale); + out->print_cr("):"); + + // Print waste for in-use chunks. + in_use_chunk_stats_t ucs_nonclass = cl._stats_total.arena_stats_nonclass.totals(); + in_use_chunk_stats_t ucs_class = cl._stats_total.arena_stats_class.totals(); + const size_t waste_in_chunks_in_use = ucs_nonclass.waste_words + ucs_class.waste_words; + const size_t free_in_chunks_in_use = ucs_nonclass.free_words + ucs_class.free_words; + + out->print(" Waste in chunks in use: "); + print_scaled_words_and_percentage(out, waste_in_chunks_in_use, committed_words, scale, 6); + out->cr(); + out->print(" Free in chunks in use: "); + print_scaled_words_and_percentage(out, free_in_chunks_in_use, committed_words, scale, 6); + out->cr(); + + // Print waste in free chunks. + const size_t committed_in_free_chunks = total_cm_stat.total_committed_word_size(); + out->print(" In free chunks: "); + print_scaled_words_and_percentage(out, committed_in_free_chunks, committed_words, scale, 6); + out->cr(); + + // Print waste in deallocated blocks. + const uintx free_blocks_num = + cl._stats_total.arena_stats_nonclass.free_blocks_num + + cl._stats_total.arena_stats_class.free_blocks_num; + const size_t free_blocks_cap_words = + cl._stats_total.arena_stats_nonclass.free_blocks_word_size + + cl._stats_total.arena_stats_class.free_blocks_word_size; + out->print("Deallocated from chunks in use: "); + print_scaled_words_and_percentage(out, free_blocks_cap_words, committed_words, scale, 6); + out->print(" (" UINTX_FORMAT " blocks)", free_blocks_num); + out->cr(); + + // Print total waste. + const size_t total_waste = + waste_in_chunks_in_use + + free_in_chunks_in_use + + committed_in_free_chunks + + free_blocks_cap_words; + out->print(" -total-: "); + print_scaled_words_and_percentage(out, total_waste, committed_words, scale, 6); + out->cr(); + + // Also print chunk header pool size. + out->cr(); + out->print("chunk header pool: %u items, ", ChunkHeaderPool::pool()->used()); + print_scaled_words(out, ChunkHeaderPool::pool()->memory_footprint_words(), scale); + out->print("."); + out->cr(); + + // Print internal statistics + out->cr(); + out->print_cr("Internal statistics:"); + out->cr(); + InternalStats::print_on(out); + out->cr(); + + // Print some interesting settings + out->cr(); + out->print_cr("Settings:"); + print_settings(out, scale); + + out->cr(); + out->cr(); + + DEBUG_ONLY(MetaspaceUtils::verify(true);) + +} // MetaspaceUtils::print_report() + +} // namespace metaspace + diff --git a/src/hotspot/share/memory/metaspace/metaspaceReport.hpp b/src/hotspot/share/memory/metaspace/metaspaceReport.hpp new file mode 100644 --- /dev/null +++ b/src/hotspot/share/memory/metaspace/metaspaceReport.hpp @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2018, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2018, 2020 SAP SE. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_MEMORY_METASPACE_METASPACEREPORT_HPP +#define SHARE_MEMORY_METASPACE_METASPACEREPORT_HPP + +#include "memory/allocation.hpp" + +namespace metaspace { + +class MetaspaceReporter : public AllStatic { +public: + + // Flags for print_report(). + enum ReportFlag { + // Show usage by class loader. + rf_show_loaders = (1 << 0), + // Breaks report down by chunk type (small, medium, ...). + rf_break_down_by_chunktype = (1 << 1), + // Breaks report down by space type (anonymous, reflection, ...). + rf_break_down_by_spacetype = (1 << 2), + // Print details about the underlying virtual spaces. + rf_show_vslist = (1 << 3), + // If show_loaders: show loaded classes for each loader. + rf_show_classes = (1 << 4) + }; + + // This will print out a basic metaspace usage report but + // unlike print_report() is guaranteed not to lock or to walk the CLDG. + static void print_basic_report(outputStream* st, size_t scale); + + // Prints a report about the current metaspace state. + // Optional parts can be enabled via flags. + // Function will walk the CLDG and will lock the expand lock; if that is not + // convenient, use print_basic_report() instead. + static void print_report(outputStream* out, size_t scale = 0, int flags = 0); + +}; + +} // namespace metaspace + +#endif // SHARE_MEMORY_METASPACE_METASPACEREPORT_HPP diff --git a/src/hotspot/share/memory/metaspace/metaspaceStatistics.cpp b/src/hotspot/share/memory/metaspace/metaspaceStatistics.cpp --- a/src/hotspot/share/memory/metaspace/metaspaceStatistics.cpp +++ b/src/hotspot/share/memory/metaspace/metaspaceStatistics.cpp @@ -1,6 +1,6 @@ /* - * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. - * Copyright (c) 2018 SAP SE. All rights reserved. + * Copyright (c) 2018, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2018, 2020 SAP SE. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -22,179 +22,157 @@ * questions. * */ + #include "precompiled.hpp" -#include "memory/metaspace/metachunk.hpp" + +#include "memory/metaspace/chunkLevel.hpp" #include "memory/metaspace/metaspaceCommon.hpp" #include "memory/metaspace/metaspaceStatistics.hpp" + #include "utilities/debug.hpp" #include "utilities/globalDefinitions.hpp" #include "utilities/ostream.hpp" namespace metaspace { -// FreeChunksStatistics methods -FreeChunksStatistics::FreeChunksStatistics() -: _num(0), _cap(0) -{} - -void FreeChunksStatistics::reset() { - _num = 0; _cap = 0; -} - -void FreeChunksStatistics::add(uintx n, size_t s) { - _num += n; _cap += s; -} - -void FreeChunksStatistics::add(const FreeChunksStatistics& other) { - _num += other._num; - _cap += other._cap; -} - -void FreeChunksStatistics::print_on(outputStream* st, size_t scale) const { - st->print(UINTX_FORMAT, _num); - st->print(" chunks, total capacity "); - print_scaled_words(st, _cap, scale); -} - -// ChunkManagerStatistics methods - -void ChunkManagerStatistics::reset() { - for (ChunkIndex i = ZeroIndex; i < NumberOfInUseLists; i = next_chunk_index(i)) { - _chunk_stats[i].reset(); +// Returns total word size of all chunks in this manager. +void cm_stats_t::add(const cm_stats_t& other) { + for (chunklevel_t l = chunklevel::LOWEST_CHUNK_LEVEL; l <= chunklevel::HIGHEST_CHUNK_LEVEL; l ++) { + num_chunks[l] += other.num_chunks[l]; + committed_word_size[l] += other.committed_word_size[l]; } } -size_t ChunkManagerStatistics::total_capacity() const { - return _chunk_stats[SpecializedIndex].cap() + - _chunk_stats[SmallIndex].cap() + - _chunk_stats[MediumIndex].cap() + - _chunk_stats[HumongousIndex].cap(); +// Returns total word size of all chunks in this manager. +size_t cm_stats_t::total_word_size() const { + size_t s = 0; + for (chunklevel_t l = chunklevel::LOWEST_CHUNK_LEVEL; l <= chunklevel::HIGHEST_CHUNK_LEVEL; l ++) { + s += num_chunks[l] * chunklevel::word_size_for_level(l); + } + return s; } -void ChunkManagerStatistics::print_on(outputStream* st, size_t scale) const { - FreeChunksStatistics totals; - for (ChunkIndex i = ZeroIndex; i < NumberOfInUseLists; i = next_chunk_index(i)) { +// Returns total committed word size of all chunks in this manager. +size_t cm_stats_t::total_committed_word_size() const { + size_t s = 0; + for (chunklevel_t l = chunklevel::LOWEST_CHUNK_LEVEL; l <= chunklevel::HIGHEST_CHUNK_LEVEL; l ++) { + s += committed_word_size[l]; + } + return s; +} + + +void cm_stats_t::print_on(outputStream* st, size_t scale) const { + // Note: used as part of MetaspaceReport so formatting matters. + size_t total_size = 0; + size_t total_committed_size = 0; + for (chunklevel_t l = chunklevel::LOWEST_CHUNK_LEVEL; l <= chunklevel::HIGHEST_CHUNK_LEVEL; l ++) { st->cr(); - st->print("%12s chunks: ", chunk_size_name(i)); - if (_chunk_stats[i].num() > 0) { - st->print(UINTX_FORMAT_W(4) ", capacity ", _chunk_stats[i].num()); - print_scaled_words(st, _chunk_stats[i].cap(), scale); + chunklevel::print_chunk_size(st, l); + st->print(": "); + if (num_chunks[l] > 0) { + const size_t word_size = num_chunks[l] * chunklevel::word_size_for_level(l); + + st->print("%4d, capacity=", num_chunks[l]); + print_scaled_words(st, word_size, scale); + + st->print(", committed="); + print_scaled_words_and_percentage(st, committed_word_size[l], word_size, scale); + + total_size += word_size; + total_committed_size += committed_word_size[l]; } else { st->print("(none)"); } - totals.add(_chunk_stats[i]); } st->cr(); - st->print("%19s: " UINTX_FORMAT_W(4) ", capacity=", "Total", totals.num()); - print_scaled_words(st, totals.cap(), scale); + st->print("Total word size: "); + print_scaled_words(st, total_size, scale); + st->print(", committed: "); + print_scaled_words_and_percentage(st, total_committed_size, total_size, scale); st->cr(); } -// UsedChunksStatistics methods +#ifdef ASSERT +void cm_stats_t::verify() const { + assert(total_committed_word_size() <= total_word_size(), + "Sanity"); +} +#endif -UsedChunksStatistics::UsedChunksStatistics() -: _num(0), _cap(0), _used(0), _free(0), _waste(0), _overhead(0) -{} -void UsedChunksStatistics::reset() { - _num = 0; - _cap = _overhead = _used = _free = _waste = 0; -} - -void UsedChunksStatistics::add(const UsedChunksStatistics& other) { - _num += other._num; - _cap += other._cap; - _used += other._used; - _free += other._free; - _waste += other._waste; - _overhead += other._overhead; - DEBUG_ONLY(check_sanity()); -} - -void UsedChunksStatistics::print_on(outputStream* st, size_t scale) const { +void in_use_chunk_stats_t::print_on(outputStream* st, size_t scale) const { int col = st->position(); - st->print(UINTX_FORMAT_W(4) " chunk%s, ", _num, _num != 1 ? "s" : ""); - if (_num > 0) { + st->print("%4d chunk%s, ", num, num != 1 ? "s" : ""); + if (num > 0) { col += 14; st->fill_to(col); - print_scaled_words(st, _cap, scale, 5); - st->print(" capacity, "); + print_scaled_words(st, word_size, scale, 5); + st->print(" capacity,"); + + col += 20; st->fill_to(col); + print_scaled_words_and_percentage(st, committed_words, word_size, scale, 5); + st->print(" committed, "); col += 18; st->fill_to(col); - print_scaled_words_and_percentage(st, _used, _cap, scale, 5); + print_scaled_words_and_percentage(st, used_words, word_size, scale, 5); st->print(" used, "); col += 20; st->fill_to(col); - print_scaled_words_and_percentage(st, _free, _cap, scale, 5); + print_scaled_words_and_percentage(st, free_words, word_size, scale, 5); st->print(" free, "); col += 20; st->fill_to(col); - print_scaled_words_and_percentage(st, _waste, _cap, scale, 5); - st->print(" waste, "); + print_scaled_words_and_percentage(st, waste_words, word_size, scale, 5); + st->print(" waste "); - col += 20; st->fill_to(col); - print_scaled_words_and_percentage(st, _overhead, _cap, scale, 5); - st->print(" overhead"); } - DEBUG_ONLY(check_sanity()); } #ifdef ASSERT -void UsedChunksStatistics::check_sanity() const { - assert(_overhead == (Metachunk::overhead() * _num), "Sanity: Overhead."); - assert(_cap == _used + _free + _waste + _overhead, "Sanity: Capacity."); +void in_use_chunk_stats_t::verify() const { + assert(word_size >= committed_words && + committed_words == used_words + free_words + waste_words, + "Sanity: cap " SIZE_FORMAT ", committed " SIZE_FORMAT ", used " SIZE_FORMAT ", free " SIZE_FORMAT ", waste " SIZE_FORMAT ".", + word_size, committed_words, used_words, free_words, waste_words); } #endif -// SpaceManagerStatistics methods - -SpaceManagerStatistics::SpaceManagerStatistics() { reset(); } - -void SpaceManagerStatistics::reset() { - for (int i = 0; i < NumberOfInUseLists; i ++) { - _chunk_stats[i].reset(); - _free_blocks_num = 0; _free_blocks_cap_words = 0; +void arena_stats_t::add(const arena_stats_t& other) { + for (chunklevel_t l = chunklevel::LOWEST_CHUNK_LEVEL; l <= chunklevel::HIGHEST_CHUNK_LEVEL; l ++) { + stats[l].add(other.stats[l]); } + free_blocks_num += other.free_blocks_num; + free_blocks_word_size += other.free_blocks_word_size; } -void SpaceManagerStatistics::add_free_blocks_info(uintx num, size_t cap) { - _free_blocks_num += num; - _free_blocks_cap_words += cap; + +// Returns total chunk statistics over all chunk types. +in_use_chunk_stats_t arena_stats_t::totals() const { + in_use_chunk_stats_t out; + for (chunklevel_t l = chunklevel::LOWEST_CHUNK_LEVEL; l <= chunklevel::HIGHEST_CHUNK_LEVEL; l ++) { + out.add(stats[l]); + } + return out; } -void SpaceManagerStatistics::add(const SpaceManagerStatistics& other) { - for (ChunkIndex i = ZeroIndex; i < NumberOfInUseLists; i = next_chunk_index(i)) { - _chunk_stats[i].add(other._chunk_stats[i]); - } - _free_blocks_num += other._free_blocks_num; - _free_blocks_cap_words += other._free_blocks_cap_words; -} - -// Returns total chunk statistics over all chunk types. -UsedChunksStatistics SpaceManagerStatistics::totals() const { - UsedChunksStatistics stat; - for (ChunkIndex i = ZeroIndex; i < NumberOfInUseLists; i = next_chunk_index(i)) { - stat.add(_chunk_stats[i]); - } - return stat; -} - -void SpaceManagerStatistics::print_on(outputStream* st, size_t scale, bool detailed) const { +void arena_stats_t::print_on(outputStream* st, size_t scale, bool detailed) const { streamIndentor sti(st); if (detailed) { st->cr_indent(); - st->print("Usage by chunk type:"); + st->print("Usage by chunk level:"); { streamIndentor sti2(st); - for (ChunkIndex i = ZeroIndex; i < NumberOfInUseLists; i = next_chunk_index(i)) { + for (chunklevel_t l = chunklevel::LOWEST_CHUNK_LEVEL; l <= chunklevel::HIGHEST_CHUNK_LEVEL; l ++) { st->cr_indent(); - st->print("%15s: ", chunk_size_name(i)); - if (_chunk_stats[i].num() == 0) { + chunklevel::print_chunk_size(st, l); + st->print(" chunks: "); + if (stats[l].num == 0) { st->print(" (none)"); } else { - _chunk_stats[i].print_on(st, scale); + stats[l].print_on(st, scale); } } @@ -202,61 +180,56 @@ st->print("%15s: ", "-total-"); totals().print_on(st, scale); } - if (_free_blocks_num > 0) { + if (free_blocks_num > 0) { st->cr_indent(); - st->print("deallocated: " UINTX_FORMAT " blocks with ", _free_blocks_num); - print_scaled_words(st, _free_blocks_cap_words, scale); + st->print("deallocated: " UINTX_FORMAT " blocks with ", free_blocks_num); + print_scaled_words(st, free_blocks_word_size, scale); } } else { totals().print_on(st, scale); st->print(", "); - st->print("deallocated: " UINTX_FORMAT " blocks with ", _free_blocks_num); - print_scaled_words(st, _free_blocks_cap_words, scale); + st->print("deallocated: " UINTX_FORMAT " blocks with ", free_blocks_num); + print_scaled_words(st, free_blocks_word_size, scale); } } -// ClassLoaderMetaspaceStatistics methods +#ifdef ASSERT -ClassLoaderMetaspaceStatistics::ClassLoaderMetaspaceStatistics() { reset(); } +void arena_stats_t::verify() const { + size_t total_used = 0; + for (chunklevel_t l = chunklevel::LOWEST_CHUNK_LEVEL; l <= chunklevel::HIGHEST_CHUNK_LEVEL; l ++) { + stats[l].verify(); + total_used += stats[l].used_words; + } + // Deallocated allocations still count as used + assert(total_used >= free_blocks_word_size, + "Sanity"); +} +#endif -void ClassLoaderMetaspaceStatistics::reset() { - nonclass_sm_stats().reset(); - if (Metaspace::using_class_space()) { - class_sm_stats().reset(); - } + +// Returns total arena statistics for both class and non-class metaspace +arena_stats_t clms_stats_t::totals() const { + arena_stats_t out; + out.add(arena_stats_nonclass); + out.add(arena_stats_class); + return out; } -// Returns total space manager statistics for both class and non-class metaspace -SpaceManagerStatistics ClassLoaderMetaspaceStatistics::totals() const { - SpaceManagerStatistics stats; - stats.add(nonclass_sm_stats()); - if (Metaspace::using_class_space()) { - stats.add(class_sm_stats()); - } - return stats; -} - -void ClassLoaderMetaspaceStatistics::add(const ClassLoaderMetaspaceStatistics& other) { - nonclass_sm_stats().add(other.nonclass_sm_stats()); - if (Metaspace::using_class_space()) { - class_sm_stats().add(other.class_sm_stats()); - } -} - -void ClassLoaderMetaspaceStatistics::print_on(outputStream* st, size_t scale, bool detailed) const { +void clms_stats_t::print_on(outputStream* st, size_t scale, bool detailed) const { streamIndentor sti(st); st->cr_indent(); if (Metaspace::using_class_space()) { st->print("Non-Class: "); } - nonclass_sm_stats().print_on(st, scale, detailed); + arena_stats_nonclass.print_on(st, scale, detailed); if (detailed) { st->cr(); } if (Metaspace::using_class_space()) { st->cr_indent(); st->print(" Class: "); - class_sm_stats().print_on(st, scale, detailed); + arena_stats_class.print_on(st, scale, detailed); if (detailed) { st->cr(); } @@ -270,6 +243,14 @@ st->cr(); } + +#ifdef ASSERT +void clms_stats_t::verify() const { + arena_stats_nonclass.verify(); + arena_stats_class.verify(); +} +#endif + } // end namespace metaspace diff --git a/src/hotspot/share/memory/metaspace/metaspaceStatistics.hpp b/src/hotspot/share/memory/metaspace/metaspaceStatistics.hpp --- a/src/hotspot/share/memory/metaspace/metaspaceStatistics.hpp +++ b/src/hotspot/share/memory/metaspace/metaspaceStatistics.hpp @@ -1,6 +1,6 @@ /* - * Copyright (c) 2018, 2019, Oracle and/or its affiliates. All rights reserved. - * Copyright (c) 2018 SAP SE. All rights reserved. + * Copyright (c) 2018, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2018, 2020 SAP SE. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -26,162 +26,141 @@ #ifndef SHARE_MEMORY_METASPACE_METASPACESTATISTICS_HPP #define SHARE_MEMORY_METASPACE_METASPACESTATISTICS_HPP +#include "memory/metaspace.hpp" // for MetadataType enum +#include "memory/metaspace/chunkLevel.hpp" #include "utilities/globalDefinitions.hpp" -#include "memory/metaspace.hpp" // for MetadataType enum -#include "memory/metaspace/metachunk.hpp" // for ChunkIndex enum class outputStream; namespace metaspace { -// Contains statistics for a number of free chunks. -class FreeChunksStatistics { - uintx _num; // Number of chunks - size_t _cap; // Total capacity, in words +// Contains a number of data output structures: +// +// - cm_stats_t +// - clms_stats_t -> arena_stats_t -> in_use_chunk_stats_t +// +// used for the various XXXX::add_to_statistic() methods in MetaspaceArena, ClassLoaderMetaspace +// and ChunkManager, respectively. -public: - FreeChunksStatistics(); +struct cm_stats_t { - void reset(); + // How many chunks per level are checked in. + int num_chunks[chunklevel::NUM_CHUNK_LEVELS]; - uintx num() const { return _num; } - size_t cap() const { return _cap; } + // Size, in words, of the sum of all committed areas in this chunk manager, per level. + size_t committed_word_size[chunklevel::NUM_CHUNK_LEVELS]; - void add(uintx n, size_t s); - void add(const FreeChunksStatistics& other); - void print_on(outputStream* st, size_t scale) const; + cm_stats_t() : num_chunks(), committed_word_size() {} -}; // end: FreeChunksStatistics + void add(const cm_stats_t& other); + // Returns total word size of all chunks in this manager. + size_t total_word_size() const; -// Contains statistics for a ChunkManager. -class ChunkManagerStatistics { - - FreeChunksStatistics _chunk_stats[NumberOfInUseLists]; - -public: - - // Free chunk statistics, by chunk index. - const FreeChunksStatistics& chunk_stats(ChunkIndex index) const { return _chunk_stats[index]; } - FreeChunksStatistics& chunk_stats(ChunkIndex index) { return _chunk_stats[index]; } - - void reset(); - size_t total_capacity() const; + // Returns total committed word size of all chunks in this manager. + size_t total_committed_word_size() const; void print_on(outputStream* st, size_t scale) const; -}; // ChunkManagerStatistics + DEBUG_ONLY(void verify() const;) -// Contains statistics for a number of chunks in use. -// Each chunk has a used and free portion; however, there are current chunks (serving -// potential future metaspace allocations) and non-current chunks. Unused portion of the -// former is counted as free, unused portion of the latter counts as waste. -class UsedChunksStatistics { - uintx _num; // Number of chunks - size_t _cap; // Total capacity in words. - size_t _used; // Total used area, in words - size_t _free; // Total free area (unused portions of current chunks), in words - size_t _waste; // Total waste area (unused portions of non-current chunks), in words - size_t _overhead; // Total sum of chunk overheads, in words. +}; -public: +// Contains statistics for one or multiple chunks in use. +struct in_use_chunk_stats_t { - UsedChunksStatistics(); + // Number of chunks + int num; - void reset(); + // Note: + // capacity = committed + uncommitted + // committed = used + free + waste - uintx num() const { return _num; } + // Capacity (total sum of all chunk sizes) in words. + // May contain committed and uncommitted space. + size_t word_size; - // Total capacity, in words - size_t cap() const { return _cap; } + // Total committed area, in words. + size_t committed_words; - // Total used area, in words - size_t used() const { return _used; } + // Total used area, in words. + size_t used_words; - // Total free area (unused portions of current chunks), in words - size_t free() const { return _free; } + // Total free committed area, in words. + size_t free_words; - // Total waste area (unused portions of non-current chunks), in words - size_t waste() const { return _waste; } + // Total waste committed area, in words. + size_t waste_words; - // Total area spent in overhead (chunk headers), in words - size_t overhead() const { return _overhead; } + in_use_chunk_stats_t() + : num(0), word_size(0), committed_words(0), + used_words(0), free_words(0), waste_words(0) + {} - void add_num(uintx n) { _num += n; } - void add_cap(size_t s) { _cap += s; } - void add_used(size_t s) { _used += s; } - void add_free(size_t s) { _free += s; } - void add_waste(size_t s) { _waste += s; } - void add_overhead(size_t s) { _overhead += s; } + void add(const in_use_chunk_stats_t& other) { + num += other.num; + word_size += other.word_size; + committed_words += other.committed_words; + used_words += other.used_words; + free_words += other.free_words; + waste_words += other.waste_words; - void add(const UsedChunksStatistics& other); + } void print_on(outputStream* st, size_t scale) const; -#ifdef ASSERT - void check_sanity() const; -#endif + DEBUG_ONLY(void verify() const;) -}; // UsedChunksStatistics +}; -// Class containing statistics for one or more space managers. -class SpaceManagerStatistics { +// Class containing statistics for one or more MetaspaceArena objects. +struct arena_stats_t { - UsedChunksStatistics _chunk_stats[NumberOfInUseLists]; - uintx _free_blocks_num; - size_t _free_blocks_cap_words; + // chunk statistics by chunk level + in_use_chunk_stats_t stats[chunklevel::NUM_CHUNK_LEVELS]; + uintx free_blocks_num; + size_t free_blocks_word_size; -public: + arena_stats_t() + : stats(), + free_blocks_num(0), + free_blocks_word_size(0) + {} - SpaceManagerStatistics(); + void add(const arena_stats_t& other); - // Chunk statistics by chunk index - const UsedChunksStatistics& chunk_stats(ChunkIndex index) const { return _chunk_stats[index]; } - UsedChunksStatistics& chunk_stats(ChunkIndex index) { return _chunk_stats[index]; } + void print_on(outputStream* st, size_t scale = K, bool detailed = true) const; - uintx free_blocks_num () const { return _free_blocks_num; } - size_t free_blocks_cap_words () const { return _free_blocks_cap_words; } + in_use_chunk_stats_t totals() const; - void reset(); + DEBUG_ONLY(void verify() const;) - void add_free_blocks_info(uintx num, size_t cap); +}; - // Returns total chunk statistics over all chunk types. - UsedChunksStatistics totals() const; +// Statistics for one or multiple ClassLoaderMetaspace objects +struct clms_stats_t { - void add(const SpaceManagerStatistics& other); + arena_stats_t arena_stats_nonclass; + arena_stats_t arena_stats_class; - void print_on(outputStream* st, size_t scale, bool detailed) const; + clms_stats_t() : arena_stats_nonclass(), arena_stats_class() {} -}; // SpaceManagerStatistics - -class ClassLoaderMetaspaceStatistics { - - SpaceManagerStatistics _sm_stats[Metaspace::MetadataTypeCount]; - -public: - - ClassLoaderMetaspaceStatistics(); - - const SpaceManagerStatistics& sm_stats(Metaspace::MetadataType mdType) const { return _sm_stats[mdType]; } - SpaceManagerStatistics& sm_stats(Metaspace::MetadataType mdType) { return _sm_stats[mdType]; } - - const SpaceManagerStatistics& nonclass_sm_stats() const { return sm_stats(Metaspace::NonClassType); } - SpaceManagerStatistics& nonclass_sm_stats() { return sm_stats(Metaspace::NonClassType); } - const SpaceManagerStatistics& class_sm_stats() const { return sm_stats(Metaspace::ClassType); } - SpaceManagerStatistics& class_sm_stats() { return sm_stats(Metaspace::ClassType); } - - void reset(); - - void add(const ClassLoaderMetaspaceStatistics& other); - - // Returns total space manager statistics for both class and non-class metaspace - SpaceManagerStatistics totals() const; + void add(const clms_stats_t& other) { + arena_stats_nonclass.add(other.arena_stats_nonclass); + arena_stats_class.add(other.arena_stats_class); + } void print_on(outputStream* st, size_t scale, bool detailed) const; -}; // ClassLoaderMetaspaceStatistics + // Returns total statistics for both class and non-class metaspace + arena_stats_t totals() const; + + + DEBUG_ONLY(void verify() const;) + +}; } // namespace metaspace #endif // SHARE_MEMORY_METASPACE_METASPACESTATISTICS_HPP + diff --git a/src/hotspot/share/memory/metaspace/occupancyMap.cpp b/src/hotspot/share/memory/metaspace/occupancyMap.cpp deleted file mode 100644 --- a/src/hotspot/share/memory/metaspace/occupancyMap.cpp +++ /dev/null @@ -1,135 +0,0 @@ -/* - * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. - * Copyright (c) 2018 SAP SE. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "utilities/debug.hpp" -#include "utilities/globalDefinitions.hpp" -#include "memory/metaspace/metachunk.hpp" -#include "memory/metaspace/occupancyMap.hpp" -#include "runtime/os.hpp" - -namespace metaspace { - -OccupancyMap::OccupancyMap(const MetaWord* reference_address, size_t word_size, size_t smallest_chunk_word_size) : - _reference_address(reference_address), _word_size(word_size), - _smallest_chunk_word_size(smallest_chunk_word_size) -{ - assert(reference_address != NULL, "invalid reference address"); - assert(is_aligned(reference_address, smallest_chunk_word_size), - "Reference address not aligned to smallest chunk size."); - assert(is_aligned(word_size, smallest_chunk_word_size), - "Word_size shall be a multiple of the smallest chunk size."); - // Calculate bitmap size: one bit per smallest_chunk_word_size'd area. - size_t num_bits = word_size / smallest_chunk_word_size; - _map_size = (num_bits + 7) / 8; - assert(_map_size * 8 >= num_bits, "sanity"); - _map[0] = (uint8_t*) os::malloc(_map_size, mtInternal); - _map[1] = (uint8_t*) os::malloc(_map_size, mtInternal); - assert(_map[0] != NULL && _map[1] != NULL, "Occupancy Map: allocation failed."); - memset(_map[1], 0, _map_size); - memset(_map[0], 0, _map_size); - // Sanity test: the first respectively last possible chunk start address in - // the covered range shall map to the first and last bit in the bitmap. - assert(get_bitpos_for_address(reference_address) == 0, - "First chunk address in range must map to fist bit in bitmap."); - assert(get_bitpos_for_address(reference_address + word_size - smallest_chunk_word_size) == num_bits - 1, - "Last chunk address in range must map to last bit in bitmap."); -} - -OccupancyMap::~OccupancyMap() { - os::free(_map[0]); - os::free(_map[1]); -} - -#ifdef ASSERT -// Verify occupancy map for the address range [from, to). -// We need to tell it the address range, because the memory the -// occupancy map is covering may not be fully comitted yet. -void OccupancyMap::verify(MetaWord* from, MetaWord* to) { - Metachunk* chunk = NULL; - int nth_bit_for_chunk = 0; - MetaWord* chunk_end = NULL; - for (MetaWord* p = from; p < to; p += _smallest_chunk_word_size) { - const unsigned pos = get_bitpos_for_address(p); - // Check the chunk-starts-info: - if (get_bit_at_position(pos, layer_chunk_start_map)) { - // Chunk start marked in bitmap. - chunk = (Metachunk*) p; - if (chunk_end != NULL) { - assert(chunk_end == p, "Unexpected chunk start found at %p (expected " - "the next chunk to start at %p).", p, chunk_end); - } - assert(chunk->is_valid_sentinel(), "Invalid chunk at address %p.", p); - if (chunk->get_chunk_type() != HumongousIndex) { - guarantee(is_aligned(p, chunk->word_size()), "Chunk %p not aligned.", p); - } - chunk_end = p + chunk->word_size(); - nth_bit_for_chunk = 0; - assert(chunk_end <= to, "Chunk end overlaps test address range."); - } else { - // No chunk start marked in bitmap. - assert(chunk != NULL, "Chunk should start at start of address range."); - assert(p < chunk_end, "Did not find expected chunk start at %p.", p); - nth_bit_for_chunk ++; - } - // Check the in-use-info: - const bool in_use_bit = get_bit_at_position(pos, layer_in_use_map); - if (in_use_bit) { - assert(!chunk->is_tagged_free(), "Chunk %p: marked in-use in map but is free (bit %u).", - chunk, nth_bit_for_chunk); - } else { - assert(chunk->is_tagged_free(), "Chunk %p: marked free in map but is in use (bit %u).", - chunk, nth_bit_for_chunk); - } - } -} - -// Verify that a given chunk is correctly accounted for in the bitmap. -void OccupancyMap::verify_for_chunk(Metachunk* chunk) { - assert(chunk_starts_at_address((MetaWord*) chunk), - "No chunk start marked in map for chunk %p.", chunk); - // For chunks larger than the minimal chunk size, no other chunk - // must start in its area. - if (chunk->word_size() > _smallest_chunk_word_size) { - assert(!is_any_bit_set_in_region(((MetaWord*) chunk) + _smallest_chunk_word_size, - chunk->word_size() - _smallest_chunk_word_size, layer_chunk_start_map), - "No chunk must start within another chunk."); - } - if (!chunk->is_tagged_free()) { - assert(is_region_in_use((MetaWord*)chunk, chunk->word_size()), - "Chunk %p is in use but marked as free in map (%d %d).", - chunk, chunk->get_chunk_type(), chunk->get_origin()); - } else { - assert(!is_region_in_use((MetaWord*)chunk, chunk->word_size()), - "Chunk %p is free but marked as in-use in map (%d %d).", - chunk, chunk->get_chunk_type(), chunk->get_origin()); - } -} - -#endif // ASSERT - -} // namespace metaspace - - diff --git a/src/hotspot/share/memory/metaspace/occupancyMap.hpp b/src/hotspot/share/memory/metaspace/occupancyMap.hpp deleted file mode 100644 --- a/src/hotspot/share/memory/metaspace/occupancyMap.hpp +++ /dev/null @@ -1,242 +0,0 @@ -/* - * Copyright (c) 2018, 2019, Oracle and/or its affiliates. All rights reserved. - * Copyright (c) 2018 SAP SE. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_MEMORY_METASPACE_OCCUPANCYMAP_HPP -#define SHARE_MEMORY_METASPACE_OCCUPANCYMAP_HPP - -#include "memory/allocation.hpp" -#include "utilities/debug.hpp" -#include "utilities/globalDefinitions.hpp" - - -namespace metaspace { - -class Metachunk; - -// Helper for Occupancy Bitmap. A type trait to give an all-bits-are-one-unsigned constant. -template struct all_ones { static const T value; }; -template <> struct all_ones { static const uint64_t value = 0xFFFFFFFFFFFFFFFFULL; }; -template <> struct all_ones { static const uint32_t value = 0xFFFFFFFF; }; - -// The OccupancyMap is a bitmap which, for a given VirtualSpaceNode, -// keeps information about -// - where a chunk starts -// - whether a chunk is in-use or free -// A bit in this bitmap represents one range of memory in the smallest -// chunk size (SpecializedChunk or ClassSpecializedChunk). -class OccupancyMap : public CHeapObj { - - // The address range this map covers. - const MetaWord* const _reference_address; - const size_t _word_size; - - // The word size of a specialized chunk, aka the number of words one - // bit in this map represents. - const size_t _smallest_chunk_word_size; - - // map data - // Data are organized in two bit layers: - // The first layer is the chunk-start-map. Here, a bit is set to mark - // the corresponding region as the head of a chunk. - // The second layer is the in-use-map. Here, a set bit indicates that - // the corresponding belongs to a chunk which is in use. - uint8_t* _map[2]; - - enum { layer_chunk_start_map = 0, layer_in_use_map = 1 }; - - // length, in bytes, of bitmap data - size_t _map_size; - - // Returns true if bit at position pos at bit-layer layer is set. - bool get_bit_at_position(unsigned pos, unsigned layer) const { - assert(layer == 0 || layer == 1, "Invalid layer %d", layer); - const unsigned byteoffset = pos / 8; - assert(byteoffset < _map_size, - "invalid byte offset (%u), map size is " SIZE_FORMAT ".", byteoffset, _map_size); - const unsigned mask = 1 << (pos % 8); - return (_map[layer][byteoffset] & mask) > 0; - } - - // Changes bit at position pos at bit-layer layer to value v. - void set_bit_at_position(unsigned pos, unsigned layer, bool v) { - assert(layer == 0 || layer == 1, "Invalid layer %d", layer); - const unsigned byteoffset = pos / 8; - assert(byteoffset < _map_size, - "invalid byte offset (%u), map size is " SIZE_FORMAT ".", byteoffset, _map_size); - const unsigned mask = 1 << (pos % 8); - if (v) { - _map[layer][byteoffset] |= mask; - } else { - _map[layer][byteoffset] &= ~mask; - } - } - - // Optimized case of is_any_bit_set_in_region for 32/64bit aligned access: - // pos is 32/64 aligned and num_bits is 32/64. - // This is the typical case when coalescing to medium chunks, whose size is - // 32 or 64 times the specialized chunk size (depending on class or non class - // case), so they occupy 64 bits which should be 64bit aligned, because - // chunks are chunk-size aligned. - template - bool is_any_bit_set_in_region_3264(unsigned pos, unsigned num_bits, unsigned layer) const { - assert(_map_size > 0, "not initialized"); - assert(layer == 0 || layer == 1, "Invalid layer %d.", layer); - assert(pos % (sizeof(T) * 8) == 0, "Bit position must be aligned (%u).", pos); - assert(num_bits == (sizeof(T) * 8), "Number of bits incorrect (%u).", num_bits); - const size_t byteoffset = pos / 8; - assert(byteoffset <= (_map_size - sizeof(T)), - "Invalid byte offset (" SIZE_FORMAT "), map size is " SIZE_FORMAT ".", byteoffset, _map_size); - const T w = *(T*)(_map[layer] + byteoffset); - return w > 0 ? true : false; - } - - // Returns true if any bit in region [pos1, pos1 + num_bits) is set in bit-layer layer. - bool is_any_bit_set_in_region(unsigned pos, unsigned num_bits, unsigned layer) const { - if (pos % 32 == 0 && num_bits == 32) { - return is_any_bit_set_in_region_3264(pos, num_bits, layer); - } else if (pos % 64 == 0 && num_bits == 64) { - return is_any_bit_set_in_region_3264(pos, num_bits, layer); - } else { - for (unsigned n = 0; n < num_bits; n ++) { - if (get_bit_at_position(pos + n, layer)) { - return true; - } - } - } - return false; - } - - // Returns true if any bit in region [p, p+word_size) is set in bit-layer layer. - bool is_any_bit_set_in_region(MetaWord* p, size_t word_size, unsigned layer) const { - assert(word_size % _smallest_chunk_word_size == 0, - "Region size " SIZE_FORMAT " not a multiple of smallest chunk size.", word_size); - const unsigned pos = get_bitpos_for_address(p); - const unsigned num_bits = (unsigned) (word_size / _smallest_chunk_word_size); - return is_any_bit_set_in_region(pos, num_bits, layer); - } - - // Optimized case of set_bits_of_region for 32/64bit aligned access: - // pos is 32/64 aligned and num_bits is 32/64. - // This is the typical case when coalescing to medium chunks, whose size - // is 32 or 64 times the specialized chunk size (depending on class or non - // class case), so they occupy 64 bits which should be 64bit aligned, - // because chunks are chunk-size aligned. - template - void set_bits_of_region_T(unsigned pos, unsigned num_bits, unsigned layer, bool v) { - assert(pos % (sizeof(T) * 8) == 0, "Bit position must be aligned to %u (%u).", - (unsigned)(sizeof(T) * 8), pos); - assert(num_bits == (sizeof(T) * 8), "Number of bits incorrect (%u), expected %u.", - num_bits, (unsigned)(sizeof(T) * 8)); - const size_t byteoffset = pos / 8; - assert(byteoffset <= (_map_size - sizeof(T)), - "invalid byte offset (" SIZE_FORMAT "), map size is " SIZE_FORMAT ".", byteoffset, _map_size); - T* const pw = (T*)(_map[layer] + byteoffset); - *pw = v ? all_ones::value : (T) 0; - } - - // Set all bits in a region starting at pos to a value. - void set_bits_of_region(unsigned pos, unsigned num_bits, unsigned layer, bool v) { - assert(_map_size > 0, "not initialized"); - assert(layer == 0 || layer == 1, "Invalid layer %d.", layer); - if (pos % 32 == 0 && num_bits == 32) { - set_bits_of_region_T(pos, num_bits, layer, v); - } else if (pos % 64 == 0 && num_bits == 64) { - set_bits_of_region_T(pos, num_bits, layer, v); - } else { - for (unsigned n = 0; n < num_bits; n ++) { - set_bit_at_position(pos + n, layer, v); - } - } - } - - // Helper: sets all bits in a region [p, p+word_size). - void set_bits_of_region(MetaWord* p, size_t word_size, unsigned layer, bool v) { - assert(word_size % _smallest_chunk_word_size == 0, - "Region size " SIZE_FORMAT " not a multiple of smallest chunk size.", word_size); - const unsigned pos = get_bitpos_for_address(p); - const unsigned num_bits = (unsigned) (word_size / _smallest_chunk_word_size); - set_bits_of_region(pos, num_bits, layer, v); - } - - // Helper: given an address, return the bit position representing that address. - unsigned get_bitpos_for_address(const MetaWord* p) const { - assert(_reference_address != NULL, "not initialized"); - assert(p >= _reference_address && p < _reference_address + _word_size, - "Address %p out of range for occupancy map [%p..%p).", - p, _reference_address, _reference_address + _word_size); - assert(is_aligned(p, _smallest_chunk_word_size * sizeof(MetaWord)), - "Address not aligned (%p).", p); - const ptrdiff_t d = (p - _reference_address) / _smallest_chunk_word_size; - assert(d >= 0 && (size_t)d < _map_size * 8, "Sanity."); - return (unsigned) d; - } - - public: - - OccupancyMap(const MetaWord* reference_address, size_t word_size, size_t smallest_chunk_word_size); - ~OccupancyMap(); - - // Returns true if at address x a chunk is starting. - bool chunk_starts_at_address(MetaWord* p) const { - const unsigned pos = get_bitpos_for_address(p); - return get_bit_at_position(pos, layer_chunk_start_map); - } - - void set_chunk_starts_at_address(MetaWord* p, bool v) { - const unsigned pos = get_bitpos_for_address(p); - set_bit_at_position(pos, layer_chunk_start_map, v); - } - - // Removes all chunk-start-bits inside a region, typically as a - // result of a chunk merge. - void wipe_chunk_start_bits_in_region(MetaWord* p, size_t word_size) { - set_bits_of_region(p, word_size, layer_chunk_start_map, false); - } - - // Returns true if there are life (in use) chunks in the region limited - // by [p, p+word_size). - bool is_region_in_use(MetaWord* p, size_t word_size) const { - return is_any_bit_set_in_region(p, word_size, layer_in_use_map); - } - - // Marks the region starting at p with the size word_size as in use - // or free, depending on v. - void set_region_in_use(MetaWord* p, size_t word_size, bool v) { - set_bits_of_region(p, word_size, layer_in_use_map, v); - } - - // Verify occupancy map for the address range [from, to). - // We need to tell it the address range, because the memory the - // occupancy map is covering may not be fully comitted yet. - DEBUG_ONLY(void verify(MetaWord* from, MetaWord* to);) - - // Verify that a given chunk is correctly accounted for in the bitmap. - DEBUG_ONLY(void verify_for_chunk(Metachunk* chunk);) - -}; - -} // namespace metaspace - -#endif // SHARE_MEMORY_METASPACE_OCCUPANCYMAP_HPP diff --git a/src/hotspot/share/memory/metaspace/rootChunkArea.cpp b/src/hotspot/share/memory/metaspace/rootChunkArea.cpp new file mode 100644 --- /dev/null +++ b/src/hotspot/share/memory/metaspace/rootChunkArea.cpp @@ -0,0 +1,565 @@ +/* + * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020 SAP SE. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" + +#include "logging/log.hpp" +#include "memory/allocation.hpp" +#include "memory/metaspace/chunkHeaderPool.hpp" +#include "memory/metaspace/chunkManager.hpp" +#include "memory/metaspace/freeChunkList.hpp" +#include "memory/metaspace/metachunk.hpp" +#include "memory/metaspace/metaspaceCommon.hpp" +#include "memory/metaspace/rootChunkArea.hpp" +#include "runtime/mutexLocker.hpp" +#include "utilities/debug.hpp" +#include "utilities/globalDefinitions.hpp" + +namespace metaspace { + +RootChunkArea::RootChunkArea(const MetaWord* base) + : _base(base), _first_chunk(NULL) +{} + +RootChunkArea::~RootChunkArea() { + // This is called when a VirtualSpaceNode is destructed (purged). + // All chunks should be free of course. In fact, there should only + // be one chunk, since all free chunks should have been merged. + if (_first_chunk != NULL) { + assert(_first_chunk->is_root_chunk() && _first_chunk->is_free(), + "Cannot delete root chunk area if not all chunks are free."); + ChunkHeaderPool::pool()->return_chunk_header(_first_chunk); + } +} + +// Initialize: allocate a root node and a root chunk header; return the +// root chunk header. It will be partly initialized. +// Note: this just allocates a memory-less header; memory itself is allocated inside VirtualSpaceNode. +Metachunk* RootChunkArea::alloc_root_chunk_header(VirtualSpaceNode* node) { + + assert(_first_chunk == 0, "already have a root"); + + Metachunk* c = ChunkHeaderPool::pool()->allocate_chunk_header(); + c->initialize(node, const_cast(_base), chunklevel::ROOT_CHUNK_LEVEL); + + _first_chunk = c; + + return c; + +} + +// Given a chunk c, split it recursively until you get a chunk of the given target_level. +// +// The resulting target chunk resides at the same address as the original chunk. +// The resulting splinters are added to freelists. +// +// Returns pointer to the result chunk; the splitted-off chunks are added as +// free chunks to the freelists. +void RootChunkArea::split(chunklevel_t target_level, Metachunk* c, FreeChunkListVector* freelists) { + + // Splitting a chunk once works like this: + // + // For a given chunk we want to split: + // - increase the chunk level (which halves its size) + // - (but leave base address as it is since it will be the leader of the newly + // created chunk pair) + // - then create a new chunk header of the same level, set its memory range + // to cover the second halfof the old chunk. + // - wire them up (prev_in_vs/next_in_vs) + // - return the follower chunk as "splinter chunk" in the splinters array. + + // Doing this multiple times will create a new free splinter chunk for every + // level we split: + // + // A <- original chunk + // + // B B <- split into two halves + // + // C C B <- first half split again + // + // D D C B <- first half split again ... + // + + // As an optimization, since we usually do not split once but multiple times, + // to not do each split separately, since we would have to wire up prev_in_vs/next_in_vs + // on every level just to tear it open in the next level when we reintroduce a new + // half chunk splinter. + // Instead, just split split split and delay building up the double linked list of the + // new chunks at the end of all splits. + + DEBUG_ONLY(check_pointer(c->base());) + DEBUG_ONLY(c->verify(false);) + assert(c->is_free(), "Can only split free chunks."); + + DEBUG_ONLY(chunklevel::check_valid_level(target_level)); + assert(target_level > c->level(), "Wrong target level"); + + const chunklevel_t starting_level = c->level(); + + while (c->level() < target_level) { + + log_trace(metaspace)("Splitting chunk: " METACHUNK_FULL_FORMAT ".", METACHUNK_FULL_FORMAT_ARGS(c)); + + c->inc_level(); + Metachunk* splinter_chunk = ChunkHeaderPool::pool()->allocate_chunk_header(); + splinter_chunk->initialize(c->vsnode(), c->end(), c->level()); + + // Fix committed words info: If over the half of the original chunk was + // committed, committed area spills over into the follower chunk. + const size_t old_committed_words = c->committed_words(); + if (old_committed_words > c->word_size()) { + c->set_committed_words(c->word_size()); + splinter_chunk->set_committed_words(old_committed_words - c->word_size()); + } else { + splinter_chunk->set_committed_words(0); + } + + // Insert splinter chunk into vs list + if (c->next_in_vs() != NULL) { + c->next_in_vs()->set_prev_in_vs(splinter_chunk); + } + splinter_chunk->set_next_in_vs(c->next_in_vs()); + splinter_chunk->set_prev_in_vs(c); + c->set_next_in_vs(splinter_chunk); + + log_trace(metaspace)(".. Result chunk: " METACHUNK_FULL_FORMAT ".", METACHUNK_FULL_FORMAT_ARGS(c)); + log_trace(metaspace)(".. Splinter chunk: " METACHUNK_FULL_FORMAT ".", METACHUNK_FULL_FORMAT_ARGS(splinter_chunk)); + + // Add splinter to free lists + freelists->add(splinter_chunk); + + } + + assert(c->level() == target_level, "Sanity"); + + DEBUG_ONLY(verify(true);) + DEBUG_ONLY(c->verify(true);) + +} + + +// Given a chunk, attempt to merge it recursively with its neighboring chunks. +// +// If successful (merged at least once), returns address of +// the merged chunk; NULL otherwise. +// +// The merged chunks are removed from the freelists. +// +// !!! Please note that if this method returns a non-NULL value, the +// original chunk will be invalid and should not be accessed anymore! !!! +Metachunk* RootChunkArea::merge(Metachunk* c, FreeChunkListVector* freelists) { + + // Note rules: + // + // - a chunk always has a buddy, unless it is a root chunk. + // - In that buddy pair, a chunk is either leader or follower. + // - a chunk's base address is always aligned at its size. + // - if chunk is leader, its base address is also aligned to the size of the next + // lower level, at least. A follower chunk is not. + + // How we merge once: + // + // For a given chunk c, which has to be free and non-root, we do: + // - find out if we are the leader or the follower chunk + // - if we are leader, next_in_vs must be the follower; if we are follower, + // prev_in_vs must be the leader. Now we have the buddy chunk. + // - However, if the buddy chunk itself is split (of a level higher than us) + // we cannot merge. + // - we can only merge if the buddy is of the same level as we are and it is + // free. + // - Then we merge by simply removing the follower chunk from the address range + // linked list (returning the now useless header to the pool) and decreasing + // the leader chunk level by one. That makes it double the size. + + // Example: + // (lower case chunks are free, the * indicates the chunk we want to merge): + // + // ........................ + // d d*c b A <- we return the second (d*) chunk... + // + // c* c b A <- we merge it with its predecessor and decrease its level... + // + // b* b A <- we merge it again, since its new neighbor was free too... + // + // a* A <- we merge it again, since its new neighbor was free too... + // + // And we are done, since its new neighbor, (A), is not free. We would also be done + // if the new neighbor itself is splintered. + + DEBUG_ONLY(check_pointer(c->base());) + assert(!c->is_root_chunk(), "Cannot be merged further."); + assert(c->is_free(), "Can only merge free chunks."); + + DEBUG_ONLY(c->verify(false);) + + log_trace(metaspace)("Attempting to merge chunk " METACHUNK_FORMAT ".", METACHUNK_FORMAT_ARGS(c)); + + const chunklevel_t starting_level = c->level(); + + bool stop = false; + Metachunk* result = NULL; + + do { + + // First find out if this chunk is the leader of its pair + const bool is_leader = c->is_leader(); + + // Note: this is either our buddy or a splinter of the buddy. + Metachunk* const buddy = c->is_leader() ? c->next_in_vs() : c->prev_in_vs(); + DEBUG_ONLY(buddy->verify(true);) + + // A buddy chunk must be of the same or higher level (so, same size or smaller) + // never be larger. + assert(buddy->level() >= c->level(), "Sanity"); + + // Is this really my buddy (same level) or a splinter of it (higher level)? + // Also, is it free? + if (buddy->level() != c->level() || buddy->is_free() == false) { + + log_trace(metaspace)("cannot merge with chunk " METACHUNK_FORMAT ".", METACHUNK_FORMAT_ARGS(buddy)); + + stop = true; + + } else { + + log_trace(metaspace)("will merge with chunk " METACHUNK_FORMAT ".", METACHUNK_FORMAT_ARGS(buddy)); + + // We can merge with the buddy. + + // First, remove buddy from the chunk manager. + assert(buddy->is_free(), "Sanity"); + freelists->remove(buddy); + + // Determine current leader and follower + Metachunk* leader; + Metachunk* follower; + if (is_leader) { + leader = c; follower = buddy; + } else { + leader = buddy; follower = c; + } + + // Last checkpoint + assert(leader->end() == follower->base() && + leader->level() == follower->level() && + leader->is_free() && follower->is_free(), "Sanity"); + + // The new merged chunk is as far committed as possible (if leader + // chunk is fully committed, as far as the follower chunk). + size_t merged_committed_words = leader->committed_words(); + if (merged_committed_words == leader->word_size()) { + merged_committed_words += follower->committed_words(); + } + + // Leader survives, follower chunk is freed. Remove follower from vslist .. + leader->set_next_in_vs(follower->next_in_vs()); + if (follower->next_in_vs() != NULL) { + follower->next_in_vs()->set_prev_in_vs(leader); + } + + // .. and return follower chunk header to pool for reuse. + ChunkHeaderPool::pool()->return_chunk_header(follower); + + // Leader level gets decreased (leader chunk doubles in size) but + // base address stays the same. + leader->dec_level(); + + // set commit boundary + leader->set_committed_words(merged_committed_words); + + // If the leader is now of root chunk size, stop merging + if (leader->is_root_chunk()) { + stop = true; + } + + result = c = leader; + + DEBUG_ONLY(leader->verify(true);) + + } + + } while (!stop); + +#ifdef ASSERT + verify(true); + if (result != NULL) { + result->verify(true); + } +#endif // ASSERT + + return result; + +} + +// Given a chunk c, which must be "in use" and must not be a root chunk, attempt to +// enlarge it in place by claiming its trailing buddy. +// +// This will only work if c is the leader of the buddy pair and the trailing buddy is free. +// +// If successful, the follower chunk will be removed from the freelists, the leader chunk c will +// double in size (level decreased by one). +// +// On success, true is returned, false otherwise. +bool RootChunkArea::attempt_enlarge_chunk(Metachunk* c, FreeChunkListVector* freelists) { + + DEBUG_ONLY(check_pointer(c->base());) + assert(!c->is_root_chunk(), "Cannot be merged further."); + + // There is no real reason for this limitation other than it is not + // needed on free chunks since they should be merged already: + assert(c->is_in_use(), "Can only enlarge in use chunks."); + + DEBUG_ONLY(c->verify(false);) + + if (!c->is_leader()) { + return false; + } + + // We are the leader, so the buddy must follow us. + Metachunk* const buddy = c->next_in_vs(); + DEBUG_ONLY(buddy->verify(true);) + + // Of course buddy cannot be larger than us. + assert(buddy->level() >= c->level(), "Sanity"); + + // We cannot merge buddy in if it is not free... + if (!buddy->is_free()) { + return false; + } + + // ... nor if it is splintered. + if (buddy->level() != c->level()) { + return false; + } + + // Okay, lets enlarge c. + + log_trace(metaspace)("Enlarging chunk " METACHUNK_FULL_FORMAT " by merging in follower " METACHUNK_FULL_FORMAT ".", + METACHUNK_FULL_FORMAT_ARGS(c), METACHUNK_FULL_FORMAT_ARGS(buddy)); + + // the enlarged c is as far committed as possible: + size_t merged_committed_words = c->committed_words(); + if (merged_committed_words == c->word_size()) { + merged_committed_words += buddy->committed_words(); + } + + // Remove buddy from vs list... + Metachunk* successor = buddy->next_in_vs(); + if (successor != NULL) { + successor->set_prev_in_vs(c); + } + c->set_next_in_vs(successor); + + // .. and from freelist ... + freelists->remove(buddy); + + // .. and return its empty husk to the pool... + ChunkHeaderPool::pool()->return_chunk_header(buddy); + + // Then decrease level of c. + c->dec_level(); + + // and correct committed words if needed. + c->set_committed_words(merged_committed_words); + + log_debug(metaspace)("Enlarged chunk " METACHUNK_FULL_FORMAT ".", METACHUNK_FULL_FORMAT_ARGS(c)); +// log_debug(metaspace)("Enlarged chunk " METACHUNK_FORMAT ".", METACHUNK_FORMAT_ARGS(c)); + + DEBUG_ONLY(verify(true)); + + return true; + +} + +// Returns true if this root chunk area is completely free: +// In that case, it should only contain one chunk (maximally merged, so a root chunk) +// and it should be free. +bool RootChunkArea::is_free() const { + return _first_chunk == NULL || + (_first_chunk->is_root_chunk() && _first_chunk->is_free()); +} + + +#ifdef ASSERT + +#define assrt_(cond, ...) \ + if (!(cond)) { \ + fdStream errst(2); \ + this->print_on(&errst); \ + vmassert(cond, __VA_ARGS__); \ + } + +void RootChunkArea::verify(bool slow) const { + + + assert_lock_strong(MetaspaceExpand_lock); + assert_is_aligned(_base, chunklevel::MAX_CHUNK_BYTE_SIZE); + + // Iterate thru all chunks in this area. They must be ordered correctly, + // being adjacent to each other, and cover the complete area + int num_chunk = 0; + + if (_first_chunk != NULL) { + + assrt_(_first_chunk->prev_in_vs() == NULL, "Sanity"); + + const Metachunk* c = _first_chunk; + const MetaWord* expected_next_base = _base; + const MetaWord* const area_end = _base + word_size(); + + while (c != NULL) { + + assrt_(c->is_free() || c->is_in_use(), + "Chunk No. %d " METACHUNK_FORMAT " - invalid state.", + num_chunk, METACHUNK_FORMAT_ARGS(c)); + + assrt_(c->base() == expected_next_base, + "Chunk No. %d " METACHUNK_FORMAT " - unexpected base.", + num_chunk, METACHUNK_FORMAT_ARGS(c)); + + assrt_(c->base() >= base() && c->end() <= end(), + "chunk %d " METACHUNK_FORMAT " oob for this root area [" PTR_FORMAT ".." PTR_FORMAT ").", + num_chunk, METACHUNK_FORMAT_ARGS(c), p2i(base()), p2i(end())); + + assrt_(is_aligned(c->base(), c->word_size()), + "misaligned chunk %d " METACHUNK_FORMAT ".", num_chunk, METACHUNK_FORMAT_ARGS(c)); + + c->verify_neighborhood(); + c->verify(slow); + + expected_next_base = c->end(); + num_chunk ++; + + c = c->next_in_vs(); + + } + assrt_(expected_next_base == _base + word_size(), "Sanity"); + } + +} + +void RootChunkArea::verify_area_is_ideally_merged() const { + + SOMETIMES(assert_lock_strong(MetaspaceExpand_lock);) + + int num_chunk = 0; + for (const Metachunk* c = _first_chunk; c != NULL; c = c->next_in_vs()) { + if (!c->is_root_chunk() && c->is_free()) { + // If a chunk is free, it must not have a buddy which is also free, because + // those chunks should have been merged. + // In other words, a buddy shall be either in-use or splintered + // (which in turn would mean part of it are in use). + Metachunk* const buddy = c->is_leader() ? c->next_in_vs() : c->prev_in_vs(); + assrt_(buddy->is_in_use() || buddy->level() > c->level(), + "Chunk No. %d " METACHUNK_FORMAT " : missed merge opportunity with neighbor " METACHUNK_FORMAT ".", + num_chunk, METACHUNK_FORMAT_ARGS(c), METACHUNK_FORMAT_ARGS(buddy)); + } + num_chunk ++; + } +} + +#endif + +void RootChunkArea::print_on(outputStream* st) const { + + st->print(PTR_FORMAT ": ", p2i(base())); + if (_first_chunk != NULL) { + const Metachunk* c = _first_chunk; + // 01234567890123 + const char* letters_for_levels_cap = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"; + const char* letters_for_levels = "abcdefghijklmnopqrstuvwxyz"; + while (c != NULL) { + const chunklevel_t l = c->level(); + if (l >= 0 && (size_t)l < strlen(letters_for_levels)) { +// c->print_on(st); st->cr(); + st->print("%c", c->is_free() ? letters_for_levels[c->level()] : letters_for_levels_cap[c->level()]); + } else { + // Obviously garbage, but lets not crash. + st->print("?"); + } + c = c->next_in_vs(); + } + } else { + st->print(" (no chunks)"); + } + st->cr(); + +} + + +// Create an array of ChunkTree objects, all initialized to NULL, covering +// a given memory range. Memory range must be a multiple of root chunk size. +RootChunkAreaLUT::RootChunkAreaLUT(const MetaWord* base, size_t word_size) + : _base(base), + _num((int)(word_size / chunklevel::MAX_CHUNK_WORD_SIZE)), + _arr(NULL) +{ + assert_is_aligned(word_size, chunklevel::MAX_CHUNK_WORD_SIZE); + _arr = NEW_C_HEAP_ARRAY(RootChunkArea, _num, mtClass); + const MetaWord* this_base = _base; + for (int i = 0; i < _num; i ++) { + RootChunkArea* rca = new(_arr + i) RootChunkArea(this_base); + assert(rca == _arr + i, "Sanity"); + this_base += chunklevel::MAX_CHUNK_WORD_SIZE; + } +} + +RootChunkAreaLUT::~RootChunkAreaLUT() { + for (int i = 0; i < _num; i ++) { + _arr[i].~RootChunkArea(); + } + FREE_C_HEAP_ARRAY(RootChunkArea, _arr); +} + +// Returns true if all areas in this area table are free (only contain free chunks). +bool RootChunkAreaLUT::is_free() const { + for (int i = 0; i < _num; i ++) { + if (!_arr[i].is_free()) { + return false; + } + } + return true; +} + +#ifdef ASSERT + +void RootChunkAreaLUT::verify(bool slow) const { + for (int i = 0; i < _num; i ++) { + check_pointer(_arr[i].base()); + _arr[i].verify(slow); + } +} + +#endif + +void RootChunkAreaLUT::print_on(outputStream* st) const { + for (int i = 0; i < _num; i ++) { + st->print("%2d:", i); + _arr[i].print_on(st); + } +} + + +} // end: namespace metaspace diff --git a/src/hotspot/share/memory/metaspace/rootChunkArea.hpp b/src/hotspot/share/memory/metaspace/rootChunkArea.hpp new file mode 100644 --- /dev/null +++ b/src/hotspot/share/memory/metaspace/rootChunkArea.hpp @@ -0,0 +1,211 @@ +/* + * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020 SAP SE. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_MEMORY_METASPACE_ROOTCHUNKAREA_HPP +#define SHARE_MEMORY_METASPACE_ROOTCHUNKAREA_HPP + +#include "memory/allocation.hpp" +#include "memory/metaspace/chunkLevel.hpp" +#include "utilities/debug.hpp" +#include "utilities/globalDefinitions.hpp" + +class outputStream; + +namespace metaspace { + +class Metachunk; +class MetachunkClosure; +class FreeChunkListVector; +class VirtualSpaceNode; + + +// RootChunkArea manages a memory area covering a single root chunk. +// +// Such an area may contain a single root chunk, or a number of chunks the +// root chunk was split into. +// +// RootChunkArea contains the functionality to merge and split chunks in +// buddy allocator fashion. +// + +class RootChunkArea { + + // The base address of this area. + // Todo: this may be somewhat superfluous since RootChunkArea only exist in the + // context of a series of chunks, so the address is somewhat implicit. Remove? + const MetaWord* const _base; + + // The first chunk in this area; if this area is maximally + // folded, this is the root chunk covering the whole area size. + Metachunk* _first_chunk; + +public: + + RootChunkArea(const MetaWord* base); + ~RootChunkArea(); + + // Initialize: allocate a root node and a root chunk header; return the + // root chunk header. It will be partly initialized. + // Note: this just allocates a memory-less header; memory itself is allocated inside VirtualSpaceNode. + Metachunk* alloc_root_chunk_header(VirtualSpaceNode* node); + + + // Given a chunk c, split it recursively until you get a chunk of the given target_level. + // + // The resulting target chunk resides at the same address as the original chunk. + // The resulting splinters are added to freelists. + // + // Returns pointer to the result chunk; the splitted-off chunks are added as + // free chunks to the freelists. + void split(chunklevel_t target_level, Metachunk* c, FreeChunkListVector* freelists); + + // Given a chunk, attempt to merge it recursively with its neighboring chunks. + // + // If successful (merged at least once), returns address of + // the merged chunk; NULL otherwise. + // + // The merged chunks are removed from the freelists. + // + // !!! Please note that if this method returns a non-NULL value, the + // original chunk will be invalid and should not be accessed anymore! !!! + Metachunk* merge(Metachunk* c, FreeChunkListVector* freelists); + + // Given a chunk c, which must be "in use" and must not be a root chunk, attempt to + // enlarge it in place by claiming its trailing buddy. + // + // This will only work if c is the leader of the buddy pair and the trailing buddy is free. + // + // If successful, the follower chunk will be removed from the freelists, the leader chunk c will + // double in size (level decreased by one). + // + // On success, true is returned, false otherwise. + bool attempt_enlarge_chunk(Metachunk* c, FreeChunkListVector* freelists); + + /// range /// + + const MetaWord* base() const { return _base; } + size_t word_size() const { return chunklevel::MAX_CHUNK_WORD_SIZE; } + const MetaWord* end() const { return _base + word_size(); } + + // Direct access to the first chunk (use with care) + Metachunk* first_chunk() { return _first_chunk; } + const Metachunk* first_chunk() const { return _first_chunk; } + + // Returns true if this root chunk area is completely free: + // In that case, it should only contain one chunk (maximally merged, so a root chunk) + // and it should be free. + bool is_free() const; + + //// Debug stuff //// + +#ifdef ASSERT + void check_pointer(const MetaWord* p) const { + assert(p >= _base && p < _base + word_size(), + "pointer " PTR_FORMAT " oob for this root area [" PTR_FORMAT ".." PTR_FORMAT ")", + p2i(p), p2i(_base), p2i(_base + word_size())); + } + void verify(bool slow) const; + + // This is a separate operation from verify(). We should be able to call verify() + // from almost anywhere, regardless of state, but verify_area_is_ideally_merged() + // can only be called outside split and merge ops. + void verify_area_is_ideally_merged() const; +#endif // ASSERT + + void print_on(outputStream* st) const; + +}; + + +// RootChunkAreaLUT (lookup table) manages a series of contiguous root chunk areas +// in memory (in the context of a VirtualSpaceNode). It allows finding the containing +// root chunk for any given memory address. It allows for easy iteration over all +// root chunks. +// Beyond that it is unexciting. +class RootChunkAreaLUT { + + // Base address of the whole area. + const MetaWord* const _base; + + // Number of root chunk areas. + const int _num; + + // Array of RootChunkArea objects. + RootChunkArea* _arr; + +#ifdef ASSERT + void check_pointer(const MetaWord* p) const { + assert(p >= base() && p < base() + word_size(), "Invalid pointer"); + } +#endif + + // Given an address into this range, return the index into the area array for the + // area this address falls into. + int index_by_address(const MetaWord* p) const { + DEBUG_ONLY(check_pointer(p);) + int idx = (int)((p - base()) / chunklevel::MAX_CHUNK_WORD_SIZE); + assert(idx >= 0 && idx < _num, "Sanity"); + return idx; + } + +public: + + RootChunkAreaLUT(const MetaWord* base, size_t word_size); + ~RootChunkAreaLUT(); + + // Given a memory address into the range this array covers, return the + // corresponding area object. If none existed at this position, create it + // on demand. + RootChunkArea* get_area_by_address(const MetaWord* p) const { + const int idx = index_by_address(p); + RootChunkArea* ra = _arr + idx; + DEBUG_ONLY(ra->check_pointer(p);) + return _arr + idx; + } + + // Access area by its index + int number_of_areas() const { return _num; } + RootChunkArea* get_area_by_index(int index) { assert(index >= 0 && index < _num, "oob"); return _arr + index; } + const RootChunkArea* get_area_by_index(int index) const { assert(index >= 0 && index < _num, "oob"); return _arr + index; } + + /// range /// + + const MetaWord* base() const { return _base; } + size_t word_size() const { return _num * chunklevel::MAX_CHUNK_WORD_SIZE; } + const MetaWord* end() const { return _base + word_size(); } + + // Returns true if all areas in this area table are free (only contain free chunks). + bool is_free() const; + + DEBUG_ONLY(void verify(bool slow) const;) + + void print_on(outputStream* st) const; + +}; + + +} // namespace metaspace + +#endif // SHARE_MEMORY_METASPACE_ROOTCHUNKAREA_HPP diff --git a/src/hotspot/share/memory/metaspace/runningCounters.cpp b/src/hotspot/share/memory/metaspace/runningCounters.cpp new file mode 100644 --- /dev/null +++ b/src/hotspot/share/memory/metaspace/runningCounters.cpp @@ -0,0 +1,99 @@ +/* + * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020 SAP SE. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "memory/metaspace/counter.hpp" +#include "memory/metaspace/runningCounters.hpp" +#include "memory/metaspace/virtualSpaceList.hpp" +#include "memory/metaspace/chunkManager.hpp" + +namespace metaspace { + +SizeAtomicCounter RunningCounters::_used_class_counter; +SizeAtomicCounter RunningCounters::_used_nonclass_counter; + +// Return reserved size, in words, for Metaspace +size_t RunningCounters::reserved_words() { + return reserved_words_class() + reserved_words_nonclass(); +} + +size_t RunningCounters::reserved_words_class() { + VirtualSpaceList* vs = VirtualSpaceList::vslist_class(); + return vs != NULL ? vs->reserved_words() : 0; +} + +size_t RunningCounters::reserved_words_nonclass() { + return VirtualSpaceList::vslist_nonclass()->reserved_words(); +} + +// Return total committed size, in words, for Metaspace +size_t RunningCounters::committed_words() { + return committed_words_class() + committed_words_nonclass(); +} + +size_t RunningCounters::committed_words_class() { + VirtualSpaceList* vs = VirtualSpaceList::vslist_class(); + return vs != NULL ? vs->committed_words() : 0; +} + +size_t RunningCounters::committed_words_nonclass() { + return VirtualSpaceList::vslist_nonclass()->committed_words(); +} + + +// ---- used chunks ----- + +// Returns size, in words, used for metadata. +size_t RunningCounters::used_words() { + return used_words_class() + used_words_nonclass(); +} + +size_t RunningCounters::used_words_class() { + return _used_class_counter.get(); +} + +size_t RunningCounters::used_words_nonclass() { + return _used_nonclass_counter.get(); +} + +// ---- free chunks ----- + +// Returns size, in words, of all chunks in all freelists. +size_t RunningCounters::free_chunks_words() { + return free_chunks_words_class() + free_chunks_words_nonclass(); +} + +size_t RunningCounters::free_chunks_words_class() { + ChunkManager* cm = ChunkManager::chunkmanager_class(); + return cm != NULL ? cm->total_word_size() : 0; +} + +size_t RunningCounters::free_chunks_words_nonclass() { + return ChunkManager::chunkmanager_nonclass()->total_word_size(); +} + +} // namespace metaspace + + diff --git a/src/hotspot/share/memory/metaspace/runningCounters.hpp b/src/hotspot/share/memory/metaspace/runningCounters.hpp new file mode 100644 --- /dev/null +++ b/src/hotspot/share/memory/metaspace/runningCounters.hpp @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020 SAP SE. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_MEMORY_METASPACE_RUNNINGCOUNTERS_HPP +#define SHARE_MEMORY_METASPACE_RUNNINGCOUNTERS_HPP + +#include "memory/allocation.hpp" +#include "memory/metaspace/counter.hpp" + +namespace metaspace { + +// This class is a convenience interface for accessing global metaspace counters. +class RunningCounters : public AllStatic { + + static SizeAtomicCounter _used_class_counter; + static SizeAtomicCounter _used_nonclass_counter; + +public: + + // ---- virtual memory ----- + + // Return reserved size, in words, for Metaspace + static size_t reserved_words(); + static size_t reserved_words_class(); + static size_t reserved_words_nonclass(); + + // Return total committed size, in words, for Metaspace + static size_t committed_words(); + static size_t committed_words_class(); + static size_t committed_words_nonclass(); + + + // ---- used chunks ----- + + // Returns size, in words, used for metadata. + static size_t used_words(); + static size_t used_words_class(); + static size_t used_words_nonclass(); + + // ---- free chunks ----- + + // Returns size, in words, of all chunks in all freelists. + static size_t free_chunks_words(); + static size_t free_chunks_words_class(); + static size_t free_chunks_words_nonclass(); + + // Direct access to the counters. + static SizeAtomicCounter* used_nonclass_counter() { return &_used_nonclass_counter; } + static SizeAtomicCounter* used_class_counter() { return &_used_class_counter; } + +}; + +} // namespace metaspace + +#endif // SHARE_MEMORY_METASPACE_RUNNINGCOUNTERS_HPP + + diff --git a/src/hotspot/share/memory/metaspace/settings.cpp b/src/hotspot/share/memory/metaspace/settings.cpp new file mode 100644 --- /dev/null +++ b/src/hotspot/share/memory/metaspace/settings.cpp @@ -0,0 +1,140 @@ +/* + * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020 SAP SE. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" + +#include "logging/log.hpp" +#include "logging/logStream.hpp" + +#include "memory/metaspace/chunkLevel.hpp" +#include "memory/metaspace/settings.hpp" + +#include "utilities/globalDefinitions.hpp" +#include "utilities/debug.hpp" + +namespace metaspace { + +size_t Settings::_commit_granule_bytes = 0; +size_t Settings::_commit_granule_words = 0; + +bool Settings::_new_chunks_are_fully_committed = false; +bool Settings::_uncommit_free_chunks = false; + +DEBUG_ONLY(bool Settings::_use_allocation_guard = false;) +DEBUG_ONLY(bool Settings::_handle_deallocations = true;) + + +void Settings::ergo_initialize() { + + if (strcmp(MetaspaceReclaimPolicy, "none") == 0) { + + log_info(metaspace)("Initialized with strategy: no reclaim."); + + _commit_granule_bytes = MAX2((size_t)os::vm_page_size(), 64 * K); + _commit_granule_words = _commit_granule_bytes / BytesPerWord; + + // In "none" reclamation mode, we do not uncommit, and we commit new chunks fully; + // that very closely mimicks the behaviour of old Metaspace. + _new_chunks_are_fully_committed = true; + _uncommit_free_chunks = false; + + + } else if (strcmp(MetaspaceReclaimPolicy, "aggressive") == 0) { + + log_info(metaspace)("Initialized with strategy: aggressive reclaim."); + + // Set the granule size rather small; may increase + // mapping fragmentation but also increase chance to uncommit. + _commit_granule_bytes = MAX2((size_t)os::vm_page_size(), 16 * K); + _commit_granule_words = _commit_granule_bytes / BytesPerWord; + + _new_chunks_are_fully_committed = false; + _uncommit_free_chunks = true; + + } else if (strcmp(MetaspaceReclaimPolicy, "balanced") == 0) { + + log_info(metaspace)("Initialized with strategy: balanced reclaim."); + + _commit_granule_bytes = MAX2((size_t)os::vm_page_size(), 64 * K); + _commit_granule_words = _commit_granule_bytes / BytesPerWord; + + _new_chunks_are_fully_committed = false; + _uncommit_free_chunks = true; + + } else { + + vm_exit_during_initialization("Invalid value for MetaspaceReclaimPolicy: \"%s\".", MetaspaceReclaimPolicy); + + } + + // Sanity checks. + assert(commit_granule_words() <= chunklevel::MAX_CHUNK_WORD_SIZE, "Too large granule size"); + assert(is_power_of_2(commit_granule_words()), "granule size must be a power of 2"); + + // Should always be true since root chunk size should be much larger than alloc granularity + assert(is_aligned(_virtual_space_node_reserve_alignment_words * BytesPerWord, + os::vm_allocation_granularity()), "Sanity"); + +#ifdef ASSERT + // Off for release builds, and by default for debug builds, but can be switched on manually to aid + // error analysis. + _use_allocation_guard = MetaspaceGuardAllocations; + + // Deallocations can be manually switched off to aid error analysis, since this removes one layer of complexity + // from allocation. + _handle_deallocations = MetaspaceHandleDeallocations; + + // We also switch it off automatically if we use allocation guards. This is to keep prefix handling in MetaspaceArena simple. + if (_use_allocation_guard) { + _handle_deallocations = false; + } +#endif + + LogStream ls(Log(metaspace)::info()); + Settings::print_on(&ls); + +} + +void Settings::print_on(outputStream* st) { + + st->print_cr(" - commit_granule_bytes: " SIZE_FORMAT ".", commit_granule_bytes()); + st->print_cr(" - commit_granule_words: " SIZE_FORMAT ".", commit_granule_words()); + + + st->print_cr(" - virtual_space_node_default_size: " SIZE_FORMAT ".", virtual_space_node_default_word_size()); + + st->print_cr(" - enlarge_chunks_in_place: %d.", (int)enlarge_chunks_in_place()); + st->print_cr(" - enlarge_chunks_in_place_max_word_size: " SIZE_FORMAT ".", enlarge_chunks_in_place_max_word_size()); + + st->print_cr(" - new_chunks_are_fully_committed: %d.", (int)new_chunks_are_fully_committed()); + st->print_cr(" - uncommit_free_chunks: %d.", (int)uncommit_free_chunks()); + + st->print_cr(" - use_allocation_guard: %d.", (int)use_allocation_guard()); + st->print_cr(" - handle_deallocations: %d.", (int)handle_deallocations()); + +} + +} // namespace metaspace + diff --git a/src/hotspot/share/memory/metaspace/settings.hpp b/src/hotspot/share/memory/metaspace/settings.hpp new file mode 100644 --- /dev/null +++ b/src/hotspot/share/memory/metaspace/settings.hpp @@ -0,0 +1,94 @@ +/* + * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020 SAP SE. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_MEMORY_METASPACE_CONSTANTS_HPP +#define SHARE_MEMORY_METASPACE_CONSTANTS_HPP + +#include "memory/allocation.hpp" +#include "memory/metaspace/chunkLevel.hpp" +#include "utilities/globalDefinitions.hpp" + +namespace metaspace { + +class Settings : public AllStatic { + + // Granularity, in bytes, metaspace is committed with. + static size_t _commit_granule_bytes; + + // Granularity, in words, metaspace is committed with. + static size_t _commit_granule_words; + + // The default size of a non-class VirtualSpaceNode (unless created differently). + // Must be a multiple of the root chunk size. + static const size_t _virtual_space_node_default_word_size = chunklevel::MAX_CHUNK_WORD_SIZE * 2; // lets go with 8mb virt size. Seems a good compromise betw. virt and mapping fragmentation. + + // Alignment of the base address of a virtual space node + static const size_t _virtual_space_node_reserve_alignment_words = chunklevel::MAX_CHUNK_WORD_SIZE; + + // When allocating from a chunk, if the remaining area in the chunk is too small to hold + // the requested size, we attempt to double the chunk size in place... + static const bool _enlarge_chunks_in_place = true; + + // .. but we do only do this for chunks below a given size to prevent unnecessary memory blowup. + static const size_t _enlarge_chunks_in_place_max_word_size = 256 * K; + + // Whether or not chunks handed out to an arena start out fully committed; + // if true, this deactivates committing-on-demand (irregardless of whether + // we uncommit free chunks). + static bool _new_chunks_are_fully_committed; + + // If true, chunks equal or larger than a commit granule are uncommitted + // after being returned to the freelist. + static bool _uncommit_free_chunks; + + // If true, metablock allocations are guarded and periodically checked. + DEBUG_ONLY(static bool _use_allocation_guard;) + + // If true, we handle deallocated blocks (default). + DEBUG_ONLY(static bool _handle_deallocations;) + +public: + + static size_t commit_granule_bytes() { return _commit_granule_bytes; } + static size_t commit_granule_words() { return _commit_granule_words; } + static bool new_chunks_are_fully_committed() { return _new_chunks_are_fully_committed; } + static size_t virtual_space_node_default_word_size() { return _virtual_space_node_default_word_size; } + static size_t virtual_space_node_reserve_alignment_words() { return _virtual_space_node_reserve_alignment_words; } + static bool enlarge_chunks_in_place() { return _enlarge_chunks_in_place; } + static size_t enlarge_chunks_in_place_max_word_size() { return _enlarge_chunks_in_place_max_word_size; } + static bool uncommit_free_chunks() { return _uncommit_free_chunks; } + + static bool use_allocation_guard() { return DEBUG_ONLY(_use_allocation_guard) NOT_DEBUG(false); } + static bool handle_deallocations() { return DEBUG_ONLY(_handle_deallocations) NOT_DEBUG(true); } + + static void ergo_initialize(); + + static void print_on(outputStream* st); + +}; + +} // namespace metaspace + +#endif // SHARE_MEMORY_METASPACE_BLOCKFREELIST_HPP diff --git a/src/hotspot/share/memory/metaspace/smallBlocks.cpp b/src/hotspot/share/memory/metaspace/smallBlocks.cpp deleted file mode 100644 --- a/src/hotspot/share/memory/metaspace/smallBlocks.cpp +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ -#include "precompiled.hpp" - -#include "memory/metaspace/smallBlocks.hpp" -#include "utilities/globalDefinitions.hpp" -#include "utilities/ostream.hpp" - -namespace metaspace { - -void SmallBlocks::print_on(outputStream* st) const { - st->print_cr("SmallBlocks:"); - for (uint i = _small_block_min_size; i < _small_block_max_size; i++) { - uint k = i - _small_block_min_size; - st->print_cr("small_lists size " SIZE_FORMAT " count " SIZE_FORMAT, _small_lists[k].size(), _small_lists[k].count()); - } -} - - -// Returns the total size, in words, of all blocks, across all block sizes. -size_t SmallBlocks::total_size() const { - size_t result = 0; - for (uint i = _small_block_min_size; i < _small_block_max_size; i++) { - uint k = i - _small_block_min_size; - result = result + _small_lists[k].count() * _small_lists[k].size(); - } - return result; -} - -// Returns the total number of all blocks across all block sizes. -uintx SmallBlocks::total_num_blocks() const { - uintx result = 0; - for (uint i = _small_block_min_size; i < _small_block_max_size; i++) { - uint k = i - _small_block_min_size; - result = result + _small_lists[k].count(); - } - return result; -} - -} // namespace metaspace - diff --git a/src/hotspot/share/memory/metaspace/smallBlocks.hpp b/src/hotspot/share/memory/metaspace/smallBlocks.hpp deleted file mode 100644 --- a/src/hotspot/share/memory/metaspace/smallBlocks.hpp +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Copyright (c) 2018, 2019, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_MEMORY_METASPACE_SMALLBLOCKS_HPP -#define SHARE_MEMORY_METASPACE_SMALLBLOCKS_HPP - -#include "memory/allocation.hpp" -#include "memory/binaryTreeDictionary.hpp" -#include "memory/metaspace/metablock.hpp" -#include "utilities/globalDefinitions.hpp" - -class outputStream; - -namespace metaspace { - -class SmallBlocks : public CHeapObj { - - const static uint _small_block_max_size = sizeof(TreeChunk >)/HeapWordSize; - // Note: this corresponds to the imposed miminum allocation size, see SpaceManager::get_allocation_word_size() - const static uint _small_block_min_size = sizeof(Metablock)/HeapWordSize; - -private: - FreeList _small_lists[_small_block_max_size - _small_block_min_size]; - - FreeList& list_at(size_t word_size) { - assert(word_size >= _small_block_min_size, "There are no metaspace objects less than %u words", _small_block_min_size); - return _small_lists[word_size - _small_block_min_size]; - } - -public: - SmallBlocks() { - for (uint i = _small_block_min_size; i < _small_block_max_size; i++) { - uint k = i - _small_block_min_size; - _small_lists[k].set_size(i); - } - } - - // Returns the total size, in words, of all blocks, across all block sizes. - size_t total_size() const; - - // Returns the total number of all blocks across all block sizes. - uintx total_num_blocks() const; - - static uint small_block_max_size() { return _small_block_max_size; } - static uint small_block_min_size() { return _small_block_min_size; } - - MetaWord* get_block(size_t word_size) { - if (list_at(word_size).count() > 0) { - MetaWord* new_block = (MetaWord*) list_at(word_size).get_chunk_at_head(); - return new_block; - } else { - return NULL; - } - } - void return_block(Metablock* free_chunk, size_t word_size) { - list_at(word_size).return_chunk_at_head(free_chunk, false); - assert(list_at(word_size).count() > 0, "Should have a chunk"); - } - - void print_on(outputStream* st) const; - -}; - -} // namespace metaspace - - -#endif // SHARE_MEMORY_METASPACE_SMALLBLOCKS_HPP diff --git a/src/hotspot/share/memory/metaspace/spaceManager.cpp b/src/hotspot/share/memory/metaspace/spaceManager.cpp deleted file mode 100644 --- a/src/hotspot/share/memory/metaspace/spaceManager.cpp +++ /dev/null @@ -1,528 +0,0 @@ -/* - * Copyright (c) 2018, 2020, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ -#include "precompiled.hpp" - -#include "logging/log.hpp" -#include "logging/logStream.hpp" -#include "memory/metaspace/chunkManager.hpp" -#include "memory/metaspace/metachunk.hpp" -#include "memory/metaspace/metaDebug.hpp" -#include "memory/metaspace/metaspaceCommon.hpp" -#include "memory/metaspace/spaceManager.hpp" -#include "memory/metaspace/virtualSpaceList.hpp" -#include "runtime/atomic.hpp" -#include "runtime/init.hpp" -#include "services/memoryService.hpp" -#include "utilities/debug.hpp" -#include "utilities/globalDefinitions.hpp" - -namespace metaspace { - -#define assert_counter(expected_value, real_value, msg) \ - assert( (expected_value) == (real_value), \ - "Counter mismatch (%s): expected " SIZE_FORMAT \ - ", but got: " SIZE_FORMAT ".", msg, expected_value, \ - real_value); - -// SpaceManager methods - -size_t SpaceManager::adjust_initial_chunk_size(size_t requested, bool is_class_space) { - size_t chunk_sizes[] = { - specialized_chunk_size(is_class_space), - small_chunk_size(is_class_space), - medium_chunk_size(is_class_space) - }; - - // Adjust up to one of the fixed chunk sizes ... - for (size_t i = 0; i < ARRAY_SIZE(chunk_sizes); i++) { - if (requested <= chunk_sizes[i]) { - return chunk_sizes[i]; - } - } - - // ... or return the size as a humongous chunk. - return requested; -} - -size_t SpaceManager::adjust_initial_chunk_size(size_t requested) const { - return adjust_initial_chunk_size(requested, is_class()); -} - -size_t SpaceManager::get_initial_chunk_size(Metaspace::MetaspaceType type) const { - size_t requested; - - if (is_class()) { - switch (type) { - case Metaspace::BootMetaspaceType: requested = Metaspace::first_class_chunk_word_size(); break; - case Metaspace::ClassMirrorHolderMetaspaceType: requested = ClassSpecializedChunk; break; - case Metaspace::ReflectionMetaspaceType: requested = ClassSpecializedChunk; break; - default: requested = ClassSmallChunk; break; - } - } else { - switch (type) { - case Metaspace::BootMetaspaceType: requested = Metaspace::first_chunk_word_size(); break; - case Metaspace::ClassMirrorHolderMetaspaceType: requested = SpecializedChunk; break; - case Metaspace::ReflectionMetaspaceType: requested = SpecializedChunk; break; - default: requested = SmallChunk; break; - } - } - - // Adjust to one of the fixed chunk sizes (unless humongous) - const size_t adjusted = adjust_initial_chunk_size(requested); - - assert(adjusted != 0, "Incorrect initial chunk size. Requested: " - SIZE_FORMAT " adjusted: " SIZE_FORMAT, requested, adjusted); - - return adjusted; -} - -void SpaceManager::locked_print_chunks_in_use_on(outputStream* st) const { - - for (ChunkIndex i = ZeroIndex; i < NumberOfInUseLists; i = next_chunk_index(i)) { - st->print("SpaceManager: " UINTX_FORMAT " %s chunks.", - num_chunks_by_type(i), chunk_size_name(i)); - } - - chunk_manager()->locked_print_free_chunks(st); -} - -size_t SpaceManager::calc_chunk_size(size_t word_size) { - - // Decide between a small chunk and a medium chunk. Up to - // _small_chunk_limit small chunks can be allocated. - // After that a medium chunk is preferred. - size_t chunk_word_size; - - // Special case for hidden metadata space. - // ClassMirrorHolder metadata space is usually small since it is used for - // class loader data's whose life cycle is governed by one class such as a - // non-strong hidden class or unsafe anonymous class. The majority within 1K - 2K range and - // rarely about 4K (64-bits JVM). - // Instead of jumping to SmallChunk after initial chunk exhausted, keeping allocation - // from SpecializeChunk up to _anon_or_delegating_metadata_specialize_chunk_limit (4) - // reduces space waste from 60+% to around 30%. - if ((_space_type == Metaspace::ClassMirrorHolderMetaspaceType || _space_type == Metaspace::ReflectionMetaspaceType) && - _mdtype == Metaspace::NonClassType && - num_chunks_by_type(SpecializedIndex) < anon_and_delegating_metadata_specialize_chunk_limit && - word_size + Metachunk::overhead() <= SpecializedChunk) { - return SpecializedChunk; - } - - if (num_chunks_by_type(MediumIndex) == 0 && - num_chunks_by_type(SmallIndex) < small_chunk_limit) { - chunk_word_size = (size_t) small_chunk_size(); - if (word_size + Metachunk::overhead() > small_chunk_size()) { - chunk_word_size = medium_chunk_size(); - } - } else { - chunk_word_size = medium_chunk_size(); - } - - // Might still need a humongous chunk. Enforce - // humongous allocations sizes to be aligned up to - // the smallest chunk size. - size_t if_humongous_sized_chunk = - align_up(word_size + Metachunk::overhead(), - smallest_chunk_size()); - chunk_word_size = - MAX2((size_t) chunk_word_size, if_humongous_sized_chunk); - - assert(!SpaceManager::is_humongous(word_size) || - chunk_word_size == if_humongous_sized_chunk, - "Size calculation is wrong, word_size " SIZE_FORMAT - " chunk_word_size " SIZE_FORMAT, - word_size, chunk_word_size); - Log(gc, metaspace, alloc) log; - if (log.is_trace() && SpaceManager::is_humongous(word_size)) { - log.trace("Metadata humongous allocation:"); - log.trace(" word_size " PTR_FORMAT, word_size); - log.trace(" chunk_word_size " PTR_FORMAT, chunk_word_size); - log.trace(" chunk overhead " PTR_FORMAT, Metachunk::overhead()); - } - return chunk_word_size; -} - -void SpaceManager::track_metaspace_memory_usage() { - if (is_init_completed()) { - if (is_class()) { - MemoryService::track_compressed_class_memory_usage(); - } - MemoryService::track_metaspace_memory_usage(); - } -} - -MetaWord* SpaceManager::grow_and_allocate(size_t word_size) { - assert_lock_strong(_lock); - assert(vs_list()->current_virtual_space() != NULL, - "Should have been set"); - assert(current_chunk() == NULL || - current_chunk()->allocate(word_size) == NULL, - "Don't need to expand"); - MutexLocker cl(MetaspaceExpand_lock, Mutex::_no_safepoint_check_flag); - - if (log_is_enabled(Trace, gc, metaspace, freelist)) { - size_t words_left = 0; - size_t words_used = 0; - if (current_chunk() != NULL) { - words_left = current_chunk()->free_word_size(); - words_used = current_chunk()->used_word_size(); - } - log_trace(gc, metaspace, freelist)("SpaceManager::grow_and_allocate for " SIZE_FORMAT " words " SIZE_FORMAT " words used " SIZE_FORMAT " words left", - word_size, words_used, words_left); - } - - // Get another chunk - size_t chunk_word_size = calc_chunk_size(word_size); - Metachunk* next = get_new_chunk(chunk_word_size); - - MetaWord* mem = NULL; - - // If a chunk was available, add it to the in-use chunk list - // and do an allocation from it. - if (next != NULL) { - // Add to this manager's list of chunks in use. - // If the new chunk is humongous, it was created to serve a single large allocation. In that - // case it usually makes no sense to make it the current chunk, since the next allocation would - // need to allocate a new chunk anyway, while we would now prematurely retire a perfectly - // good chunk which could be used for more normal allocations. - bool make_current = true; - if (next->get_chunk_type() == HumongousIndex && - current_chunk() != NULL) { - make_current = false; - } - add_chunk(next, make_current); - mem = next->allocate(word_size); - } - - // Track metaspace memory usage statistic. - track_metaspace_memory_usage(); - - return mem; -} - -void SpaceManager::print_on(outputStream* st) const { - SpaceManagerStatistics stat; - add_to_statistics(&stat); // will lock _lock. - stat.print_on(st, 1*K, false); -} - -SpaceManager::SpaceManager(Metaspace::MetadataType mdtype, - Metaspace::MetaspaceType space_type,// - Mutex* lock) : - _lock(lock), - _mdtype(mdtype), - _space_type(space_type), - _chunk_list(NULL), - _current_chunk(NULL), - _overhead_words(0), - _capacity_words(0), - _used_words(0), - _block_freelists(NULL) { - Metadebug::init_allocation_fail_alot_count(); - memset(_num_chunks_by_type, 0, sizeof(_num_chunks_by_type)); - log_trace(gc, metaspace, freelist)("SpaceManager(): " PTR_FORMAT, p2i(this)); -} - -void SpaceManager::account_for_new_chunk(const Metachunk* new_chunk) { - - assert_lock_strong(MetaspaceExpand_lock); - - _capacity_words += new_chunk->word_size(); - _overhead_words += Metachunk::overhead(); - DEBUG_ONLY(new_chunk->verify()); - _num_chunks_by_type[new_chunk->get_chunk_type()] ++; - - // Adjust global counters: - MetaspaceUtils::inc_capacity(mdtype(), new_chunk->word_size()); - MetaspaceUtils::inc_overhead(mdtype(), Metachunk::overhead()); -} - -void SpaceManager::account_for_allocation(size_t words) { - // Note: we should be locked with the ClassloaderData-specific metaspace lock. - // We may or may not be locked with the global metaspace expansion lock. - assert_lock_strong(lock()); - - // Add to the per SpaceManager totals. This can be done non-atomically. - _used_words += words; - - // Adjust global counters. This will be done atomically. - MetaspaceUtils::inc_used(mdtype(), words); -} - -void SpaceManager::account_for_spacemanager_death() { - - assert_lock_strong(MetaspaceExpand_lock); - - MetaspaceUtils::dec_capacity(mdtype(), _capacity_words); - MetaspaceUtils::dec_overhead(mdtype(), _overhead_words); - MetaspaceUtils::dec_used(mdtype(), _used_words); -} - -SpaceManager::~SpaceManager() { - - // This call this->_lock which can't be done while holding MetaspaceExpand_lock - DEBUG_ONLY(verify_metrics()); - - MutexLocker fcl(MetaspaceExpand_lock, Mutex::_no_safepoint_check_flag); - - account_for_spacemanager_death(); - - Log(gc, metaspace, freelist) log; - if (log.is_trace()) { - log.trace("~SpaceManager(): " PTR_FORMAT, p2i(this)); - ResourceMark rm; - LogStream ls(log.trace()); - locked_print_chunks_in_use_on(&ls); - if (block_freelists() != NULL) { - block_freelists()->print_on(&ls); - } - } - - // Add all the chunks in use by this space manager - // to the global list of free chunks. - - // Follow each list of chunks-in-use and add them to the - // free lists. Each list is NULL terminated. - chunk_manager()->return_chunk_list(chunk_list()); -#ifdef ASSERT - _chunk_list = NULL; - _current_chunk = NULL; -#endif - -#ifdef ASSERT - EVERY_NTH(VerifyMetaspaceInterval) - chunk_manager()->locked_verify(true); - END_EVERY_NTH -#endif - - if (_block_freelists != NULL) { - delete _block_freelists; - } -} - -void SpaceManager::deallocate(MetaWord* p, size_t word_size) { - assert_lock_strong(lock()); - // Allocations and deallocations are in raw_word_size - size_t raw_word_size = get_allocation_word_size(word_size); - // Lazily create a block_freelist - if (block_freelists() == NULL) { - _block_freelists = new BlockFreelist(); - } - block_freelists()->return_block(p, raw_word_size); - DEBUG_ONLY(Atomic::inc(&(g_internal_statistics.num_deallocs))); -} - -// Adds a chunk to the list of chunks in use. -void SpaceManager::add_chunk(Metachunk* new_chunk, bool make_current) { - - assert_lock_strong(_lock); - assert(new_chunk != NULL, "Should not be NULL"); - assert(new_chunk->next() == NULL, "Should not be on a list"); - - new_chunk->reset_empty(); - - // Find the correct list and and set the current - // chunk for that list. - ChunkIndex index = chunk_manager()->list_index(new_chunk->word_size()); - - if (make_current) { - // If we are to make the chunk current, retire the old current chunk and replace - // it with the new chunk. - retire_current_chunk(); - set_current_chunk(new_chunk); - } - - // Add the new chunk at the head of its respective chunk list. - new_chunk->set_next(_chunk_list); - _chunk_list = new_chunk; - - // Adjust counters. - account_for_new_chunk(new_chunk); - - assert(new_chunk->is_empty(), "Not ready for reuse"); - Log(gc, metaspace, freelist) log; - if (log.is_trace()) { - log.trace("SpaceManager::added chunk: "); - ResourceMark rm; - LogStream ls(log.trace()); - new_chunk->print_on(&ls); - chunk_manager()->locked_print_free_chunks(&ls); - } -} - -void SpaceManager::retire_current_chunk() { - if (current_chunk() != NULL) { - size_t remaining_words = current_chunk()->free_word_size(); - if (remaining_words >= SmallBlocks::small_block_min_size()) { - MetaWord* ptr = current_chunk()->allocate(remaining_words); - deallocate(ptr, remaining_words); - account_for_allocation(remaining_words); - } - } -} - -Metachunk* SpaceManager::get_new_chunk(size_t chunk_word_size) { - // Get a chunk from the chunk freelist - Metachunk* next = chunk_manager()->chunk_freelist_allocate(chunk_word_size); - - if (next == NULL) { - next = vs_list()->get_new_chunk(chunk_word_size, - medium_chunk_bunch()); - } - - Log(gc, metaspace, alloc) log; - if (log.is_trace() && next != NULL && - SpaceManager::is_humongous(next->word_size())) { - log.trace(" new humongous chunk word size " PTR_FORMAT, next->word_size()); - } - - return next; -} - -MetaWord* SpaceManager::allocate(size_t word_size) { - MutexLocker cl(lock(), Mutex::_no_safepoint_check_flag); - size_t raw_word_size = get_allocation_word_size(word_size); - BlockFreelist* fl = block_freelists(); - MetaWord* p = NULL; - - // Allocation from the dictionary is expensive in the sense that - // the dictionary has to be searched for a size. Don't allocate - // from the dictionary until it starts to get fat. Is this - // a reasonable policy? Maybe an skinny dictionary is fast enough - // for allocations. Do some profiling. JJJ - if (fl != NULL && fl->total_size() > allocation_from_dictionary_limit) { - p = fl->get_block(raw_word_size); - if (p != NULL) { - DEBUG_ONLY(Atomic::inc(&g_internal_statistics.num_allocs_from_deallocated_blocks)); - } - } - if (p == NULL) { - p = allocate_work(raw_word_size); - } - -#ifdef ASSERT - EVERY_NTH(VerifyMetaspaceInterval) - verify_metrics_locked(); - END_EVERY_NTH -#endif - - return p; -} - -// Returns the address of spaced allocated for "word_size". -// This methods does not know about blocks (Metablocks) -MetaWord* SpaceManager::allocate_work(size_t word_size) { - assert_lock_strong(lock()); -#ifdef ASSERT - if (Metadebug::test_metadata_failure()) { - return NULL; - } -#endif - // Is there space in the current chunk? - MetaWord* result = NULL; - - if (current_chunk() != NULL) { - result = current_chunk()->allocate(word_size); - } - - if (result == NULL) { - result = grow_and_allocate(word_size); - } - - if (result != NULL) { - account_for_allocation(word_size); - } - - return result; -} - -void SpaceManager::verify() { - Metachunk* curr = chunk_list(); - while (curr != NULL) { - DEBUG_ONLY(do_verify_chunk(curr);) - assert(curr->is_tagged_free() == false, "Chunk should be tagged as in use."); - curr = curr->next(); - } -} - -void SpaceManager::verify_chunk_size(Metachunk* chunk) { - assert(is_humongous(chunk->word_size()) || - chunk->word_size() == medium_chunk_size() || - chunk->word_size() == small_chunk_size() || - chunk->word_size() == specialized_chunk_size(), - "Chunk size is wrong"); - return; -} - -void SpaceManager::add_to_statistics_locked(SpaceManagerStatistics* out) const { - assert_lock_strong(lock()); - Metachunk* chunk = chunk_list(); - while (chunk != NULL) { - UsedChunksStatistics& chunk_stat = out->chunk_stats(chunk->get_chunk_type()); - chunk_stat.add_num(1); - chunk_stat.add_cap(chunk->word_size()); - chunk_stat.add_overhead(Metachunk::overhead()); - chunk_stat.add_used(chunk->used_word_size() - Metachunk::overhead()); - if (chunk != current_chunk()) { - chunk_stat.add_waste(chunk->free_word_size()); - } else { - chunk_stat.add_free(chunk->free_word_size()); - } - chunk = chunk->next(); - } - if (block_freelists() != NULL) { - out->add_free_blocks_info(block_freelists()->num_blocks(), block_freelists()->total_size()); - } -} - -void SpaceManager::add_to_statistics(SpaceManagerStatistics* out) const { - MutexLocker cl(lock(), Mutex::_no_safepoint_check_flag); - add_to_statistics_locked(out); -} - -#ifdef ASSERT -void SpaceManager::verify_metrics_locked() const { - assert_lock_strong(lock()); - - SpaceManagerStatistics stat; - add_to_statistics_locked(&stat); - - UsedChunksStatistics chunk_stats = stat.totals(); - - DEBUG_ONLY(chunk_stats.check_sanity()); - - assert_counter(_capacity_words, chunk_stats.cap(), "SpaceManager::_capacity_words"); - assert_counter(_used_words, chunk_stats.used(), "SpaceManager::_used_words"); - assert_counter(_overhead_words, chunk_stats.overhead(), "SpaceManager::_overhead_words"); -} - -void SpaceManager::verify_metrics() const { - MutexLocker cl(lock(), Mutex::_no_safepoint_check_flag); - verify_metrics_locked(); -} -#endif // ASSERT - - -} // namespace metaspace - diff --git a/src/hotspot/share/memory/metaspace/spaceManager.hpp b/src/hotspot/share/memory/metaspace/spaceManager.hpp deleted file mode 100644 --- a/src/hotspot/share/memory/metaspace/spaceManager.hpp +++ /dev/null @@ -1,233 +0,0 @@ -/* - * Copyright (c) 2018, 2019, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_MEMORY_METASPACE_SPACEMANAGER_HPP -#define SHARE_MEMORY_METASPACE_SPACEMANAGER_HPP - -#include "memory/allocation.hpp" -#include "memory/metaspace.hpp" -#include "memory/metaspace/blockFreelist.hpp" -#include "memory/metaspace/metaspaceCommon.hpp" -#include "memory/metaspace/metachunk.hpp" -#include "memory/metaspace/metaspaceStatistics.hpp" -#include "utilities/debug.hpp" -#include "utilities/globalDefinitions.hpp" - -class outputStream; -class Mutex; - -namespace metaspace { - -// SpaceManager - used by Metaspace to handle allocations -class SpaceManager : public CHeapObj { - friend class ::ClassLoaderMetaspace; - friend class Metadebug; - - private: - - // protects allocations - Mutex* const _lock; - - // Type of metadata allocated. - const Metaspace::MetadataType _mdtype; - - // Type of metaspace - const Metaspace::MetaspaceType _space_type; - - // List of chunks in use by this SpaceManager. Allocations - // are done from the current chunk. The list is used for deallocating - // chunks when the SpaceManager is freed. - Metachunk* _chunk_list; - Metachunk* _current_chunk; - - enum { - - // Maximum number of small chunks to allocate to a SpaceManager - small_chunk_limit = 4, - - // Maximum number of specialize chunks to allocate for anonymous and delegating - // metadata space to a SpaceManager - anon_and_delegating_metadata_specialize_chunk_limit = 4, - - allocation_from_dictionary_limit = 4 * K - - }; - - // Some running counters, but lets keep their number small to not add to much to - // the per-classloader footprint. - // Note: capacity = used + free + waste + overhead. We do not keep running counters for - // free and waste. Their sum can be deduced from the three other values. - size_t _overhead_words; - size_t _capacity_words; - size_t _used_words; - uintx _num_chunks_by_type[NumberOfInUseLists]; - - // Free lists of blocks are per SpaceManager since they - // are assumed to be in chunks in use by the SpaceManager - // and all chunks in use by a SpaceManager are freed when - // the class loader using the SpaceManager is collected. - BlockFreelist* _block_freelists; - - private: - // Accessors - Metachunk* chunk_list() const { return _chunk_list; } - - BlockFreelist* block_freelists() const { return _block_freelists; } - - Metaspace::MetadataType mdtype() { return _mdtype; } - - VirtualSpaceList* vs_list() const { return Metaspace::get_space_list(_mdtype); } - ChunkManager* chunk_manager() const { return Metaspace::get_chunk_manager(_mdtype); } - - Metachunk* current_chunk() const { return _current_chunk; } - void set_current_chunk(Metachunk* v) { - _current_chunk = v; - } - - Metachunk* find_current_chunk(size_t word_size); - - // Add chunk to the list of chunks in use - void add_chunk(Metachunk* v, bool make_current); - void retire_current_chunk(); - - Mutex* lock() const { return _lock; } - - // Adds to the given statistic object. Expects to be locked with lock(). - void add_to_statistics_locked(SpaceManagerStatistics* out) const; - - // Verify internal counters against the current state. Expects to be locked with lock(). - DEBUG_ONLY(void verify_metrics_locked() const;) - - public: - SpaceManager(Metaspace::MetadataType mdtype, - Metaspace::MetaspaceType space_type, - Mutex* lock); - ~SpaceManager(); - - enum ChunkMultiples { - MediumChunkMultiple = 4 - }; - - static size_t specialized_chunk_size(bool is_class) { return is_class ? ClassSpecializedChunk : SpecializedChunk; } - static size_t small_chunk_size(bool is_class) { return is_class ? ClassSmallChunk : SmallChunk; } - static size_t medium_chunk_size(bool is_class) { return is_class ? ClassMediumChunk : MediumChunk; } - - static size_t smallest_chunk_size(bool is_class) { return specialized_chunk_size(is_class); } - - // Accessors - bool is_class() const { return _mdtype == Metaspace::ClassType; } - - size_t specialized_chunk_size() const { return specialized_chunk_size(is_class()); } - size_t small_chunk_size() const { return small_chunk_size(is_class()); } - size_t medium_chunk_size() const { return medium_chunk_size(is_class()); } - - size_t smallest_chunk_size() const { return smallest_chunk_size(is_class()); } - - size_t medium_chunk_bunch() const { return medium_chunk_size() * MediumChunkMultiple; } - - bool is_humongous(size_t word_size) { return word_size > medium_chunk_size(); } - - size_t capacity_words() const { return _capacity_words; } - size_t used_words() const { return _used_words; } - size_t overhead_words() const { return _overhead_words; } - - // Adjust local, global counters after a new chunk has been added. - void account_for_new_chunk(const Metachunk* new_chunk); - - // Adjust local, global counters after space has been allocated from the current chunk. - void account_for_allocation(size_t words); - - // Adjust global counters just before the SpaceManager dies, after all its chunks - // have been returned to the freelist. - void account_for_spacemanager_death(); - - // Adjust the initial chunk size to match one of the fixed chunk list sizes, - // or return the unadjusted size if the requested size is humongous. - static size_t adjust_initial_chunk_size(size_t requested, bool is_class_space); - size_t adjust_initial_chunk_size(size_t requested) const; - - // Get the initial chunks size for this metaspace type. - size_t get_initial_chunk_size(Metaspace::MetaspaceType type) const; - - // Todo: remove this once we have counters by chunk type. - uintx num_chunks_by_type(ChunkIndex chunk_type) const { return _num_chunks_by_type[chunk_type]; } - - Metachunk* get_new_chunk(size_t chunk_word_size); - - // Block allocation and deallocation. - // Allocates a block from the current chunk - MetaWord* allocate(size_t word_size); - - // Helper for allocations - MetaWord* allocate_work(size_t word_size); - - // Returns a block to the per manager freelist - void deallocate(MetaWord* p, size_t word_size); - - // Based on the allocation size and a minimum chunk size, - // returned chunk size (for expanding space for chunk allocation). - size_t calc_chunk_size(size_t allocation_word_size); - - // Called when an allocation from the current chunk fails. - // Gets a new chunk (may require getting a new virtual space), - // and allocates from that chunk. - MetaWord* grow_and_allocate(size_t word_size); - - // Notify memory usage to MemoryService. - void track_metaspace_memory_usage(); - - // debugging support. - - void print_on(outputStream* st) const; - void locked_print_chunks_in_use_on(outputStream* st) const; - - void verify(); - void verify_chunk_size(Metachunk* chunk); - - // This adjusts the size given to be greater than the minimum allocation size in - // words for data in metaspace. Esentially the minimum size is currently 3 words. - size_t get_allocation_word_size(size_t word_size) { - size_t byte_size = word_size * BytesPerWord; - - size_t raw_bytes_size = MAX2(byte_size, sizeof(Metablock)); - raw_bytes_size = align_up(raw_bytes_size, Metachunk::object_alignment()); - - size_t raw_word_size = raw_bytes_size / BytesPerWord; - assert(raw_word_size * BytesPerWord == raw_bytes_size, "Size problem"); - - return raw_word_size; - } - - // Adds to the given statistic object. - void add_to_statistics(SpaceManagerStatistics* out) const; - - // Verify internal counters against the current state. - DEBUG_ONLY(void verify_metrics() const;) - -}; - - -} // namespace metaspace - -#endif // SHARE_MEMORY_METASPACE_SPACEMANAGER_HPP diff --git a/src/hotspot/share/memory/metaspace/virtualSpaceList.cpp b/src/hotspot/share/memory/metaspace/virtualSpaceList.cpp --- a/src/hotspot/share/memory/metaspace/virtualSpaceList.cpp +++ b/src/hotspot/share/memory/metaspace/virtualSpaceList.cpp @@ -1,5 +1,6 @@ /* - * Copyright (c) 2018, 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2018, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2018, 2020 SAP SE. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -22,426 +23,252 @@ * */ - #include "precompiled.hpp" #include "logging/log.hpp" -#include "logging/logStream.hpp" #include "memory/metaspace.hpp" #include "memory/metaspace/chunkManager.hpp" -#include "memory/metaspace/metachunk.hpp" -#include "memory/metaspace/metaspaceCommon.hpp" +#include "memory/metaspace/counter.hpp" +#include "memory/metaspace/commitLimiter.hpp" +#include "memory/metaspace/counter.hpp" +#include "memory/metaspace/freeChunkList.hpp" +#include "memory/metaspace/metaspaceContext.hpp" #include "memory/metaspace/virtualSpaceList.hpp" #include "memory/metaspace/virtualSpaceNode.hpp" -#include "runtime/atomic.hpp" -#include "runtime/orderAccess.hpp" #include "runtime/mutexLocker.hpp" -#include "runtime/safepoint.hpp" + namespace metaspace { +#define LOGFMT "VsList @" PTR_FORMAT " (%s)" +#define LOGFMT_ARGS p2i(this), this->_name + +// Create a new, empty, expandable list. +VirtualSpaceList::VirtualSpaceList(const char* name, CommitLimiter* commit_limiter) + : _name(name), + _first_node(NULL), + _can_expand(true), + _can_purge(true), + _commit_limiter(commit_limiter), + _reserved_words_counter(), + _committed_words_counter() +{ +} + +// Create a new list. The list will contain one node only, which uses the given ReservedSpace. +// It will be not expandable beyond that first node. +VirtualSpaceList::VirtualSpaceList(const char* name, ReservedSpace rs, CommitLimiter* commit_limiter) +: _name(name), + _first_node(NULL), + _can_expand(false), + _can_purge(false), + _commit_limiter(commit_limiter), + _reserved_words_counter(), + _committed_words_counter() +{ + // Create the first node spanning the existing ReservedSpace. This will be the only node created + // for this list since we cannot expand. + VirtualSpaceNode* vsn = VirtualSpaceNode::create_node(rs, _commit_limiter, + &_reserved_words_counter, &_committed_words_counter); + assert(vsn != NULL, "node creation failed"); + _first_node = vsn; + _first_node->set_next(NULL); + _nodes_counter.increment(); +} VirtualSpaceList::~VirtualSpaceList() { - VirtualSpaceListIterator iter(virtual_space_list()); - while (iter.repeat()) { - VirtualSpaceNode* vsl = iter.get_next(); - delete vsl; + assert_lock_strong(MetaspaceExpand_lock); + // Note: normally, there is no reason ever to delete a vslist since they are + // global objects, but for gtests it makes sense to allow this. + VirtualSpaceNode* vsn = _first_node; + VirtualSpaceNode* vsn2 = vsn; + while (vsn != NULL) { + vsn2 = vsn->next(); + delete vsn; + vsn = vsn2; } } -void VirtualSpaceList::inc_reserved_words(size_t v) { +// Create a new node and append it to the list. After +// this function, _current_node shall point to a new empty node. +// List must be expandable for this to work. +void VirtualSpaceList::create_new_node() { + assert(_can_expand, "List is not expandable"); assert_lock_strong(MetaspaceExpand_lock); - _reserved_words = _reserved_words + v; -} -void VirtualSpaceList::dec_reserved_words(size_t v) { - assert_lock_strong(MetaspaceExpand_lock); - _reserved_words = _reserved_words - v; + + VirtualSpaceNode* vsn = VirtualSpaceNode::create_node(Settings::virtual_space_node_default_word_size(), + _commit_limiter, + &_reserved_words_counter, &_committed_words_counter); + assert(vsn != NULL, "node creation failed"); + vsn->set_next(_first_node); + _first_node = vsn; + _nodes_counter.increment(); } -#define assert_committed_below_limit() \ - assert(MetaspaceUtils::committed_bytes() <= MaxMetaspaceSize, \ - "Too much committed memory. Committed: " SIZE_FORMAT \ - " limit (MaxMetaspaceSize): " SIZE_FORMAT, \ - MetaspaceUtils::committed_bytes(), MaxMetaspaceSize); +// Allocate a root chunk from this list. +// Note: this just returns a chunk whose memory is reserved; no memory is committed yet. +// Hence, before using this chunk, it must be committed. +// Also, no limits are checked, since no committing takes place. +Metachunk* VirtualSpaceList::allocate_root_chunk() { + assert_lock_strong(MetaspaceExpand_lock); -void VirtualSpaceList::inc_committed_words(size_t v) { - assert_lock_strong(MetaspaceExpand_lock); - _committed_words = _committed_words + v; + if (_first_node == NULL || + _first_node->free_words() == 0) { - assert_committed_below_limit(); -} -void VirtualSpaceList::dec_committed_words(size_t v) { - assert_lock_strong(MetaspaceExpand_lock); - _committed_words = _committed_words - v; + // Since all allocations from a VirtualSpaceNode happen in + // root-chunk-size units, and the node size must be root-chunk-size aligned, + // we should never have left-over space. + assert(_first_node == NULL || + _first_node->free_words() == 0, "Sanity"); - assert_committed_below_limit(); -} - -void VirtualSpaceList::inc_virtual_space_count() { - assert_lock_strong(MetaspaceExpand_lock); - _virtual_space_count++; -} - -void VirtualSpaceList::dec_virtual_space_count() { - assert_lock_strong(MetaspaceExpand_lock); - _virtual_space_count--; -} - -// Walk the list of VirtualSpaceNodes and delete -// nodes with a 0 container_count. Remove Metachunks in -// the node from their respective freelists. -void VirtualSpaceList::purge(ChunkManager* chunk_manager) { - assert_lock_strong(MetaspaceExpand_lock); - // Don't use a VirtualSpaceListIterator because this - // list is being changed and a straightforward use of an iterator is not safe. - VirtualSpaceNode* prev_vsl = virtual_space_list(); - VirtualSpaceNode* next_vsl = prev_vsl; - int num_purged_nodes = 0; - while (next_vsl != NULL) { - VirtualSpaceNode* vsl = next_vsl; - DEBUG_ONLY(vsl->verify(false);) - next_vsl = vsl->next(); - // Don't free the current virtual space since it will likely - // be needed soon. - if (vsl->container_count() == 0 && vsl != current_virtual_space()) { - log_trace(gc, metaspace, freelist)("Purging VirtualSpaceNode " PTR_FORMAT " (capacity: " SIZE_FORMAT - ", used: " SIZE_FORMAT ").", p2i(vsl), vsl->capacity_words_in_vs(), vsl->used_words_in_vs()); - DEBUG_ONLY(Atomic::inc(&g_internal_statistics.num_vsnodes_purged)); - // Unlink it from the list - if (prev_vsl == vsl) { - // This is the case of the current node being the first node. - assert(vsl == virtual_space_list(), "Expected to be the first node"); - set_virtual_space_list(vsl->next()); - } else { - prev_vsl->set_next(vsl->next()); - } - - vsl->purge(chunk_manager); - dec_reserved_words(vsl->reserved_words()); - dec_committed_words(vsl->committed_words()); - dec_virtual_space_count(); - delete vsl; - num_purged_nodes ++; + if (_can_expand) { + create_new_node(); + UL2(debug, "added new node (now: %d).", num_nodes()); } else { - prev_vsl = vsl; + UL(debug, "list cannot expand."); + return NULL; // We cannot expand this list. } } - // Verify list -#ifdef ASSERT - if (num_purged_nodes > 0) { - verify(false); - } -#endif + Metachunk* c = _first_node->allocate_root_chunk(); + + assert(c != NULL, "This should have worked"); + + return c; + } +// Attempts to purge nodes. This will remove and delete nodes which only contain free chunks. +// The free chunks are removed from the freelists before the nodes are deleted. +// Return number of purged nodes. +int VirtualSpaceList::purge(FreeChunkListVector* freelists) { -// This function looks at the mmap regions in the metaspace without locking. -// The chunks are added with store ordering and not deleted except for at -// unloading time during a safepoint. -VirtualSpaceNode* VirtualSpaceList::find_enclosing_space(const void* ptr) { - // List should be stable enough to use an iterator here because removing virtual - // space nodes is only allowed at a safepoint. - if (is_within_envelope((address)ptr)) { - VirtualSpaceListIterator iter(virtual_space_list()); - while (iter.repeat()) { - VirtualSpaceNode* vsn = iter.get_next(); - if (vsn->contains(ptr)) { - return vsn; + assert_lock_strong(MetaspaceExpand_lock); + + if (_can_purge == false) { + return 0; + } + + UL(debug, "purging."); + + VirtualSpaceNode* vsn = _first_node; + VirtualSpaceNode* prev_vsn = NULL; + int num = 0, num_purged = 0; + while (vsn != NULL) { + VirtualSpaceNode* next_vsn = vsn->next(); + bool purged = vsn->attempt_purge(freelists); + if (purged) { + // Note: from now on do not dereference vsn! + UL2(debug, "purged node @" PTR_FORMAT ".", p2i(vsn)); + if (_first_node == vsn) { + _first_node = next_vsn; } + DEBUG_ONLY(vsn = (VirtualSpaceNode*)((uintptr_t)(0xdeadbeef));) + if (prev_vsn != NULL) { + prev_vsn->set_next(next_vsn); + } + num_purged ++; + _nodes_counter.decrement(); + } else { + prev_vsn = vsn; } + vsn = next_vsn; + num ++; } - return NULL; + + UL2(debug, "purged %d nodes (now: %d)", num_purged, num_nodes()); + + return num_purged; + } -void VirtualSpaceList::retire_current_virtual_space() { +// Print all nodes in this space list. +void VirtualSpaceList::print_on(outputStream* st) const { + MutexLocker fcl(MetaspaceExpand_lock, Mutex::_no_safepoint_check_flag); + + st->print_cr("vsl %s:", _name); + const VirtualSpaceNode* vsn = _first_node; + int n = 0; + while (vsn != NULL) { + st->print("- node #%d: ", n); + vsn->print_on(st); + vsn = vsn->next(); + n ++; + } + st->print_cr("- total %d nodes, " SIZE_FORMAT " reserved words, " SIZE_FORMAT " committed words.", + n, reserved_words(), committed_words()); +} + +#ifdef ASSERT +void VirtualSpaceList::verify_locked(bool slow) const { + assert_lock_strong(MetaspaceExpand_lock); - VirtualSpaceNode* vsn = current_virtual_space(); + assert(_name != NULL, "Sanity"); - ChunkManager* cm = is_class() ? Metaspace::chunk_manager_class() : - Metaspace::chunk_manager_metadata(); + int n = 0; - vsn->retire(cm); -} + if (_first_node != NULL) { -VirtualSpaceList::VirtualSpaceList(size_t word_size) : - _virtual_space_list(NULL), - _current_virtual_space(NULL), - _is_class(false), - _reserved_words(0), - _committed_words(0), - _virtual_space_count(0), - _envelope_lo((address)max_uintx), - _envelope_hi(NULL) { - MutexLocker cl(MetaspaceExpand_lock, Mutex::_no_safepoint_check_flag); - create_new_virtual_space(word_size); -} + size_t total_reserved_words = 0; + size_t total_committed_words = 0; + const VirtualSpaceNode* vsn = _first_node; + while (vsn != NULL) { + n ++; + vsn->verify_locked(slow); + total_reserved_words += vsn->word_size(); + total_committed_words += vsn->committed_words(); + vsn = vsn->next(); + } -VirtualSpaceList::VirtualSpaceList(ReservedSpace rs) : - _virtual_space_list(NULL), - _current_virtual_space(NULL), - _is_class(true), - _reserved_words(0), - _committed_words(0), - _virtual_space_count(0), - _envelope_lo((address)max_uintx), - _envelope_hi(NULL) { - MutexLocker cl(MetaspaceExpand_lock, Mutex::_no_safepoint_check_flag); - VirtualSpaceNode* class_entry = new VirtualSpaceNode(is_class(), rs); - bool succeeded = class_entry->initialize(); - if (succeeded) { - expand_envelope_to_include_node(class_entry); - // ensure lock-free iteration sees fully initialized node - OrderAccess::storestore(); - link_vs(class_entry); + _nodes_counter.check(n); + _reserved_words_counter.check(total_reserved_words); + _committed_words_counter.check(total_committed_words); + + } else { + + _reserved_words_counter.check(0); + _committed_words_counter.check(0); + } } -size_t VirtualSpaceList::free_bytes() { - return current_virtual_space()->free_words_in_vs() * BytesPerWord; +void VirtualSpaceList::verify(bool slow) const { + MutexLocker fcl(MetaspaceExpand_lock, Mutex::_no_safepoint_check_flag); + verify_locked(slow); } +#endif -// Allocate another meta virtual space and add it to the list. -bool VirtualSpaceList::create_new_virtual_space(size_t vs_word_size) { - assert_lock_strong(MetaspaceExpand_lock); - - if (is_class()) { - assert(false, "We currently don't support more than one VirtualSpace for" - " the compressed class space. The initialization of the" - " CCS uses another code path and should not hit this path."); - return false; - } - - if (vs_word_size == 0) { - assert(false, "vs_word_size should always be at least _reserve_alignment large."); - return false; - } - - // Reserve the space - size_t vs_byte_size = vs_word_size * BytesPerWord; - assert_is_aligned(vs_byte_size, Metaspace::reserve_alignment()); - - // Allocate the meta virtual space and initialize it. - VirtualSpaceNode* new_entry = new VirtualSpaceNode(is_class(), vs_byte_size); - if (!new_entry->initialize()) { - delete new_entry; - return false; - } else { - assert(new_entry->reserved_words() == vs_word_size, - "Reserved memory size differs from requested memory size"); - expand_envelope_to_include_node(new_entry); - // ensure lock-free iteration sees fully initialized node - OrderAccess::storestore(); - link_vs(new_entry); - DEBUG_ONLY(Atomic::inc(&g_internal_statistics.num_vsnodes_created)); - return true; - } - - DEBUG_ONLY(verify(false);) - -} - -void VirtualSpaceList::link_vs(VirtualSpaceNode* new_entry) { - if (virtual_space_list() == NULL) { - set_virtual_space_list(new_entry); - } else { - current_virtual_space()->set_next(new_entry); - } - set_current_virtual_space(new_entry); - inc_reserved_words(new_entry->reserved_words()); - inc_committed_words(new_entry->committed_words()); - inc_virtual_space_count(); -#ifdef ASSERT - new_entry->mangle(); -#endif - LogTarget(Trace, gc, metaspace) lt; - if (lt.is_enabled()) { - LogStream ls(lt); - VirtualSpaceNode* vsl = current_virtual_space(); - ResourceMark rm; - vsl->print_on(&ls); - } -} - -bool VirtualSpaceList::expand_node_by(VirtualSpaceNode* node, - size_t min_words, - size_t preferred_words) { - size_t before = node->committed_words(); - - bool result = node->expand_by(min_words, preferred_words); - - size_t after = node->committed_words(); - - // after and before can be the same if the memory was pre-committed. - assert(after >= before, "Inconsistency"); - inc_committed_words(after - before); - - return result; -} - -bool VirtualSpaceList::expand_by(size_t min_words, size_t preferred_words) { - assert_is_aligned(min_words, Metaspace::commit_alignment_words()); - assert_is_aligned(preferred_words, Metaspace::commit_alignment_words()); - assert(min_words <= preferred_words, "Invalid arguments"); - - const char* const class_or_not = (is_class() ? "class" : "non-class"); - - if (!MetaspaceGC::can_expand(min_words, this->is_class())) { - log_trace(gc, metaspace, freelist)("Cannot expand %s virtual space list.", - class_or_not); - return false; - } - - size_t allowed_expansion_words = MetaspaceGC::allowed_expansion(); - if (allowed_expansion_words < min_words) { - log_trace(gc, metaspace, freelist)("Cannot expand %s virtual space list (must try gc first).", - class_or_not); - return false; - } - - size_t max_expansion_words = MIN2(preferred_words, allowed_expansion_words); - - // Commit more memory from the the current virtual space. - bool vs_expanded = expand_node_by(current_virtual_space(), - min_words, - max_expansion_words); - if (vs_expanded) { - log_trace(gc, metaspace, freelist)("Expanded %s virtual space list.", - class_or_not); - return true; - } - log_trace(gc, metaspace, freelist)("%s virtual space list: retire current node.", - class_or_not); - retire_current_virtual_space(); - - // Get another virtual space. - size_t grow_vs_words = MAX2((size_t)VirtualSpaceSize, preferred_words); - grow_vs_words = align_up(grow_vs_words, Metaspace::reserve_alignment_words()); - - if (create_new_virtual_space(grow_vs_words)) { - if (current_virtual_space()->is_pre_committed()) { - // The memory was pre-committed, so we are done here. - assert(min_words <= current_virtual_space()->committed_words(), - "The new VirtualSpace was pre-committed, so it" - "should be large enough to fit the alloc request."); +// Returns true if this pointer is contained in one of our nodes. +bool VirtualSpaceList::contains(const MetaWord* p) const { + const VirtualSpaceNode* vsn = _first_node; + while (vsn != NULL) { + if (vsn->contains(p)) { return true; } - - return expand_node_by(current_virtual_space(), - min_words, - max_expansion_words); + vsn = vsn->next(); } - return false; } -// Given a chunk, calculate the largest possible padding space which -// could be required when allocating it. -static size_t largest_possible_padding_size_for_chunk(size_t chunk_word_size, bool is_class) { - const ChunkIndex chunk_type = get_chunk_type_by_size(chunk_word_size, is_class); - if (chunk_type != HumongousIndex) { - // Normal, non-humongous chunks are allocated at chunk size - // boundaries, so the largest padding space required would be that - // minus the smallest chunk size. - const size_t smallest_chunk_size = is_class ? ClassSpecializedChunk : SpecializedChunk; - return chunk_word_size - smallest_chunk_size; - } else { - // Humongous chunks are allocated at smallest-chunksize - // boundaries, so there is no padding required. - return 0; +// Returns true if the vslist is not expandable and no more root chunks +// can be allocated. +bool VirtualSpaceList::is_full() const { + if (!_can_expand && _first_node != NULL && _first_node->free_words() == 0) { + return true; } + return false; } - -Metachunk* VirtualSpaceList::get_new_chunk(size_t chunk_word_size, size_t suggested_commit_granularity) { - - // Allocate a chunk out of the current virtual space. - Metachunk* next = current_virtual_space()->get_chunk_vs(chunk_word_size); - - if (next != NULL) { - return next; - } - - // The expand amount is currently only determined by the requested sizes - // and not how much committed memory is left in the current virtual space. - - // We must have enough space for the requested size and any - // additional reqired padding chunks. - const size_t size_for_padding = largest_possible_padding_size_for_chunk(chunk_word_size, this->is_class()); - - size_t min_word_size = align_up(chunk_word_size + size_for_padding, Metaspace::commit_alignment_words()); - size_t preferred_word_size = align_up(suggested_commit_granularity, Metaspace::commit_alignment_words()); - if (min_word_size >= preferred_word_size) { - // Can happen when humongous chunks are allocated. - preferred_word_size = min_word_size; - } - - bool expanded = expand_by(min_word_size, preferred_word_size); - if (expanded) { - next = current_virtual_space()->get_chunk_vs(chunk_word_size); - assert(next != NULL, "The allocation was expected to succeed after the expansion"); - } - - return next; +// Convenience methods to return the global class-space chunkmanager +// and non-class chunkmanager, respectively. +VirtualSpaceList* VirtualSpaceList::vslist_class() { + return MetaspaceContext::context_class() == NULL ? NULL : MetaspaceContext::context_class()->vslist(); } -void VirtualSpaceList::print_on(outputStream* st, size_t scale) const { - st->print_cr(SIZE_FORMAT " nodes, current node: " PTR_FORMAT, - _virtual_space_count, p2i(_current_virtual_space)); - VirtualSpaceListIterator iter(virtual_space_list()); - while (iter.repeat()) { - st->cr(); - VirtualSpaceNode* node = iter.get_next(); - node->print_on(st, scale); - } +VirtualSpaceList* VirtualSpaceList::vslist_nonclass() { + return MetaspaceContext::context_nonclass() == NULL ? NULL : MetaspaceContext::context_nonclass()->vslist(); } -void VirtualSpaceList::print_map(outputStream* st) const { - VirtualSpaceNode* list = virtual_space_list(); - VirtualSpaceListIterator iter(list); - unsigned i = 0; - while (iter.repeat()) { - st->print_cr("Node %u:", i); - VirtualSpaceNode* node = iter.get_next(); - node->print_map(st, this->is_class()); - i ++; - } -} -// Given a node, expand range such that it includes the node. -void VirtualSpaceList::expand_envelope_to_include_node(const VirtualSpaceNode* node) { - _envelope_lo = MIN2(_envelope_lo, (address)node->low_boundary()); - _envelope_hi = MAX2(_envelope_hi, (address)node->high_boundary()); -} - - -#ifdef ASSERT -void VirtualSpaceList::verify(bool slow) { - VirtualSpaceNode* list = virtual_space_list(); - VirtualSpaceListIterator iter(list); - size_t reserved = 0; - size_t committed = 0; - size_t node_count = 0; - while (iter.repeat()) { - VirtualSpaceNode* node = iter.get_next(); - if (slow) { - node->verify(true); - } - // Check that the node resides fully within our envelope. - assert((address)node->low_boundary() >= _envelope_lo && (address)node->high_boundary() <= _envelope_hi, - "Node " SIZE_FORMAT " [" PTR_FORMAT ", " PTR_FORMAT ") outside envelope [" PTR_FORMAT ", " PTR_FORMAT ").", - node_count, p2i(node->low_boundary()), p2i(node->high_boundary()), p2i(_envelope_lo), p2i(_envelope_hi)); - reserved += node->reserved_words(); - committed += node->committed_words(); - node_count ++; - } - assert(reserved == reserved_words() && committed == committed_words() && node_count == _virtual_space_count, - "Mismatch: reserved real: " SIZE_FORMAT " expected: " SIZE_FORMAT - ", committed real: " SIZE_FORMAT " expected: " SIZE_FORMAT - ", node count real: " SIZE_FORMAT " expected: " SIZE_FORMAT ".", - reserved, reserved_words(), committed, committed_words(), - node_count, _virtual_space_count); -} -#endif // ASSERT } // namespace metaspace diff --git a/src/hotspot/share/memory/metaspace/virtualSpaceList.hpp b/src/hotspot/share/memory/metaspace/virtualSpaceList.hpp --- a/src/hotspot/share/memory/metaspace/virtualSpaceList.hpp +++ b/src/hotspot/share/memory/metaspace/virtualSpaceList.hpp @@ -1,5 +1,6 @@ /* - * Copyright (c) 2018, 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2018, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2018, 2020 SAP SE. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -26,143 +27,128 @@ #define SHARE_MEMORY_METASPACE_VIRTUALSPACELIST_HPP #include "memory/allocation.hpp" +#include "memory/metaspace/counter.hpp" +#include "memory/metaspace/commitLimiter.hpp" #include "memory/metaspace/virtualSpaceNode.hpp" +#include "memory/virtualspace.hpp" #include "utilities/globalDefinitions.hpp" +class outputStream; namespace metaspace { class Metachunk; -class ChunkManager; +class FreeChunkListVector; -// List of VirtualSpaces for metadata allocation. + +// VirtualSpaceList manages a single (if its non-expandable) or +// a series of (if its expandable) virtual memory regions used +// for metaspace. +// +// Internally it holds a list of nodes (VirtualSpaceNode) each +// managing a single contiguous memory region. The first node of +// this list is the current node and used for allocation of new +// root chunks. +// +// Beyond access to those nodes and the ability to grow new nodes +// (if expandable) it allows for purging: purging this list means +// removing and unmapping all memory regions which are unused. + class VirtualSpaceList : public CHeapObj { - friend class VirtualSpaceNode; - enum VirtualSpaceSizes { - VirtualSpaceSize = 256 * K - }; + // Name + const char* const _name; - // Head of the list - VirtualSpaceNode* _virtual_space_list; - // virtual space currently being used for allocations - VirtualSpaceNode* _current_virtual_space; + // Head of the list. + VirtualSpaceNode* _first_node; - // Is this VirtualSpaceList used for the compressed class space - bool _is_class; + // Number of nodes (kept for statistics only). + IntCounter _nodes_counter; - // Sum of reserved and committed memory in the virtual spaces - size_t _reserved_words; - size_t _committed_words; + // Whether this list can expand by allocating new nodes. + const bool _can_expand; - // Number of virtual spaces - size_t _virtual_space_count; + // Whether this list can be purged. + const bool _can_purge; - // Optimization: we keep an address range to quickly exclude pointers - // which are clearly not pointing into metaspace. This is an optimization for - // VirtualSpaceList::contains(). - address _envelope_lo; - address _envelope_hi; + // Used to check limits before committing memory. + CommitLimiter* const _commit_limiter; - bool is_within_envelope(address p) const { - return p >= _envelope_lo && p < _envelope_hi; - } + // Statistics - // Given a node, expand range such that it includes the node. - void expand_envelope_to_include_node(const VirtualSpaceNode* node); + // Holds sum of reserved space, in words, over all list nodes. + SizeCounter _reserved_words_counter; - ~VirtualSpaceList(); + // Holds sum of committed space, in words, over all list nodes. + SizeCounter _committed_words_counter; - VirtualSpaceNode* virtual_space_list() const { return _virtual_space_list; } + // Create a new node and append it to the list. After + // this function, _current_node shall point to a new empty node. + // List must be expandable for this to work. + void create_new_node(); - void set_virtual_space_list(VirtualSpaceNode* v) { - _virtual_space_list = v; - } - void set_current_virtual_space(VirtualSpaceNode* v) { - _current_virtual_space = v; - } +public: - void link_vs(VirtualSpaceNode* new_entry); + // Create a new, empty, expandable list. + VirtualSpaceList(const char* name, CommitLimiter* commit_limiter); - // Get another virtual space and add it to the list. This - // is typically prompted by a failed attempt to allocate a chunk - // and is typically followed by the allocation of a chunk. - bool create_new_virtual_space(size_t vs_word_size); + // Create a new list. The list will contain one node only, which uses the given ReservedSpace. + // It will be not expandable beyond that first node. + VirtualSpaceList(const char* name, ReservedSpace rs, CommitLimiter* commit_limiter); - // Chunk up the unused committed space in the current - // virtual space and add the chunks to the free list. - void retire_current_virtual_space(); + virtual ~VirtualSpaceList(); - DEBUG_ONLY(bool contains_node(const VirtualSpaceNode* node) const;) + // Allocate a root chunk from this list. + // Note: this just returns a chunk whose memory is reserved; no memory is committed yet. + // Hence, before using this chunk, it must be committed. + // May return NULL if vslist would need to be expanded to hold the new root node but + // the list cannot be expanded (in practice this means we reached CompressedClassSpaceSize). + Metachunk* allocate_root_chunk(); - public: - VirtualSpaceList(size_t word_size); - VirtualSpaceList(ReservedSpace rs); + // Attempts to purge nodes. This will remove and delete nodes which only contain free chunks. + // The free chunks are removed from the freelists before the nodes are deleted. + // Return number of purged nodes. + int purge(FreeChunkListVector* freelists); - size_t free_bytes(); + //// Statistics //// - Metachunk* get_new_chunk(size_t chunk_word_size, - size_t suggested_commit_granularity); + // Return sum of reserved words in all nodes. + size_t reserved_words() const { return _reserved_words_counter.get(); } - bool expand_node_by(VirtualSpaceNode* node, - size_t min_words, - size_t preferred_words); + // Return sum of committed words in all nodes. + size_t committed_words() const { return _committed_words_counter.get(); } - bool expand_by(size_t min_words, - size_t preferred_words); + // Return number of nodes in this list. + int num_nodes() const { return _nodes_counter.get(); } - VirtualSpaceNode* current_virtual_space() { - return _current_virtual_space; - } + //// Debug stuff //// + DEBUG_ONLY(void verify(bool slow) const;) + DEBUG_ONLY(void verify_locked(bool slow) const;) - bool is_class() const { return _is_class; } + // Print all nodes in this space list. + void print_on(outputStream* st) const; - bool initialization_succeeded() { return _virtual_space_list != NULL; } + // Returns true if this pointer is contained in one of our nodes. + bool contains(const MetaWord* p) const; - size_t reserved_words() { return _reserved_words; } - size_t reserved_bytes() { return reserved_words() * BytesPerWord; } - size_t committed_words() { return _committed_words; } - size_t committed_bytes() { return committed_words() * BytesPerWord; } + // Returns true if the list is not expandable and no more root chunks + // can be allocated. + bool is_full() const; - void inc_reserved_words(size_t v); - void dec_reserved_words(size_t v); - void inc_committed_words(size_t v); - void dec_committed_words(size_t v); - void inc_virtual_space_count(); - void dec_virtual_space_count(); + // Convenience methods to return the global class-space vslist + // and non-class vslist, respectively. + static VirtualSpaceList* vslist_class(); + static VirtualSpaceList* vslist_nonclass(); - VirtualSpaceNode* find_enclosing_space(const void* ptr); - bool contains(const void* ptr) { return find_enclosing_space(ptr) != NULL; } + // These exist purely to print limits of the compressed class space; + // if we ever change the ccs to not use a degenerated-list-of-one-node this + // will go away. + MetaWord* base_of_first_node() const { return _first_node != NULL ? _first_node->base() : NULL; } + size_t word_size_of_first_node() const { return _first_node != NULL ? _first_node->word_size() : 0; } - // Unlink empty VirtualSpaceNodes and free it. - void purge(ChunkManager* chunk_manager); - - void print_on(outputStream* st) const { print_on(st, K); } - void print_on(outputStream* st, size_t scale) const; - void print_map(outputStream* st) const; - - DEBUG_ONLY(void verify(bool slow);) - - class VirtualSpaceListIterator : public StackObj { - VirtualSpaceNode* _virtual_spaces; - public: - VirtualSpaceListIterator(VirtualSpaceNode* virtual_spaces) : - _virtual_spaces(virtual_spaces) {} - - bool repeat() { - return _virtual_spaces != NULL; - } - - VirtualSpaceNode* get_next() { - VirtualSpaceNode* result = _virtual_spaces; - if (_virtual_spaces != NULL) { - _virtual_spaces = _virtual_spaces->next(); - } - return result; - } - }; }; } // namespace metaspace #endif // SHARE_MEMORY_METASPACE_VIRTUALSPACELIST_HPP + diff --git a/src/hotspot/share/memory/metaspace/virtualSpaceNode.cpp b/src/hotspot/share/memory/metaspace/virtualSpaceNode.cpp --- a/src/hotspot/share/memory/metaspace/virtualSpaceNode.cpp +++ b/src/hotspot/share/memory/metaspace/virtualSpaceNode.cpp @@ -1,5 +1,6 @@ /* - * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2018, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2018, 2020 SAP SE. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -25,564 +26,495 @@ #include "precompiled.hpp" #include "logging/log.hpp" -#include "logging/logStream.hpp" + +#include "memory/metaspace/chunkLevel.hpp" +#include "memory/metaspace/chunkHeaderPool.hpp" +#include "memory/metaspace/commitLimiter.hpp" +#include "memory/metaspace/counter.hpp" +#include "memory/metaspace/freeChunkList.hpp" +#include "memory/metaspace/internStat.hpp" #include "memory/metaspace/metachunk.hpp" +#include "memory/metaspace/metaspaceCommon.hpp" +#include "memory/metaspace/rootChunkArea.hpp" +#include "memory/metaspace/runningCounters.hpp" +#include "memory/metaspace/settings.hpp" +#include "memory/metaspace/virtualSpaceNode.hpp" #include "memory/metaspace.hpp" -#include "memory/metaspace/chunkManager.hpp" -#include "memory/metaspace/metaDebug.hpp" -#include "memory/metaspace/metaspaceCommon.hpp" -#include "memory/metaspace/occupancyMap.hpp" -#include "memory/metaspace/virtualSpaceNode.hpp" -#include "memory/virtualspace.hpp" -#include "runtime/atomic.hpp" + +#include "runtime/globals.hpp" +#include "runtime/mutexLocker.hpp" #include "runtime/os.hpp" -#include "services/memTracker.hpp" -#include "utilities/copy.hpp" + +#include "utilities/align.hpp" #include "utilities/debug.hpp" #include "utilities/globalDefinitions.hpp" +#include "utilities/ostream.hpp" namespace metaspace { -// Decide if large pages should be committed when the memory is reserved. -static bool should_commit_large_pages_when_reserving(size_t bytes) { - if (UseLargePages && UseLargePagesInMetaspace && !os::can_commit_large_page_memory()) { - size_t words = bytes / BytesPerWord; - bool is_class = false; // We never reserve large pages for the class space. - if (MetaspaceGC::can_expand(words, is_class) && - MetaspaceGC::allowed_expansion() >= words) { - return true; +#define LOGFMT "VsListNode @" PTR_FORMAT " base " PTR_FORMAT " " +#define LOGFMT_ARGS p2i(this), p2i(_base) + +#ifdef ASSERT +void check_pointer_is_aligned_to_commit_granule(const MetaWord* p) { + assert(is_aligned(p, Settings::commit_granule_bytes()), + "Pointer not aligned to commit granule size: " PTR_FORMAT ".", + p2i(p)); +} +void check_word_size_is_aligned_to_commit_granule(size_t word_size) { + assert(is_aligned(word_size, Settings::commit_granule_words()), + "Not aligned to commit granule size: " SIZE_FORMAT ".", word_size); +} +#endif + + +// Given an address range, ensure it is committed. +// +// The range has to be aligned to granule size. +// +// Function will: +// - check how many granules in that region are uncommitted; If all are committed, it +// returns true immediately. +// - check if committing those uncommitted granules would bring us over the commit limit +// (GC threshold, MaxMetaspaceSize). If true, it returns false. +// - commit the memory. +// - mark the range as committed in the commit mask +// +// Returns true if success, false if it did hit a commit limit. +bool VirtualSpaceNode::commit_range(MetaWord* p, size_t word_size) { + + DEBUG_ONLY(check_pointer_is_aligned_to_commit_granule(p);) + DEBUG_ONLY(check_word_size_is_aligned_to_commit_granule(word_size);) + assert_lock_strong(MetaspaceExpand_lock); + + // First calculate how large the committed regions in this range are + const size_t committed_words_in_range = _commit_mask.get_committed_size_in_range(p, word_size); + DEBUG_ONLY(check_word_size_is_aligned_to_commit_granule(committed_words_in_range);) + + // By how much words we would increase commit charge + // were we to commit the given address range completely. + const size_t commit_increase_words = word_size - committed_words_in_range; + + UL2(debug, "committing range " PTR_FORMAT ".." PTR_FORMAT "(" SIZE_FORMAT " words)", + p2i(p), p2i(p + word_size), word_size); + + if (commit_increase_words == 0) { + UL(debug, "... already fully committed."); + return true; // Already fully committed, nothing to do. + } + + // Before committing any more memory, check limits. + if (_commit_limiter->possible_expansion_words() < commit_increase_words) { + UL(debug, "... cannot commit (limit)."); + return false; + } + + // Commit... + if (os::commit_memory((char*)p, word_size * BytesPerWord, false) == false) { + vm_exit_out_of_memory(word_size * BytesPerWord, OOM_MMAP_ERROR, "Failed to commit metaspace."); + } + + if (AlwaysPreTouch) { + os::pretouch_memory(p, p + word_size); + } + + UL2(debug, "... committed " SIZE_FORMAT " additional words.", commit_increase_words); + + // ... tell commit limiter... + _commit_limiter->increase_committed(commit_increase_words); + + // ... update counters in containing vslist ... + _total_committed_words_counter->increment_by(commit_increase_words); + + // ... and update the commit mask. + _commit_mask.mark_range_as_committed(p, word_size); + +#ifdef ASSERT + // The commit boundary maintained in the CommitLimiter should be equal the sum of committed words + // in both class and non-class vslist (outside gtests). + if (_commit_limiter == CommitLimiter::globalLimiter()) { + assert(_commit_limiter->committed_words() == RunningCounters::committed_words(), "counter mismatch"); + } +#endif + + InternalStats::inc_num_space_committed(); + + return true; + +} + +// Given an address range, ensure it is committed. +// +// The range does not have to be aligned to granule size. However, the function will always commit +// whole granules. +// +// Function will: +// - check how many granules in that region are uncommitted; If all are committed, it +// returns true immediately. +// - check if committing those uncommitted granules would bring us over the commit limit +// (GC threshold, MaxMetaspaceSize). If true, it returns false. +// - commit the memory. +// - mark the range as committed in the commit mask +// +// !! Careful: +// calling ensure_range_is_committed on a range which contains both committed and uncommitted +// areas will commit the whole area, thus erase the content in the existing committed parts. +// Make sure you never call this on an address range containing live data. !! +// +// Returns true if success, false if it did hit a commit limit. +bool VirtualSpaceNode::ensure_range_is_committed(MetaWord* p, size_t word_size) { + + assert_lock_strong(MetaspaceExpand_lock); + assert(p != NULL && word_size > 0, "Sanity"); + + MetaWord* p_start = align_down(p, Settings::commit_granule_bytes()); + MetaWord* p_end = align_up(p + word_size, Settings::commit_granule_bytes()); + + // Todo: simple for now. Make it more intelligent late + return commit_range(p_start, p_end - p_start); + +} + +// Given an address range (which has to be aligned to commit granule size): +// - uncommit it +// - mark it as uncommitted in the commit mask +void VirtualSpaceNode::uncommit_range(MetaWord* p, size_t word_size) { + + DEBUG_ONLY(check_pointer_is_aligned_to_commit_granule(p);) + DEBUG_ONLY(check_word_size_is_aligned_to_commit_granule(word_size);) + assert_lock_strong(MetaspaceExpand_lock); + + // First calculate how large the committed regions in this range are + const size_t committed_words_in_range = _commit_mask.get_committed_size_in_range(p, word_size); + DEBUG_ONLY(check_word_size_is_aligned_to_commit_granule(committed_words_in_range);) + + UL2(debug, "uncommitting range " PTR_FORMAT ".." PTR_FORMAT "(" SIZE_FORMAT " words)", + p2i(p), p2i(p + word_size), word_size); + + if (committed_words_in_range == 0) { + UL(debug, "... already fully uncommitted."); + return; // Already fully uncommitted, nothing to do. + } + + // Uncommit... + if (os::uncommit_memory((char*)p, word_size * BytesPerWord) == false) { + // Note: this can actually happen, since uncommit may increase the number of mappings. + fatal("Failed to uncommit metaspace."); + } + + UL2(debug, "... uncommitted " SIZE_FORMAT " words.", committed_words_in_range); + + // ... tell commit limiter... + _commit_limiter->decrease_committed(committed_words_in_range); + + // ... and global counters... + _total_committed_words_counter->decrement_by(committed_words_in_range); + + // ... and update the commit mask. + _commit_mask.mark_range_as_uncommitted(p, word_size); + +#ifdef ASSERT + // The commit boundary maintained in the CommitLimiter should be equal the sum of committed words + // in both class and non-class vslist (outside gtests). + if (_commit_limiter == CommitLimiter::globalLimiter()) { // We are outside a test scenario + assert(_commit_limiter->committed_words() == RunningCounters::committed_words(), "counter mismatch"); + } +#endif + + InternalStats::inc_num_space_uncommitted(); + +} + +//// creation, destruction //// + +VirtualSpaceNode::VirtualSpaceNode(ReservedSpace rs, bool owns_rs, CommitLimiter* limiter, + SizeCounter* reserve_counter, SizeCounter* commit_counter) + : _next(NULL), + _rs(rs), + _owns_rs(owns_rs), + _base((MetaWord*)rs.base()), + _word_size(rs.size() / BytesPerWord), + _used_words(0), + _commit_mask((MetaWord*)rs.base(), rs.size() / BytesPerWord), + _root_chunk_area_lut((MetaWord*)rs.base(), rs.size() / BytesPerWord), + _commit_limiter(limiter), + _total_reserved_words_counter(reserve_counter), + _total_committed_words_counter(commit_counter) +{ + UL2(debug, "born (word_size " SIZE_FORMAT ").", _word_size); + + // Update reserved counter in vslist + _total_reserved_words_counter->increment_by(_word_size); + + assert_is_aligned(_base, chunklevel::MAX_CHUNK_BYTE_SIZE); + assert_is_aligned(_word_size, chunklevel::MAX_CHUNK_WORD_SIZE); + +} + + +// Create a node of a given size (it will create its own space). +VirtualSpaceNode* VirtualSpaceNode::create_node(size_t word_size, + CommitLimiter* limiter, SizeCounter* reserve_words_counter, + SizeCounter* commit_words_counter) +{ + + DEBUG_ONLY(assert_is_aligned(word_size, chunklevel::MAX_CHUNK_WORD_SIZE);) + + ReservedSpace rs(word_size * BytesPerWord, + Settings::virtual_space_node_reserve_alignment_words() * BytesPerWord, + false // large + ); + + if (!rs.is_reserved()) { + vm_exit_out_of_memory(word_size * BytesPerWord, OOM_MMAP_ERROR, "Failed to reserve memory for metaspace"); + } + + assert_is_aligned(rs.base(), chunklevel::MAX_CHUNK_BYTE_SIZE); + + InternalStats::inc_num_vsnodes_births(); + return new VirtualSpaceNode(rs, true, limiter, reserve_words_counter, commit_words_counter); + +} + +// Create a node over an existing space +VirtualSpaceNode* VirtualSpaceNode::create_node(ReservedSpace rs, CommitLimiter* limiter, + SizeCounter* reserve_words_counter, SizeCounter* commit_words_counter) +{ + InternalStats::inc_num_vsnodes_births(); + return new VirtualSpaceNode(rs, false, limiter, reserve_words_counter, commit_words_counter); +} + +VirtualSpaceNode::~VirtualSpaceNode() { + + DEBUG_ONLY(verify_locked(true);) + + UL(debug, ": dies."); + + if (_owns_rs) { + _rs.release(); + } + + // Update counters in vslist + size_t committed = committed_words(); + _total_committed_words_counter->decrement_by(committed); + _total_reserved_words_counter->decrement_by(_word_size); + + // ... and tell commit limiter + _commit_limiter->decrease_committed(committed); + + InternalStats::inc_num_vsnodes_deaths(); + +} + +//// Chunk allocation, splitting, merging ///// + +// Allocate a root chunk from this node. Will fail and return NULL if the node is full +// - if we used up the whole address space of this node's memory region. +// (in case this node backs compressed class space, this is how we hit +// CompressedClassSpaceSize). +// Note that this just returns reserved memory; caller must take care of committing this +// chunk before using it. +Metachunk* VirtualSpaceNode::allocate_root_chunk() { + + assert_lock_strong(MetaspaceExpand_lock); + + assert_is_aligned(free_words(), chunklevel::MAX_CHUNK_WORD_SIZE); + + if (free_words() >= chunklevel::MAX_CHUNK_WORD_SIZE) { + + MetaWord* loc = _base + _used_words; + _used_words += chunklevel::MAX_CHUNK_WORD_SIZE; + + RootChunkArea* rca = _root_chunk_area_lut.get_area_by_address(loc); + + // Create a root chunk header and initialize it; + Metachunk* c = rca->alloc_root_chunk_header(this); + + assert(c->base() == loc && c->vsnode() == this && + c->is_free(), "Sanity"); + + DEBUG_ONLY(c->verify(true);) + + UL2(debug, "new root chunk " METACHUNK_FORMAT ".", METACHUNK_FORMAT_ARGS(c)); + + return c; + + } + + return NULL; // Node is full. + +} + +// Given a chunk c, split it recursively until you get a chunk of the given target_level. +// +// The resulting target chunk resides at the same address as the original chunk. +// The resulting splinters are added to freelists. +void VirtualSpaceNode::split(chunklevel_t target_level, Metachunk* c, FreeChunkListVector* freelists) { + + assert_lock_strong(MetaspaceExpand_lock); + + // Get the area associated with this chunk and let it handle the splitting + RootChunkArea* rca = _root_chunk_area_lut.get_area_by_address(c->base()); + + DEBUG_ONLY(rca->verify_area_is_ideally_merged();) + + rca->split(target_level, c, freelists); + +} + +// Given a chunk, attempt to merge it recursively with its neighboring chunks. +// +// If successful (merged at least once), returns address of +// the merged chunk; NULL otherwise. +// +// The merged chunks are removed from the freelists. +// +// !!! Please note that if this method returns a non-NULL value, the +// original chunk will be invalid and should not be accessed anymore! !!! +Metachunk* VirtualSpaceNode::merge(Metachunk* c, FreeChunkListVector* freelists) { + + assert(c != NULL && c->is_free(), "Sanity"); + assert_lock_strong(MetaspaceExpand_lock); + + // Get the rca associated with this chunk and let it handle the merging + RootChunkArea* rca = _root_chunk_area_lut.get_area_by_address(c->base()); + + Metachunk* c2 = rca->merge(c, freelists); + + DEBUG_ONLY(rca->verify_area_is_ideally_merged();) + + return c2; + +} + +// Given a chunk c, which must be "in use" and must not be a root chunk, attempt to +// enlarge it in place by claiming its trailing buddy. +// +// This will only work if c is the leader of the buddy pair and the trailing buddy is free. +// +// If successful, the follower chunk will be removed from the freelists, the leader chunk c will +// double in size (level decreased by one). +// +// On success, true is returned, false otherwise. +bool VirtualSpaceNode::attempt_enlarge_chunk(Metachunk* c, FreeChunkListVector* freelists) { + + assert(c != NULL && c->is_in_use() && !c->is_root_chunk(), "Sanity"); + assert_lock_strong(MetaspaceExpand_lock); + + // Get the rca associated with this chunk and let it handle the merging + RootChunkArea* rca = _root_chunk_area_lut.get_area_by_address(c->base()); + + bool rc = rca->attempt_enlarge_chunk(c, freelists); + + DEBUG_ONLY(rca->verify_area_is_ideally_merged();) + + if (rc) { + InternalStats::inc_num_chunks_enlarged(); + } + + return rc; + +} + +// Attempts to purge the node: +// +// If all chunks living in this node are free, they will all be removed from +// the freelist they currently reside in. Then, the node will be deleted. +// +// Returns true if the node has been deleted, false if not. +// !! If this returns true, do not access the node from this point on. !! +bool VirtualSpaceNode::attempt_purge(FreeChunkListVector* freelists) { + + assert_lock_strong(MetaspaceExpand_lock); + + if (!_owns_rs) { + // We do not allow purging of nodes if we do not own the + // underlying ReservedSpace (CompressClassSpace case). + return false; + } + + // First find out if all areas are empty. Since empty chunks collapse to root chunk + // size, if all chunks in this node are free root chunks we are good to go. + if (!_root_chunk_area_lut.is_free()) { + return false; + } + + UL(debug, ": purging."); + + // Okay, we can purge. Before we can do this, we need to remove all chunks from the freelist. + for (int narea = 0; narea < _root_chunk_area_lut.number_of_areas(); narea ++) { + RootChunkArea* ra = _root_chunk_area_lut.get_area_by_index(narea); + Metachunk* c = ra->first_chunk(); + if (c != NULL) { + UL2(trace, "removing chunk from to-be-purged node: " + METACHUNK_FULL_FORMAT ".", METACHUNK_FULL_FORMAT_ARGS(c)); + assert(c->is_free() && c->is_root_chunk(), "Sanity"); + freelists->remove(c); } } - return false; + // Now, delete the node, then right away return since this object is invalid. + delete this; + + return true; + } -// byte_size is the size of the associated virtualspace. -VirtualSpaceNode::VirtualSpaceNode(bool is_class, size_t bytes) : - _next(NULL), _is_class(is_class), _rs(), _top(NULL), _container_count(0), _occupancy_map(NULL) { - assert_is_aligned(bytes, Metaspace::reserve_alignment()); - bool large_pages = should_commit_large_pages_when_reserving(bytes); - _rs = ReservedSpace(bytes, Metaspace::reserve_alignment(), large_pages); - if (_rs.is_reserved()) { - assert(_rs.base() != NULL, "Catch if we get a NULL address"); - assert(_rs.size() != 0, "Catch if we get a 0 size"); - assert_is_aligned(_rs.base(), Metaspace::reserve_alignment()); - assert_is_aligned(_rs.size(), Metaspace::reserve_alignment()); +void VirtualSpaceNode::print_on(outputStream* st) const { - MemTracker::record_virtual_memory_type((address)_rs.base(), mtClass); - } + size_t scale = K; + + st->print("base " PTR_FORMAT ": ", p2i(base())); + st->print("reserved="); + print_scaled_words(st, word_size(), scale); + st->print(", committed="); + print_scaled_words_and_percentage(st, committed_words(), word_size(), scale); + st->print(", used="); + print_scaled_words_and_percentage(st, used_words(), word_size(), scale); + + st->cr(); + _root_chunk_area_lut.print_on(st); + _commit_mask.print_on(st); + } -void VirtualSpaceNode::purge(ChunkManager* chunk_manager) { - // When a node is purged, lets give it a thorough examination. - DEBUG_ONLY(verify(true);) - Metachunk* chunk = first_chunk(); - Metachunk* invalid_chunk = (Metachunk*) top(); - while (chunk < invalid_chunk ) { - assert(chunk->is_tagged_free(), "Should be tagged free"); - MetaWord* next = ((MetaWord*)chunk) + chunk->word_size(); - chunk_manager->remove_chunk(chunk); - chunk->remove_sentinel(); - assert(chunk->next() == NULL && - chunk->prev() == NULL, - "Was not removed from its list"); - chunk = (Metachunk*) next; - } -} - -void VirtualSpaceNode::print_map(outputStream* st, bool is_class) const { - - if (bottom() == top()) { - return; - } - - const size_t spec_chunk_size = is_class ? ClassSpecializedChunk : SpecializedChunk; - const size_t small_chunk_size = is_class ? ClassSmallChunk : SmallChunk; - const size_t med_chunk_size = is_class ? ClassMediumChunk : MediumChunk; - - int line_len = 100; - const size_t section_len = align_up(spec_chunk_size * line_len, med_chunk_size); - line_len = (int)(section_len / spec_chunk_size); - - static const int NUM_LINES = 4; - - char* lines[NUM_LINES]; - for (int i = 0; i < NUM_LINES; i ++) { - lines[i] = (char*)os::malloc(line_len, mtInternal); - } - int pos = 0; - const MetaWord* p = bottom(); - const Metachunk* chunk = (const Metachunk*)p; - const MetaWord* chunk_end = p + chunk->word_size(); - while (p < top()) { - if (pos == line_len) { - pos = 0; - for (int i = 0; i < NUM_LINES; i ++) { - st->fill_to(22); - st->print_raw(lines[i], line_len); - st->cr(); - } - } - if (pos == 0) { - st->print(PTR_FORMAT ":", p2i(p)); - } - if (p == chunk_end) { - chunk = (Metachunk*)p; - chunk_end = p + chunk->word_size(); - } - // line 1: chunk starting points (a dot if that area is a chunk start). - lines[0][pos] = p == (const MetaWord*)chunk ? '.' : ' '; - - // Line 2: chunk type (x=spec, s=small, m=medium, h=humongous), uppercase if - // chunk is in use. - const bool chunk_is_free = ((Metachunk*)chunk)->is_tagged_free(); - if (chunk->word_size() == spec_chunk_size) { - lines[1][pos] = chunk_is_free ? 'x' : 'X'; - } else if (chunk->word_size() == small_chunk_size) { - lines[1][pos] = chunk_is_free ? 's' : 'S'; - } else if (chunk->word_size() == med_chunk_size) { - lines[1][pos] = chunk_is_free ? 'm' : 'M'; - } else if (chunk->word_size() > med_chunk_size) { - lines[1][pos] = chunk_is_free ? 'h' : 'H'; - } else { - ShouldNotReachHere(); - } - - // Line 3: chunk origin - const ChunkOrigin origin = chunk->get_origin(); - lines[2][pos] = origin == origin_normal ? ' ' : '0' + (int) origin; - - // Line 4: Virgin chunk? Virgin chunks are chunks created as a byproduct of padding or splitting, - // but were never used. - lines[3][pos] = chunk->get_use_count() > 0 ? ' ' : 'v'; - - p += spec_chunk_size; - pos ++; - } - if (pos > 0) { - for (int i = 0; i < NUM_LINES; i ++) { - st->fill_to(22); - st->print_raw(lines[i], line_len); - st->cr(); - } - } - for (int i = 0; i < NUM_LINES; i ++) { - os::free(lines[i]); - } -} - - -#ifdef ASSERT - -// Verify counters, all chunks in this list node and the occupancy map. -void VirtualSpaceNode::verify(bool slow) { - log_trace(gc, metaspace, freelist)("verifying %s virtual space node (%s).", - (is_class() ? "class space" : "metaspace"), (slow ? "slow" : "quick")); - // Fast mode: just verify chunk counters and basic geometry - // Slow mode: verify chunks and occupancy map - uintx num_in_use_chunks = 0; - Metachunk* chunk = first_chunk(); - Metachunk* invalid_chunk = (Metachunk*) top(); - - // Iterate the chunks in this node and verify each chunk. - while (chunk < invalid_chunk ) { - if (slow) { - do_verify_chunk(chunk); - } - if (!chunk->is_tagged_free()) { - num_in_use_chunks ++; - } - const size_t s = chunk->word_size(); - // Prevent endless loop on invalid chunk size. - assert(is_valid_chunksize(is_class(), s), "Invalid chunk size: " SIZE_FORMAT ".", s); - MetaWord* next = ((MetaWord*)chunk) + s; - chunk = (Metachunk*) next; - } - assert(_container_count == num_in_use_chunks, "Container count mismatch (real: " UINTX_FORMAT - ", counter: " UINTX_FORMAT ".", num_in_use_chunks, _container_count); - // Also verify the occupancy map. - if (slow) { - occupancy_map()->verify(bottom(), top()); - } -} - -// Verify that all free chunks in this node are ideally merged -// (there not should be multiple small chunks where a large chunk could exist.) -void VirtualSpaceNode::verify_free_chunks_are_ideally_merged() { - Metachunk* chunk = first_chunk(); - Metachunk* invalid_chunk = (Metachunk*) top(); - // Shorthands. - const size_t size_med = (is_class() ? ClassMediumChunk : MediumChunk) * BytesPerWord; - const size_t size_small = (is_class() ? ClassSmallChunk : SmallChunk) * BytesPerWord; - int num_free_chunks_since_last_med_boundary = -1; - int num_free_chunks_since_last_small_boundary = -1; - bool error = false; - char err[256]; - while (!error && chunk < invalid_chunk ) { - // Test for missed chunk merge opportunities: count number of free chunks since last chunk boundary. - // Reset the counter when encountering a non-free chunk. - if (chunk->get_chunk_type() != HumongousIndex) { - if (chunk->is_tagged_free()) { - // Count successive free, non-humongous chunks. - if (is_aligned(chunk, size_small)) { - if (num_free_chunks_since_last_small_boundary > 0) { - error = true; - jio_snprintf(err, sizeof(err), "Missed chunk merge opportunity to merge a small chunk preceding " PTR_FORMAT ".", p2i(chunk)); - } else { - num_free_chunks_since_last_small_boundary = 0; - } - } else if (num_free_chunks_since_last_small_boundary != -1) { - num_free_chunks_since_last_small_boundary ++; - } - if (is_aligned(chunk, size_med)) { - if (num_free_chunks_since_last_med_boundary > 0) { - error = true; - jio_snprintf(err, sizeof(err), "Missed chunk merge opportunity to merge a medium chunk preceding " PTR_FORMAT ".", p2i(chunk)); - } else { - num_free_chunks_since_last_med_boundary = 0; - } - } else if (num_free_chunks_since_last_med_boundary != -1) { - num_free_chunks_since_last_med_boundary ++; - } - } else { - // Encountering a non-free chunk, reset counters. - num_free_chunks_since_last_med_boundary = -1; - num_free_chunks_since_last_small_boundary = -1; - } - } else { - // One cannot merge areas with a humongous chunk in the middle. Reset counters. - num_free_chunks_since_last_med_boundary = -1; - num_free_chunks_since_last_small_boundary = -1; - } - - if (error) { - print_map(tty, is_class()); - fatal("%s", err); - } - - MetaWord* next = ((MetaWord*)chunk) + chunk->word_size(); - chunk = (Metachunk*) next; - } -} -#endif // ASSERT - -void VirtualSpaceNode::inc_container_count() { - assert_lock_strong(MetaspaceExpand_lock); - _container_count++; -} - -void VirtualSpaceNode::dec_container_count() { - assert_lock_strong(MetaspaceExpand_lock); - _container_count--; -} - -VirtualSpaceNode::~VirtualSpaceNode() { - _rs.release(); - if (_occupancy_map != NULL) { - delete _occupancy_map; - } -#ifdef ASSERT - size_t word_size = sizeof(*this) / BytesPerWord; - Copy::fill_to_words((HeapWord*) this, word_size, 0xf1f1f1f1); -#endif -} - -size_t VirtualSpaceNode::used_words_in_vs() const { - return pointer_delta(top(), bottom(), sizeof(MetaWord)); -} - -// Space committed in the VirtualSpace -size_t VirtualSpaceNode::capacity_words_in_vs() const { - return pointer_delta(end(), bottom(), sizeof(MetaWord)); -} - -size_t VirtualSpaceNode::free_words_in_vs() const { - return pointer_delta(end(), top(), sizeof(MetaWord)); -} - -// Given an address larger than top(), allocate padding chunks until top is at the given address. -void VirtualSpaceNode::allocate_padding_chunks_until_top_is_at(MetaWord* target_top) { - - assert(target_top > top(), "Sanity"); - - // Padding chunks are added to the freelist. - ChunkManager* const chunk_manager = Metaspace::get_chunk_manager(is_class()); - - // shorthands - const size_t spec_word_size = chunk_manager->specialized_chunk_word_size(); - const size_t small_word_size = chunk_manager->small_chunk_word_size(); - const size_t med_word_size = chunk_manager->medium_chunk_word_size(); - - while (top() < target_top) { - - // We could make this coding more generic, but right now we only deal with two possible chunk sizes - // for padding chunks, so it is not worth it. - size_t padding_chunk_word_size = small_word_size; - if (is_aligned(top(), small_word_size * sizeof(MetaWord)) == false) { - assert_is_aligned(top(), spec_word_size * sizeof(MetaWord)); // Should always hold true. - padding_chunk_word_size = spec_word_size; - } - MetaWord* here = top(); - assert_is_aligned(here, padding_chunk_word_size * sizeof(MetaWord)); - inc_top(padding_chunk_word_size); - - // Create new padding chunk. - ChunkIndex padding_chunk_type = get_chunk_type_by_size(padding_chunk_word_size, is_class()); - assert(padding_chunk_type == SpecializedIndex || padding_chunk_type == SmallIndex, "sanity"); - - Metachunk* const padding_chunk = - ::new (here) Metachunk(padding_chunk_type, is_class(), padding_chunk_word_size, this); - assert(padding_chunk == (Metachunk*)here, "Sanity"); - DEBUG_ONLY(padding_chunk->set_origin(origin_pad);) - log_trace(gc, metaspace, freelist)("Created padding chunk in %s at " - PTR_FORMAT ", size " SIZE_FORMAT_HEX ".", - (is_class() ? "class space " : "metaspace"), - p2i(padding_chunk), padding_chunk->word_size() * sizeof(MetaWord)); - - // Mark chunk start in occupancy map. - occupancy_map()->set_chunk_starts_at_address((MetaWord*)padding_chunk, true); - - // Chunks are born as in-use (see MetaChunk ctor). So, before returning - // the padding chunk to its chunk manager, mark it as in use (ChunkManager - // will assert that). - do_update_in_use_info_for_chunk(padding_chunk, true); - - // Return Chunk to freelist. - inc_container_count(); - chunk_manager->return_single_chunk(padding_chunk); - // Please note: at this point, ChunkManager::return_single_chunk() - // may already have merged the padding chunk with neighboring chunks, so - // it may have vanished at this point. Do not reference the padding - // chunk beyond this point. - } - - assert(top() == target_top, "Sanity"); - -} // allocate_padding_chunks_until_top_is_at() - -// Allocates the chunk from the virtual space only. -// This interface is also used internally for debugging. Not all -// chunks removed here are necessarily used for allocation. -Metachunk* VirtualSpaceNode::take_from_committed(size_t chunk_word_size) { - // Non-humongous chunks are to be allocated aligned to their chunk - // size. So, start addresses of medium chunks are aligned to medium - // chunk size, those of small chunks to small chunk size and so - // forth. This facilitates merging of free chunks and reduces - // fragmentation. Chunk sizes are spec < small < medium, with each - // larger chunk size being a multiple of the next smaller chunk - // size. - // Because of this alignment, me may need to create a number of padding - // chunks. These chunks are created and added to the freelist. - - // The chunk manager to which we will give our padding chunks. - ChunkManager* const chunk_manager = Metaspace::get_chunk_manager(is_class()); - - // shorthands - const size_t spec_word_size = chunk_manager->specialized_chunk_word_size(); - const size_t small_word_size = chunk_manager->small_chunk_word_size(); - const size_t med_word_size = chunk_manager->medium_chunk_word_size(); - - assert(chunk_word_size == spec_word_size || chunk_word_size == small_word_size || - chunk_word_size >= med_word_size, "Invalid chunk size requested."); - - // Chunk alignment (in bytes) == chunk size unless humongous. - // Humongous chunks are aligned to the smallest chunk size (spec). - const size_t required_chunk_alignment = (chunk_word_size > med_word_size ? - spec_word_size : chunk_word_size) * sizeof(MetaWord); - - // Do we have enough space to create the requested chunk plus - // any padding chunks needed? - MetaWord* const next_aligned = - static_cast(align_up(top(), required_chunk_alignment)); - if (!is_available((next_aligned - top()) + chunk_word_size)) { - return NULL; - } - - // Before allocating the requested chunk, allocate padding chunks if necessary. - // We only need to do this for small or medium chunks: specialized chunks are the - // smallest size, hence always aligned. Homungous chunks are allocated unaligned - // (implicitly, also aligned to smallest chunk size). - if ((chunk_word_size == med_word_size || chunk_word_size == small_word_size) && next_aligned > top()) { - log_trace(gc, metaspace, freelist)("Creating padding chunks in %s between %p and %p...", - (is_class() ? "class space " : "metaspace"), - top(), next_aligned); - allocate_padding_chunks_until_top_is_at(next_aligned); - // Now, top should be aligned correctly. - assert_is_aligned(top(), required_chunk_alignment); - } - - // Now, top should be aligned correctly. - assert_is_aligned(top(), required_chunk_alignment); - - // Bottom of the new chunk - MetaWord* chunk_limit = top(); - assert(chunk_limit != NULL, "Not safe to call this method"); - - // The virtual spaces are always expanded by the - // commit granularity to enforce the following condition. - // Without this the is_available check will not work correctly. - assert(_virtual_space.committed_size() == _virtual_space.actual_committed_size(), - "The committed memory doesn't match the expanded memory."); - - if (!is_available(chunk_word_size)) { - LogTarget(Trace, gc, metaspace, freelist) lt; - if (lt.is_enabled()) { - LogStream ls(lt); - ls.print("VirtualSpaceNode::take_from_committed() not available " SIZE_FORMAT " words ", chunk_word_size); - // Dump some information about the virtual space that is nearly full - print_on(&ls); - } - return NULL; - } - - // Take the space (bump top on the current virtual space). - inc_top(chunk_word_size); - - // Initialize the chunk - ChunkIndex chunk_type = get_chunk_type_by_size(chunk_word_size, is_class()); - Metachunk* result = ::new (chunk_limit) Metachunk(chunk_type, is_class(), chunk_word_size, this); - assert(result == (Metachunk*)chunk_limit, "Sanity"); - occupancy_map()->set_chunk_starts_at_address((MetaWord*)result, true); - do_update_in_use_info_for_chunk(result, true); - - inc_container_count(); - -#ifdef ASSERT - EVERY_NTH(VerifyMetaspaceInterval) - chunk_manager->locked_verify(true); - verify(true); - END_EVERY_NTH - do_verify_chunk(result); -#endif - - result->inc_use_count(); - - return result; -} - - -// Expand the virtual space (commit more of the reserved space) -bool VirtualSpaceNode::expand_by(size_t min_words, size_t preferred_words) { - size_t min_bytes = min_words * BytesPerWord; - size_t preferred_bytes = preferred_words * BytesPerWord; - - size_t uncommitted = virtual_space()->reserved_size() - virtual_space()->actual_committed_size(); - - if (uncommitted < min_bytes) { - return false; - } - - size_t commit = MIN2(preferred_bytes, uncommitted); - bool result = virtual_space()->expand_by(commit, false); - - if (result) { - log_trace(gc, metaspace, freelist)("Expanded %s virtual space list node by " SIZE_FORMAT " words.", - (is_class() ? "class" : "non-class"), commit); - DEBUG_ONLY(Atomic::inc(&g_internal_statistics.num_committed_space_expanded)); - } else { - log_trace(gc, metaspace, freelist)("Failed to expand %s virtual space list node by " SIZE_FORMAT " words.", - (is_class() ? "class" : "non-class"), commit); - } - - assert(result, "Failed to commit memory"); - - return result; -} - -Metachunk* VirtualSpaceNode::get_chunk_vs(size_t chunk_word_size) { - assert_lock_strong(MetaspaceExpand_lock); - Metachunk* result = take_from_committed(chunk_word_size); - return result; -} - -bool VirtualSpaceNode::initialize() { - - if (!_rs.is_reserved()) { - return false; - } - - // These are necessary restriction to make sure that the virtual space always - // grows in steps of Metaspace::commit_alignment(). If both base and size are - // aligned only the middle alignment of the VirtualSpace is used. - assert_is_aligned(_rs.base(), Metaspace::commit_alignment()); - assert_is_aligned(_rs.size(), Metaspace::commit_alignment()); - - // ReservedSpaces marked as special will have the entire memory - // pre-committed. Setting a committed size will make sure that - // committed_size and actual_committed_size agrees. - size_t pre_committed_size = _rs.special() ? _rs.size() : 0; - - bool result = virtual_space()->initialize_with_granularity(_rs, pre_committed_size, - Metaspace::commit_alignment()); - if (result) { - assert(virtual_space()->committed_size() == virtual_space()->actual_committed_size(), - "Checking that the pre-committed memory was registered by the VirtualSpace"); - - set_top((MetaWord*)virtual_space()->low()); - } - - // Initialize Occupancy Map. - const size_t smallest_chunk_size = is_class() ? ClassSpecializedChunk : SpecializedChunk; - _occupancy_map = new OccupancyMap(bottom(), reserved_words(), smallest_chunk_size); - - return result; -} - -void VirtualSpaceNode::print_on(outputStream* st, size_t scale) const { - size_t used_words = used_words_in_vs(); - size_t commit_words = committed_words(); - size_t res_words = reserved_words(); - VirtualSpace* vs = virtual_space(); - - st->print("node @" PTR_FORMAT ": ", p2i(this)); - st->print("reserved="); - print_scaled_words(st, res_words, scale); - st->print(", committed="); - print_scaled_words_and_percentage(st, commit_words, res_words, scale); - st->print(", used="); - print_scaled_words_and_percentage(st, used_words, res_words, scale); - st->cr(); - st->print(" [" PTR_FORMAT ", " PTR_FORMAT ", " - PTR_FORMAT ", " PTR_FORMAT ")", - p2i(bottom()), p2i(top()), p2i(end()), - p2i(vs->high_boundary())); +// Returns size, in words, of committed space in this node alone. +// Note: iterates over commit mask and hence may be a tad expensive on large nodes. +size_t VirtualSpaceNode::committed_words() const { + return _commit_mask.get_committed_size(); } #ifdef ASSERT -void VirtualSpaceNode::mangle() { - size_t word_size = capacity_words_in_vs(); - Copy::fill_to_words((HeapWord*) low(), word_size, 0xf1f1f1f1); -} -#endif // ASSERT - -void VirtualSpaceNode::retire(ChunkManager* chunk_manager) { - assert(is_class() == chunk_manager->is_class(), "Wrong ChunkManager?"); -#ifdef ASSERT - verify(false); - EVERY_NTH(VerifyMetaspaceInterval) - verify(true); - END_EVERY_NTH -#endif - for (int i = (int)MediumIndex; i >= (int)ZeroIndex; --i) { - ChunkIndex index = (ChunkIndex)i; - size_t chunk_size = chunk_manager->size_by_index(index); - - while (free_words_in_vs() >= chunk_size) { - Metachunk* chunk = get_chunk_vs(chunk_size); - // Chunk will be allocated aligned, so allocation may require - // additional padding chunks. That may cause above allocation to - // fail. Just ignore the failed allocation and continue with the - // next smaller chunk size. As the VirtualSpaceNode comitted - // size should be a multiple of the smallest chunk size, we - // should always be able to fill the VirtualSpace completely. - if (chunk == NULL) { - break; - } - chunk_manager->return_single_chunk(chunk); - } - } - assert(free_words_in_vs() == 0, "should be empty now"); +void VirtualSpaceNode::verify(bool slow) const { + MutexLocker fcl(MetaspaceExpand_lock, Mutex::_no_safepoint_check_flag); + verify_locked(slow); } +// Verify counters and basic structure. Slow mode: verify all chunks in depth +void VirtualSpaceNode::verify_locked(bool slow) const { + + assert_lock_strong(MetaspaceExpand_lock); + + assert(base() != NULL, "Invalid base"); + assert(base() == (MetaWord*)_rs.base() && + word_size() == _rs.size() / BytesPerWord, + "Sanity"); + assert_is_aligned(base(), chunklevel::MAX_CHUNK_BYTE_SIZE); + assert(used_words() <= word_size(), "Sanity"); + + // Since we only ever hand out root chunks from a vsnode, top should always be aligned + // to root chunk size. + assert_is_aligned(used_words(), chunklevel::MAX_CHUNK_WORD_SIZE); + + _commit_mask.verify(slow); + assert(committed_words() <= word_size(), "Sanity"); + assert_is_aligned(committed_words(), Settings::commit_granule_words()); + _root_chunk_area_lut.verify(slow); + +} + +#endif + + } // namespace metaspace diff --git a/src/hotspot/share/memory/metaspace/virtualSpaceNode.hpp b/src/hotspot/share/memory/metaspace/virtualSpaceNode.hpp --- a/src/hotspot/share/memory/metaspace/virtualSpaceNode.hpp +++ b/src/hotspot/share/memory/metaspace/virtualSpaceNode.hpp @@ -1,5 +1,6 @@ /* - * Copyright (c) 2018, 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2018, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2018, 2020 SAP SE. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -25,139 +26,268 @@ #ifndef SHARE_MEMORY_METASPACE_VIRTUALSPACENODE_HPP #define SHARE_MEMORY_METASPACE_VIRTUALSPACENODE_HPP + +#include "memory/allocation.hpp" +#include "memory/metaspace/counter.hpp" +#include "memory/metaspace/commitMask.hpp" +#include "memory/metaspace/rootChunkArea.hpp" +#include "memory/metaspace/settings.hpp" +#include "memory/memRegion.hpp" #include "memory/virtualspace.hpp" -#include "memory/memRegion.hpp" #include "utilities/debug.hpp" +#include "utilities/bitMap.hpp" #include "utilities/globalDefinitions.hpp" + class outputStream; namespace metaspace { -class Metachunk; -class ChunkManager; -class OccupancyMap; +class CommitLimiter; +class FreeChunkListVector; -// A VirtualSpaceList node. +// VirtualSpaceNode manages a single contiguous address range of metaspace. Logically that memory +// region is split up into a sequence of "root chunk areas", each one containing one root chunk +// or splinters of a root chunk. +// +// The underlying memory is also logically divided into a number of "commit granules", units of memory +// which may be committed or uncommitted independently from each other. +// +// (Both root chunk areas and commit granules have not much to do with each other - one is a way to +// reserve memory for the upper regions, see ChunkManager. One is a way to manage commited memory.) +// +// VirtualSpaceNode: +// - exposes a function to allocate a new root chunk (see VirtualSpaceNode::allocate_root_chunk()). +// +// - knows about the commit state of the memory region - which commit granule are committed, which +// are not. It exposes functions to commit and uncommit regions (without actively committing +// itself) +// +// - It has a reference to a "CommitLimiter", an interface to query whether committing is +// possible. That interface hides the various ways committing may be limited (GC threshold, +// MaxMetaspaceSize, ...) +// +// - It uses ReservedSpace to reserve its memory. It either owns the ReservedSpace or that +// space got handed in from outside (ccs). +// +// +// +// +// | root chunk area | root chunk area | root chunk area | <-- root chunk areas +// +// +-----------------------------------------------------------------------------------------------+ +// | | +// | `VirtualSpaceNode` memory | +// | | +// +-----------------------------------------------------------------------------------------------+ +// +// |x| |x|x|x| | | | |x|x|x| | | |x|x| | | |x|x|x|x| | | | | | | | |x| | | |x|x|x|x| | | |x| | | |x| <-- commit granules +// +// (x = committed) +// + class VirtualSpaceNode : public CHeapObj { - friend class VirtualSpaceList; // Link to next VirtualSpaceNode VirtualSpaceNode* _next; - // Whether this node is contained in class or metaspace. - const bool _is_class; + // The underlying space. This has been either created by this node + // and is owned by it, or has been handed in from outside (e.g. in + // case of CompressedClassSpace). + ReservedSpace _rs; - // total in the VirtualSpace - ReservedSpace _rs; - VirtualSpace _virtual_space; - MetaWord* _top; - // count of chunks contained in this VirtualSpace - uintx _container_count; + // True if the node owns the reserved space, false if not. + const bool _owns_rs; - OccupancyMap* _occupancy_map; + // Start pointer of the area. + MetaWord* const _base; - // Convenience functions to access the _virtual_space - char* low() const { return virtual_space()->low(); } - char* high() const { return virtual_space()->high(); } - char* low_boundary() const { return virtual_space()->low_boundary(); } - char* high_boundary() const { return virtual_space()->high_boundary(); } + // Size, in words, of the whole node + const size_t _word_size; - // The first Metachunk will be allocated at the bottom of the - // VirtualSpace - Metachunk* first_chunk() { return (Metachunk*) bottom(); } + // Size, in words, of the range of this node which has been handed out in + // the form of root chunks. + size_t _used_words; - // Committed but unused space in the virtual space - size_t free_words_in_vs() const; + // The bitmap describing the commit state of the region: + // Each bit covers a region of 64K (see constants::commit_granule_size). + CommitMask _commit_mask; - // True if this node belongs to class metaspace. - bool is_class() const { return _is_class; } + // An array/lookup table of RootChunkArea objects. Each one describes a root chunk area. + RootChunkAreaLUT _root_chunk_area_lut; - // Helper function for take_from_committed: allocate padding chunks - // until top is at the given address. - void allocate_padding_chunks_until_top_is_at(MetaWord* target_top); + // Limiter object to ask before expanding the committed size of this node. + CommitLimiter* const _commit_limiter; - public: + // Points to outside size counters which we are to increase/decrease when we commit/uncommit + // space from this node. + SizeCounter* const _total_reserved_words_counter; + SizeCounter* const _total_committed_words_counter; - VirtualSpaceNode(bool is_class, size_t byte_size); - VirtualSpaceNode(bool is_class, ReservedSpace rs) : - _next(NULL), _is_class(is_class), _rs(rs), _top(NULL), _container_count(0), _occupancy_map(NULL) {} + /// committing, uncommitting /// + + // Given a pointer into this node, calculate the start of the commit granule + // the pointer points into. + MetaWord* calc_start_of_granule(MetaWord* p) const { + DEBUG_ONLY(check_pointer(p)); + return align_down(p, Settings::commit_granule_bytes()); + } + + // Given an address range, ensure it is committed. + // + // The range has to be aligned to granule size. + // + // Function will: + // - check how many granules in that region are uncommitted; If all are committed, it + // returns true immediately. + // - check if committing those uncommitted granules would bring us over the commit limit + // (GC threshold, MaxMetaspaceSize). If true, it returns false. + // - commit the memory. + // - mark the range as committed in the commit mask + // + // Returns true if success, false if it did hit a commit limit. + bool commit_range(MetaWord* p, size_t word_size); + + //// creation //// + + // Create a new empty node spanning the given given reserved space. + VirtualSpaceNode(ReservedSpace rs, bool owns_rs, CommitLimiter* limiter, + SizeCounter* reserve_counter, SizeCounter* commit_counter); + +public: + + // Create a node of a given size (it will create its own space). + static VirtualSpaceNode* create_node(size_t word_size, CommitLimiter* limiter, SizeCounter* reserve_words_counter, + SizeCounter* commit_words_counter); + + // Create a node over an existing space + static VirtualSpaceNode* create_node(ReservedSpace rs, CommitLimiter* limiter, SizeCounter* reserve_words_counter, + SizeCounter* commit_words_counter); + ~VirtualSpaceNode(); - // Convenience functions for logical bottom and (committed) end - MetaWord* bottom() const { return (MetaWord*) _virtual_space.low(); } - MetaWord* end() const { return (MetaWord*) _virtual_space.high(); } + // Note: public for gtests only, could be private. + MetaWord* base() const { return _base; } - const OccupancyMap* occupancy_map() const { return _occupancy_map; } - OccupancyMap* occupancy_map() { return _occupancy_map; } + // Reserved size of the whole node. + size_t word_size() const { return _word_size; } - bool contains(const void* ptr) { return ptr >= low() && ptr < high(); } + //// Chunk allocation, splitting, merging ///// - size_t reserved_words() const { return _virtual_space.reserved_size() / BytesPerWord; } - size_t committed_words() const { return _virtual_space.actual_committed_size() / BytesPerWord; } + // Allocate a root chunk from this node. Will fail and return NULL if the node is full + // - if we used up the whole address space of this node's memory region. + // (in case this node backs compressed class space, this is how we hit + // CompressedClassSpaceSize). + // Note that this just returns reserved memory; caller must take care of committing this + // chunk before using it. + Metachunk* allocate_root_chunk(); - bool is_pre_committed() const { return _virtual_space.special(); } + // Given a chunk c, split it recursively until you get a chunk of the given target_level. + // + // The resulting target chunk resides at the same address as the original chunk. + // The resulting splinters are added to freelists. + void split(chunklevel_t target_level, Metachunk* c, FreeChunkListVector* freelists); - // address of next available space in _virtual_space; - // Accessors - VirtualSpaceNode* next() { return _next; } - void set_next(VirtualSpaceNode* v) { _next = v; } + // Given a chunk, attempt to merge it recursively with its neighboring chunks. + // + // If successful (merged at least once), returns address of + // the merged chunk; NULL otherwise. + // + // The merged chunks are removed from the freelists. + // + // !!! Please note that if this method returns a non-NULL value, the + // original chunk will be invalid and should not be accessed anymore! !!! + Metachunk* merge(Metachunk* c, FreeChunkListVector* freelists); - void set_top(MetaWord* v) { _top = v; } + // Given a chunk c, which must be "in use" and must not be a root chunk, attempt to + // enlarge it in place by claiming its trailing buddy. + // + // This will only work if c is the leader of the buddy pair and the trailing buddy is free. + // + // If successful, the follower chunk will be removed from the freelists, the leader chunk c will + // double in size (level decreased by one). + // + // On success, true is returned, false otherwise. + bool attempt_enlarge_chunk(Metachunk* c, FreeChunkListVector* freelists); - // Accessors - VirtualSpace* virtual_space() const { return (VirtualSpace*) &_virtual_space; } + // Attempts to purge the node: + // + // If all chunks living in this node are free, they will all be removed from + // the freelist they currently reside in. Then, the node will be deleted. + // + // Returns true if the node has been deleted, false if not. + // !! If this returns true, do not access the node from this point on. !! + bool attempt_purge(FreeChunkListVector* freelists); - // Returns true if "word_size" is available in the VirtualSpace - bool is_available(size_t word_size) { return word_size <= pointer_delta(end(), _top, sizeof(MetaWord)); } + // Attempts to uncommit free areas according to the rules set in settings. + // Returns number of words uncommitted. + size_t uncommit_free_areas(); - MetaWord* top() const { return _top; } - void inc_top(size_t word_size) { _top += word_size; } + /// misc ///// - uintx container_count() { return _container_count; } - void inc_container_count(); - void dec_container_count(); + // Returns size, in words, of the used space in this node alone. + // (Notes: + // - This is the space handed out to the ChunkManager, so it is "used" from the viewpoint of this node, + // but not necessarily used for Metadata. + // - This may or may not be committed memory. + size_t used_words() const { return _used_words; } - // used and capacity in this single entry in the list - size_t used_words_in_vs() const; - size_t capacity_words_in_vs() const; + // Returns size, in words, of how much space is left in this node alone. + size_t free_words() const { return _word_size - _used_words; } - bool initialize(); + // Returns size, in words, of committed space in this node alone. + // Note: iterates over commit mask and hence may be a tad expensive on large nodes. + size_t committed_words() const; - // get space from the virtual space - Metachunk* take_from_committed(size_t chunk_word_size); + //// Committing/uncommitting memory ///// - // Allocate a chunk from the virtual space and return it. - Metachunk* get_chunk_vs(size_t chunk_word_size); + // Given an address range, ensure it is committed. + // + // The range does not have to be aligned to granule size. However, the function will always commit + // whole granules. + // + // Function will: + // - check how many granules in that region are uncommitted; If all are committed, it + // returns true immediately. + // - check if committing those uncommitted granules would bring us over the commit limit + // (GC threshold, MaxMetaspaceSize). If true, it returns false. + // - commit the memory. + // - mark the range as committed in the commit mask + // + // Returns true if success, false if it did hit a commit limit. + bool ensure_range_is_committed(MetaWord* p, size_t word_size); - // Expands the committed space by at least min_words words. - bool expand_by(size_t min_words, size_t preferred_words); + // Given an address range (which has to be aligned to commit granule size): + // - uncommit it + // - mark it as uncommitted in the commit mask + void uncommit_range(MetaWord* p, size_t word_size); - // In preparation for deleting this node, remove all the chunks - // in the node from any freelist. - void purge(ChunkManager* chunk_manager); + //// List stuff //// + VirtualSpaceNode* next() const { return _next; } + void set_next(VirtualSpaceNode* vsn) { _next = vsn; } - // If an allocation doesn't fit in the current node a new node is created. - // Allocate chunks out of the remaining committed space in this node - // to avoid wasting that memory. - // This always adds up because all the chunk sizes are multiples of - // the smallest chunk size. - void retire(ChunkManager* chunk_manager); - void print_on(outputStream* st) const { print_on(st, K); } - void print_on(outputStream* st, size_t scale) const; - void print_map(outputStream* st, bool is_class) const; + /// Debug stuff //// - // Debug support - DEBUG_ONLY(void mangle();) - // Verify counters and basic structure. Slow mode: verify all chunks in depth and occupancy map. - DEBUG_ONLY(void verify(bool slow);) - // Verify that all free chunks in this node are ideally merged - // (there should not be multiple small chunks where a large chunk could exist.) - DEBUG_ONLY(void verify_free_chunks_are_ideally_merged();) + // Print a description about this node. + void print_on(outputStream* st) const; + + // Verify counters and basic structure. Slow mode: verify all chunks in depth + bool contains(const MetaWord* p) const { + return p >= _base && p < _base + _used_words; + } + +#ifdef ASSERT + void check_pointer(const MetaWord* p) const { + assert(contains(p), "invalid pointer"); + } + void verify(bool slow) const; + void verify_locked(bool slow) const; +#endif }; + } // namespace metaspace #endif // SHARE_MEMORY_METASPACE_VIRTUALSPACENODE_HPP