--- old/src/hotspot/os_cpu/linux_x86/gc/z/zPhysicalMemoryBacking_linux_x86.cpp 2019-05-02 09:57:43.596853739 +0200 +++ new/src/hotspot/os_cpu/linux_x86/gc/z/zPhysicalMemoryBacking_linux_x86.cpp 2019-05-02 09:57:43.375846440 +0200 @@ -32,6 +32,7 @@ #include "gc/z/zPhysicalMemory.inline.hpp" #include "gc/z/zPhysicalMemoryBacking_linux_x86.hpp" #include "logging/log.hpp" +#include "runtime/init.hpp" #include "runtime/os.hpp" #include "utilities/align.hpp" #include "utilities/debug.hpp" @@ -40,7 +41,11 @@ #include #include +// // Support for building on older Linux systems +// + +// madvise(2) flags #ifndef MADV_HUGEPAGE #define MADV_HUGEPAGE 14 #endif @@ -48,22 +53,37 @@ // Proc file entry for max map mount #define ZFILENAME_PROC_MAX_MAP_COUNT "/proc/sys/vm/max_map_count" -ZPhysicalMemoryBacking::ZPhysicalMemoryBacking(size_t max_capacity) : - _manager(), - _file() { +bool ZPhysicalMemoryBacking::is_initialized() const { + return _file.is_initialized(); +} - if (!_file.is_initialized()) { +void ZPhysicalMemoryBacking::warn_available_space(size_t max) const { + // Note that the available space on a tmpfs or a hugetlbfs filesystem + // will be zero if no size limit was specified when it was mounted. + const size_t available = _file.available(); + if (available == 0) { + // No size limit set, skip check + log_info(gc, init)("Available space on backing filesystem: N/A"); return; } - // Check and warn if max map count is too low - check_max_map_count(max_capacity); + log_info(gc, init)("Available space on backing filesystem: " SIZE_FORMAT "M", available / M); - // Check and warn if available space on filesystem is too low - check_available_space_on_filesystem(max_capacity); + // Warn if the filesystem doesn't currently have enough space available to hold + // the max heap size. The max heap size will be capped if we later hit this limit + // when trying to expand the heap. + if (available < max) { + log_warning(gc)("***** WARNING! INCORRECT SYSTEM CONFIGURATION DETECTED! *****"); + log_warning(gc)("Not enough space available on the backing filesystem to hold the current max Java heap"); + log_warning(gc)("size (" SIZE_FORMAT "M). Please adjust the size of the backing filesystem accordingly " + "(available", max / M); + log_warning(gc)("space is currently " SIZE_FORMAT "M). Continuing execution with the current filesystem " + "size could", available / M); + log_warning(gc)("lead to a premature OutOfMemoryError being thrown, due to failure to map memory."); + } } -void ZPhysicalMemoryBacking::check_max_map_count(size_t max_capacity) const { +void ZPhysicalMemoryBacking::warn_max_map_count(size_t max) const { const char* const filename = ZFILENAME_PROC_MAX_MAP_COUNT; FILE* const file = fopen(filename, "r"); if (file == NULL) { @@ -86,62 +106,101 @@ // However, ZGC tends to create the most mappings and dominate the total count. // In the worst cases, ZGC will map each granule three times, i.e. once per heap view. // We speculate that we need another 20% to allow for non-ZGC subsystems to map memory. - const size_t required_max_map_count = (max_capacity / ZGranuleSize) * 3 * 1.2; + const size_t required_max_map_count = (max / ZGranuleSize) * 3 * 1.2; if (actual_max_map_count < required_max_map_count) { - log_warning(gc, init)("***** WARNING! INCORRECT SYSTEM CONFIGURATION DETECTED! *****"); - log_warning(gc, init)("The system limit on number of memory mappings per process might be too low " - "for the given"); - log_warning(gc, init)("max Java heap size (" SIZE_FORMAT "M). Please adjust %s to allow for at", - max_capacity / M, filename); - log_warning(gc, init)("least " SIZE_FORMAT " mappings (current limit is " SIZE_FORMAT "). Continuing " - "execution with the current", required_max_map_count, actual_max_map_count); - log_warning(gc, init)("limit could lead to a fatal error, due to failure to map memory."); + log_warning(gc)("***** WARNING! INCORRECT SYSTEM CONFIGURATION DETECTED! *****"); + log_warning(gc)("The system limit on number of memory mappings per process might be too low for the given"); + log_warning(gc)("max Java heap size (" SIZE_FORMAT "M). Please adjust %s to allow for at", + max / M, filename); + log_warning(gc)("least " SIZE_FORMAT " mappings (current limit is " SIZE_FORMAT "). Continuing execution " + "with the current", required_max_map_count, actual_max_map_count); + log_warning(gc)("limit could lead to a fatal error, due to failure to map memory."); } } -void ZPhysicalMemoryBacking::check_available_space_on_filesystem(size_t max_capacity) const { - // Note that the available space on a tmpfs or a hugetlbfs filesystem - // will be zero if no size limit was specified when it was mounted. - const size_t available = _file.available(); - if (available == 0) { - // No size limit set, skip check - log_info(gc, init)("Available space on backing filesystem: N/A"); - return; - } +void ZPhysicalMemoryBacking::warn_commit_limits(size_t max) const { + // Warn if available space is too low + warn_available_space(max); - log_info(gc, init)("Available space on backing filesystem: " SIZE_FORMAT "M", - available / M); + // Warn if max map count is too low + warn_max_map_count(max); +} - // Warn if the filesystem doesn't currently have enough space available to hold - // the max heap size. The max heap size will be capped if we later hit this limit - // when trying to expand the heap. - if (available < max_capacity) { - log_warning(gc, init)("***** WARNING! INCORRECT SYSTEM CONFIGURATION DETECTED! *****"); - log_warning(gc, init)("Not enough space available on the backing filesystem to hold the current " - "max Java heap"); - log_warning(gc, init)("size (" SIZE_FORMAT "M). Please adjust the size of the backing filesystem " - "accordingly (available", max_capacity / M); - log_warning(gc, init)("space is currently " SIZE_FORMAT "M). Continuing execution with the current " - "filesystem size could", available / M); - log_warning(gc, init)("lead to a premature OutOfMemoryError being thrown, due to failure to map " - "memory."); - } +bool ZPhysicalMemoryBacking::supports_uncommit() { + assert(!is_init_completed(), "Invalid state"); + assert(_file.size() >= ZGranuleSize, "Invalid size"); + + // Test if uncommit is supported by uncommitting and then re-committing a granule + return commit(uncommit(ZGranuleSize)) == ZGranuleSize; } -bool ZPhysicalMemoryBacking::is_initialized() const { - return _file.is_initialized(); +size_t ZPhysicalMemoryBacking::commit(size_t size) { + size_t committed = 0; + + // Fill holes in the backing file + while (committed < size) { + size_t allocated = 0; + const size_t remaining = size - committed; + const uintptr_t start = _uncommitted.alloc_from_front_at_most(remaining, &allocated); + if (start == UINTPTR_MAX) { + // No holes to commit + break; + } + + // Try commit hole + const size_t filled = _file.commit(start, allocated); + if (filled > 0) { + // Successful or partialy successful + _committed.free(start, filled); + committed += filled; + } + if (filled < allocated) { + // Failed or partialy failed + _uncommitted.free(start + filled, allocated - filled); + return committed; + } + } + + // Expand backing file + if (committed < size) { + const size_t remaining = size - committed; + const uintptr_t start = _file.size(); + const size_t expanded = _file.commit(start, remaining); + if (expanded > 0) { + // Successful or partialy successful + _committed.free(start, expanded); + committed += expanded; + } + } + + return committed; } -size_t ZPhysicalMemoryBacking::try_expand(size_t old_capacity, size_t new_capacity) { - assert(old_capacity < new_capacity, "Invalid old/new capacity"); +size_t ZPhysicalMemoryBacking::uncommit(size_t size) { + size_t uncommitted = 0; + + // Punch holes in backing file + while (uncommitted < size) { + size_t allocated = 0; + const size_t remaining = size - uncommitted; + const uintptr_t start = _committed.alloc_from_back_at_most(remaining, &allocated); + assert(start != UINTPTR_MAX, "Allocation should never fail"); - const size_t capacity = _file.try_expand(old_capacity, new_capacity - old_capacity, ZGranuleSize); - if (capacity > old_capacity) { - // Add expanded capacity to free list - _manager.free(old_capacity, capacity - old_capacity); + // Try punch hole + const size_t punched = _file.uncommit(start, allocated); + if (punched > 0) { + // Successful or partialy successful + _uncommitted.free(start, punched); + uncommitted += punched; + } + if (punched < allocated) { + // Failed or partialy failed + _committed.free(start + punched, allocated - punched); + return uncommitted; + } } - return capacity; + return uncommitted; } ZPhysicalMemory ZPhysicalMemoryBacking::alloc(size_t size) { @@ -151,7 +210,7 @@ // Allocate segments for (size_t allocated = 0; allocated < size; allocated += ZGranuleSize) { - const uintptr_t start = _manager.alloc_from_front(ZGranuleSize); + const uintptr_t start = _committed.alloc_from_front(ZGranuleSize); assert(start != UINTPTR_MAX, "Allocation should never fail"); pmem.add_segment(ZPhysicalMemorySegment(start, ZGranuleSize)); } @@ -159,13 +218,13 @@ return pmem; } -void ZPhysicalMemoryBacking::free(ZPhysicalMemory pmem) { +void ZPhysicalMemoryBacking::free(const ZPhysicalMemory& pmem) { const size_t nsegments = pmem.nsegments(); // Free segments for (size_t i = 0; i < nsegments; i++) { - const ZPhysicalMemorySegment segment = pmem.segment(i); - _manager.free(segment.start(), segment.size()); + const ZPhysicalMemorySegment& segment = pmem.segment(i); + _committed.free(segment.start(), segment.size()); } } @@ -178,10 +237,10 @@ } } -void ZPhysicalMemoryBacking::advise_view(uintptr_t addr, size_t size) const { - if (madvise((void*)addr, size, MADV_HUGEPAGE) == -1) { +void ZPhysicalMemoryBacking::advise_view(uintptr_t addr, size_t size, int advice) const { + if (madvise((void*)addr, size, advice) == -1) { ZErrno err; - log_error(gc)("Failed to advise use of transparent huge pages (%s)", err.to_string()); + log_error(gc)("Failed to advise on memory (advice %d, %s)", advice, err.to_string()); } } @@ -190,41 +249,42 @@ os::pretouch_memory((void*)addr, (void*)(addr + size), page_size); } -void ZPhysicalMemoryBacking::map_view(ZPhysicalMemory pmem, uintptr_t addr, bool pretouch) const { +void ZPhysicalMemoryBacking::map_view(const ZPhysicalMemory& pmem, uintptr_t addr, bool pretouch) const { const size_t nsegments = pmem.nsegments(); + size_t size = 0; // Map segments for (size_t i = 0; i < nsegments; i++) { - const ZPhysicalMemorySegment segment = pmem.segment(i); - const size_t size = segment.size(); - const void* const res = mmap((void*)addr, size, PROT_READ|PROT_WRITE, MAP_FIXED|MAP_SHARED, _file.fd(), segment.start()); + const ZPhysicalMemorySegment& segment = pmem.segment(i); + const uintptr_t segment_addr = addr + size; + const void* const res = mmap((void*)segment_addr, segment.size(), PROT_READ|PROT_WRITE, MAP_FIXED|MAP_SHARED, _file.fd(), segment.start()); if (res == MAP_FAILED) { ZErrno err; map_failed(err); } - // Advise on use of transparent huge pages before touching it - if (ZLargePages::is_transparent()) { - advise_view(addr, size); - } + size += segment.size(); + } - // NUMA interleave memory before touching it - ZNUMA::memory_interleave(addr, size); + // Advise on use of transparent huge pages before touching it + if (ZLargePages::is_transparent()) { + advise_view(addr, size, MADV_HUGEPAGE); + } - if (pretouch) { - pretouch_view(addr, size); - } + // NUMA interleave memory before touching it + ZNUMA::memory_interleave(addr, size); - addr += size; + // Pre-touch memory + if (pretouch) { + pretouch_view(addr, size); } } -void ZPhysicalMemoryBacking::unmap_view(ZPhysicalMemory pmem, uintptr_t addr) const { +void ZPhysicalMemoryBacking::unmap_view(const ZPhysicalMemory& pmem, uintptr_t addr) const { // Note that we must keep the address space reservation intact and just detach // the backing memory. For this reason we map a new anonymous, non-accessible // and non-reserved page over the mapping instead of actually unmapping. - const size_t size = pmem.size(); - const void* const res = mmap((void*)addr, size, PROT_NONE, MAP_FIXED|MAP_ANONYMOUS|MAP_PRIVATE|MAP_NORESERVE, -1, 0); + const void* const res = mmap((void*)addr, pmem.size(), PROT_NONE, MAP_FIXED|MAP_ANONYMOUS|MAP_PRIVATE|MAP_NORESERVE, -1, 0); if (res == MAP_FAILED) { ZErrno err; map_failed(err); @@ -232,11 +292,11 @@ } uintptr_t ZPhysicalMemoryBacking::nmt_address(uintptr_t offset) const { - // From an NMT point of view we treat the first heap mapping (marked0) as committed + // From an NMT point of view we treat the first heap view (marked0) as committed return ZAddress::marked0(offset); } -void ZPhysicalMemoryBacking::map(ZPhysicalMemory pmem, uintptr_t offset) const { +void ZPhysicalMemoryBacking::map(const ZPhysicalMemory& pmem, uintptr_t offset) const { if (ZVerifyViews) { // Map good view map_view(pmem, ZAddress::good(offset), AlwaysPreTouch); @@ -248,7 +308,7 @@ } } -void ZPhysicalMemoryBacking::unmap(ZPhysicalMemory pmem, uintptr_t offset) const { +void ZPhysicalMemoryBacking::unmap(const ZPhysicalMemory& pmem, uintptr_t offset) const { if (ZVerifyViews) { // Unmap good view unmap_view(pmem, ZAddress::good(offset)); @@ -260,13 +320,13 @@ } } -void ZPhysicalMemoryBacking::debug_map(ZPhysicalMemory pmem, uintptr_t offset) const { +void ZPhysicalMemoryBacking::debug_map(const ZPhysicalMemory& pmem, uintptr_t offset) const { // Map good view assert(ZVerifyViews, "Should be enabled"); map_view(pmem, ZAddress::good(offset), false /* pretouch */); } -void ZPhysicalMemoryBacking::debug_unmap(ZPhysicalMemory pmem, uintptr_t offset) const { +void ZPhysicalMemoryBacking::debug_unmap(const ZPhysicalMemory& pmem, uintptr_t offset) const { // Unmap good view assert(ZVerifyViews, "Should be enabled"); unmap_view(pmem, ZAddress::good(offset));