--- old/src/hotspot/os/linux/gc/z/zNUMA_linux.cpp 2019-05-02 09:57:42.246809154 +0200 +++ new/src/hotspot/os/linux/gc/z/zNUMA_linux.cpp 2019-05-02 09:57:42.022801756 +0200 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016, 2017, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2016, 2019, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -39,7 +39,7 @@ #endif static int z_get_mempolicy(uint32_t* mode, const unsigned long *nmask, unsigned long maxnode, uintptr_t addr, int flags) { - return syscall(__NR_get_mempolicy, mode, nmask, maxnode, addr, flags); + return syscall(SYS_get_mempolicy, mode, nmask, maxnode, addr, flags); } void ZNUMA::initialize_platform() { --- old/src/hotspot/os_cpu/linux_x86/gc/z/zBackingFile_linux_x86.cpp 2019-05-02 09:57:42.585820350 +0200 +++ new/src/hotspot/os_cpu/linux_x86/gc/z/zBackingFile_linux_x86.cpp 2019-05-02 09:57:42.344812391 +0200 @@ -26,8 +26,10 @@ #include "gc/z/zBackingFile_linux_x86.hpp" #include "gc/z/zBackingPath_linux_x86.hpp" #include "gc/z/zErrno.hpp" +#include "gc/z/zGlobals.hpp" #include "gc/z/zLargePages.inline.hpp" #include "logging/log.hpp" +#include "runtime/init.hpp" #include "runtime/os.hpp" #include "utilities/align.hpp" #include "utilities/debug.hpp" @@ -36,29 +38,31 @@ #include #include #include +#include #include #include -// Filesystem names -#define ZFILESYSTEM_TMPFS "tmpfs" -#define ZFILESYSTEM_HUGETLBFS "hugetlbfs" - -// Sysfs file for transparent huge page on tmpfs -#define ZFILENAME_SHMEM_ENABLED "/sys/kernel/mm/transparent_hugepage/shmem_enabled" - -// Java heap filename -#define ZFILENAME_HEAP "java_heap" - +// // Support for building on older Linux systems -#ifndef __NR_memfd_create -#define __NR_memfd_create 319 +// + +// System calls +#ifndef SYS_fallocate +#define SYS_fallocate 285 #endif +#ifndef SYS_memfd_create +#define SYS_memfd_create 319 +#endif + +// memfd_create(2) flags #ifndef MFD_CLOEXEC #define MFD_CLOEXEC 0x0001U #endif #ifndef MFD_HUGETLB #define MFD_HUGETLB 0x0004U #endif + +// open(2) flags #ifndef O_CLOEXEC #define O_CLOEXEC 02000000 #endif @@ -66,6 +70,14 @@ #define O_TMPFILE (020000000 | O_DIRECTORY) #endif +// fallocate(2) flags +#ifndef FALLOC_FL_KEEP_SIZE +#define FALLOC_FL_KEEP_SIZE 0x01 +#endif +#ifndef FALLOC_FL_PUNCH_HOLE +#define FALLOC_FL_PUNCH_HOLE 0x02 +#endif + // Filesystem types, see statfs(2) #ifndef TMPFS_MAGIC #define TMPFS_MAGIC 0x01021994 @@ -74,6 +86,16 @@ #define HUGETLBFS_MAGIC 0x958458f6 #endif +// Filesystem names +#define ZFILESYSTEM_TMPFS "tmpfs" +#define ZFILESYSTEM_HUGETLBFS "hugetlbfs" + +// Sysfs file for transparent huge page on tmpfs +#define ZFILENAME_SHMEM_ENABLED "/sys/kernel/mm/transparent_hugepage/shmem_enabled" + +// Java heap filename +#define ZFILENAME_HEAP "java_heap" + // Preferred tmpfs mount points, ordered by priority static const char* z_preferred_tmpfs_mountpoints[] = { "/dev/shm", @@ -88,15 +110,22 @@ NULL }; -static int z_memfd_create(const char *name, unsigned int flags) { - return syscall(__NR_memfd_create, name, flags); +static int z_fallocate_hugetlbfs_attempts = 3; +static bool z_fallocate_supported = true; + +static int z_fallocate(int fd, int mode, size_t offset, size_t length) { + return syscall(SYS_fallocate, fd, mode, offset, length); } -bool ZBackingFile::_hugetlbfs_mmap_retry = true; +static int z_memfd_create(const char *name, unsigned int flags) { + return syscall(SYS_memfd_create, name, flags); +} ZBackingFile::ZBackingFile() : _fd(-1), + _size(0), _filesystem(0), + _block_size(0), _available(0), _initialized(false) { @@ -107,46 +136,53 @@ } // Get filesystem statistics - struct statfs statfs_buf; - if (fstatfs(_fd, &statfs_buf) == -1) { + struct statfs buf; + if (fstatfs(_fd, &buf) == -1) { ZErrno err; - log_error(gc, init)("Failed to determine filesystem type for backing file (%s)", - err.to_string()); + log_error(gc)("Failed to determine filesystem type for backing file (%s)", err.to_string()); return; } - _filesystem = statfs_buf.f_type; - _available = statfs_buf.f_bavail * statfs_buf.f_bsize; + _filesystem = buf.f_type; + _block_size = buf.f_bsize; + _available = buf.f_bavail * _block_size; // Make sure we're on a supported filesystem if (!is_tmpfs() && !is_hugetlbfs()) { - log_error(gc, init)("Backing file must be located on a %s or a %s filesystem", - ZFILESYSTEM_TMPFS, ZFILESYSTEM_HUGETLBFS); + log_error(gc)("Backing file must be located on a %s or a %s filesystem", + ZFILESYSTEM_TMPFS, ZFILESYSTEM_HUGETLBFS); return; } // Make sure the filesystem type matches requested large page type if (ZLargePages::is_transparent() && !is_tmpfs()) { - log_error(gc, init)("-XX:+UseTransparentHugePages can only be enable when using a %s filesystem", - ZFILESYSTEM_TMPFS); + log_error(gc)("-XX:+UseTransparentHugePages can only be enable when using a %s filesystem", + ZFILESYSTEM_TMPFS); return; } if (ZLargePages::is_transparent() && !tmpfs_supports_transparent_huge_pages()) { - log_error(gc, init)("-XX:+UseTransparentHugePages on a %s filesystem not supported by kernel", - ZFILESYSTEM_TMPFS); + log_error(gc)("-XX:+UseTransparentHugePages on a %s filesystem not supported by kernel", + ZFILESYSTEM_TMPFS); return; } if (ZLargePages::is_explicit() && !is_hugetlbfs()) { - log_error(gc, init)("-XX:+UseLargePages (without -XX:+UseTransparentHugePages) can only be enabled when using a %s filesystem", - ZFILESYSTEM_HUGETLBFS); + log_error(gc)("-XX:+UseLargePages (without -XX:+UseTransparentHugePages) can only be enabled " + "when using a %s filesystem", ZFILESYSTEM_HUGETLBFS); return; } if (!ZLargePages::is_explicit() && is_hugetlbfs()) { - log_error(gc, init)("-XX:+UseLargePages must be enabled when using a %s filesystem", - ZFILESYSTEM_HUGETLBFS); + log_error(gc)("-XX:+UseLargePages must be enabled when using a %s filesystem", + ZFILESYSTEM_HUGETLBFS); + return; + } + + const size_t expected_block_size = is_tmpfs() ? os::vm_page_size() : os::large_page_size(); + if (expected_block_size != _block_size) { + log_error(gc)("%s filesystem has unexpected block size " SIZE_FORMAT " (expected " SIZE_FORMAT ")", + is_tmpfs() ? ZFILESYSTEM_TMPFS : ZFILESYSTEM_HUGETLBFS, _block_size, expected_block_size); return; } @@ -165,7 +201,7 @@ if (fd == -1) { ZErrno err; log_debug(gc, init)("Failed to create memfd file (%s)", - ((UseLargePages && err == EINVAL) ? "Hugepages not supported" : err.to_string())); + ((ZLargePages::is_explicit() && err == EINVAL) ? "Hugepages not supported" : err.to_string())); return -1; } @@ -185,7 +221,7 @@ // Find mountpoint ZBackingPath path(filesystem, preferred_mountpoints); if (path.get() == NULL) { - log_error(gc, init)("Use -XX:ZPath to specify the path to a %s filesystem", filesystem); + log_error(gc)("Use -XX:ZPath to specify the path to a %s filesystem", filesystem); return -1; } @@ -201,7 +237,7 @@ struct stat stat_buf; if (fstat(fd_anon, &stat_buf) == -1) { ZErrno err; - log_error(gc, init)("Failed to determine inode number for anonymous file (%s)", err.to_string()); + log_error(gc)("Failed to determine inode number for anonymous file (%s)", err.to_string()); return -1; } @@ -220,14 +256,14 @@ const int fd = os::open(filename, O_CREAT|O_EXCL|O_RDWR|O_CLOEXEC, S_IRUSR|S_IWUSR); if (fd == -1) { ZErrno err; - log_error(gc, init)("Failed to create file %s (%s)", filename, err.to_string()); + log_error(gc)("Failed to create file %s (%s)", filename, err.to_string()); return -1; } // Unlink file if (unlink(filename) == -1) { ZErrno err; - log_error(gc, init)("Failed to unlink file %s (%s)", filename, err.to_string()); + log_error(gc)("Failed to unlink file %s (%s)", filename, err.to_string()); return -1; } @@ -262,6 +298,10 @@ return _fd; } +size_t ZBackingFile::size() const { + return _size; +} + size_t ZBackingFile::available() const { return _available; } @@ -280,147 +320,271 @@ return access(ZFILENAME_SHMEM_ENABLED, R_OK) == 0; } -bool ZBackingFile::try_split_and_expand_tmpfs(size_t offset, size_t length, size_t alignment) const { - // Try first smaller part. - const size_t offset0 = offset; - const size_t length0 = align_up(length / 2, alignment); - if (!try_expand_tmpfs(offset0, length0, alignment)) { - return false; - } - - // Try second smaller part. - const size_t offset1 = offset0 + length0; - const size_t length1 = length - length0; - if (!try_expand_tmpfs(offset1, length1, alignment)) { - return false; +ZErrno ZBackingFile::fallocate_compat_ftruncate(size_t size) const { + while (ftruncate(_fd, size) == -1) { + if (errno != EINTR) { + // Failed + return errno; + } } - return true; + // Success + return 0; } -bool ZBackingFile::try_expand_tmpfs(size_t offset, size_t length, size_t alignment) const { - assert(length > 0, "Invalid length"); - assert(is_aligned(length, alignment), "Invalid length"); - - ZErrno err = posix_fallocate(_fd, offset, length); - - if (err == EINTR && length > alignment) { - // Calling posix_fallocate() with a large length can take a long - // time to complete. When running profilers, such as VTune, this - // syscall will be constantly interrupted by signals. Expanding - // the file in smaller steps avoids this problem. - return try_split_and_expand_tmpfs(offset, length, alignment); +ZErrno ZBackingFile::fallocate_compat_mmap(size_t offset, size_t length, bool touch) const { + // On hugetlbfs, mapping a file segment will fail immediately, without + // the need to touch the mapped pages first, if there aren't enough huge + // pages available to back the mapping. + void* const addr = mmap(0, length, PROT_READ|PROT_WRITE, MAP_SHARED, _fd, offset); + if (addr == MAP_FAILED) { + // Failed + return errno; } - if (err) { - log_error(gc)("Failed to allocate backing file (%s)", err.to_string()); - return false; + // Once mapped, the huge pages are only reserved. We need to touch them + // to associate them with the file segment. Note that we can not punch + // hole in file segments which only have reserved pages. + if (touch) { + char* const start = (char*)addr; + char* const end = start + length; + os::pretouch_memory(start, end, _block_size); } - return true; + // Unmap again. From now on, the huge pages that were mapped are allocated + // to this file. There's no risk in getting SIGBUS when touching them. + if (munmap(addr, length) == -1) { + // Failed + return errno; + } + + // Success + return 0; } -bool ZBackingFile::try_expand_tmpfs(size_t offset, size_t length) const { - assert(is_tmpfs(), "Wrong filesystem"); - return try_expand_tmpfs(offset, length, os::vm_page_size()); +ZErrno ZBackingFile::fallocate_compat_pwrite(size_t offset, size_t length) const { + uint8_t data = 0; + + // Allocate backing memory by writing to each block + for (size_t pos = offset; pos < offset + length; pos += _block_size) { + if (pwrite(_fd, &data, sizeof(data), pos) == -1) { + // Failed + return errno; + } + } + + // Success + return 0; } -bool ZBackingFile::try_expand_hugetlbfs(size_t offset, size_t length) const { - assert(is_hugetlbfs(), "Wrong filesystem"); - - // Prior to kernel 4.3, hugetlbfs did not support posix_fallocate(). - // Instead of posix_fallocate() we can use a well-known workaround, - // which involves truncating the file to requested size and then try - // to map it to verify that there are enough huge pages available to - // back it. - while (ftruncate(_fd, offset + length) == -1) { - ZErrno err; - if (err != EINTR) { - log_error(gc)("Failed to truncate backing file (%s)", err.to_string()); - return false; +ZErrno ZBackingFile::fallocate_fill_hole_compat(size_t offset, size_t length) { + // fallocate(2) is only supported by tmpfs since Linux 3.5, and by hugetlbfs + // since Linux 4.3. When fallocate(2) is not supported we emulate it using + // ftruncate/pwrite (for tmpfs) or ftruncate/mmap/munmap (for hugetlbfs). + + const size_t end = offset + length; + if (end > _size) { + // Increase file size + const ZErrno err = fallocate_compat_ftruncate(end); + if (err) { + // Failed + return err; } } - // If we fail mapping during initialization, i.e. when we are pre-mapping - // the heap, then we wait and retry a few times before giving up. Otherwise - // there is a risk that running JVMs back-to-back will fail, since there - // is a delay between process termination and the huge pages owned by that - // process being returned to the huge page pool and made available for new - // allocations. - void* addr = MAP_FAILED; - const int max_attempts = 5; - for (int attempt = 1; attempt <= max_attempts; attempt++) { - addr = mmap(0, length, PROT_READ|PROT_WRITE, MAP_SHARED, _fd, offset); - if (addr != MAP_FAILED || !_hugetlbfs_mmap_retry) { - // Mapping was successful or mmap retry is disabled - break; + // Allocate backing memory + const ZErrno err = is_hugetlbfs() ? fallocate_compat_mmap(offset, length, false /* touch */) + : fallocate_compat_pwrite(offset, length); + if (err) { + if (end > _size) { + // Restore file size + fallocate_compat_ftruncate(_size); } - ZErrno err; - log_debug(gc)("Failed to map backing file (%s), attempt %d of %d", - err.to_string(), attempt, max_attempts); + // Failed + return err; + } - // Wait and retry in one second, in the hope that - // huge pages will be available by then. - sleep(1); + if (end > _size) { + // Record new file size + _size = end; } - // Disable mmap retry from now on - if (_hugetlbfs_mmap_retry) { - _hugetlbfs_mmap_retry = false; + // Success + return 0; +} + +ZErrno ZBackingFile::fallocate_fill_hole_syscall(size_t offset, size_t length) { + const int mode = 0; // Allocate + const int res = z_fallocate(_fd, mode, offset, length); + if (res == -1) { + // Failed + return errno; } - if (addr == MAP_FAILED) { - // Not enough huge pages left - ZErrno err; - log_error(gc)("Failed to map backing file (%s)", err.to_string()); - return false; + const size_t end = offset + length; + if (end > _size) { + // Record new file size + _size = end; } - // Successful mapping, unmap again. From now on the pages we mapped - // will be reserved for this file. - if (munmap(addr, length) == -1) { - ZErrno err; - log_error(gc)("Failed to unmap backing file (%s)", err.to_string()); - return false; + // Success + return 0; +} + +ZErrno ZBackingFile::fallocate_fill_hole(size_t offset, size_t length) { + // Using compat mode is more efficient when allocating space on hugetlbfs. + // Note that allocating huge pages this way will only reserve them, and not + // associate them with segments of the file. We must guarantee that we at + // some point touch these segments, otherwise we can not punch hole in them. + if (z_fallocate_supported && !is_hugetlbfs()) { + const ZErrno err = fallocate_fill_hole_syscall(offset, length); + if (!err) { + // Success + return 0; + } + + if (err != ENOSYS && err != EOPNOTSUPP) { + // Failed + return err; + } + + // Not supported + log_debug(gc)("Falling back to fallocate() compatibility mode"); + z_fallocate_supported = false; } - return true; + return fallocate_fill_hole_compat(offset, length); } -bool ZBackingFile::try_expand_tmpfs_or_hugetlbfs(size_t offset, size_t length, size_t alignment) const { - assert(is_aligned(offset, alignment), "Invalid offset"); - assert(is_aligned(length, alignment), "Invalid length"); +ZErrno ZBackingFile::fallocate_punch_hole(size_t offset, size_t length) { + if (is_hugetlbfs()) { + // We can only punch hole in pages that have been touched. Non-touched + // pages are only reserved, and not associated with any specific file + // segment. We don't know which pages have been previously touched, so + // we always touch them here to guarantee that we can punch hole. + const ZErrno err = fallocate_compat_mmap(offset, length, true /* touch */); + if (err) { + // Failed + return err; + } + } - log_debug(gc)("Expanding heap from " SIZE_FORMAT "M to " SIZE_FORMAT "M", offset / M, (offset + length) / M); + const int mode = FALLOC_FL_PUNCH_HOLE|FALLOC_FL_KEEP_SIZE; + if (z_fallocate(_fd, mode, offset, length) == -1) { + // Failed + return errno; + } - return is_hugetlbfs() ? try_expand_hugetlbfs(offset, length) : try_expand_tmpfs(offset, length); + // Success + return 0; } -size_t ZBackingFile::try_expand(size_t offset, size_t length, size_t alignment) const { - size_t start = offset; - size_t end = offset + length; +ZErrno ZBackingFile::split_and_fallocate(bool punch_hole, size_t offset, size_t length) { + // Try first half + const size_t offset0 = offset; + const size_t length0 = align_up(length / 2, _block_size); + const ZErrno err0 = fallocate(punch_hole, offset0, length0); + if (err0) { + return err0; + } - // Try to expand - if (try_expand_tmpfs_or_hugetlbfs(start, length, alignment)) { + // Try second half + const size_t offset1 = offset0 + length0; + const size_t length1 = length - length0; + const ZErrno err1 = fallocate(punch_hole, offset1, length1); + if (err1) { + return err1; + } + + // Success + return 0; +} + +ZErrno ZBackingFile::fallocate(bool punch_hole, size_t offset, size_t length) { + assert(is_aligned(offset, _block_size), "Invalid offset"); + assert(is_aligned(length, _block_size), "Invalid length"); + + const ZErrno err = punch_hole ? fallocate_punch_hole(offset, length) : fallocate_fill_hole(offset, length); + if (err == EINTR && length > _block_size) { + // Calling fallocate(2) with a large length can take a long time to + // complete. When running profilers, such as VTune, this syscall will + // be constantly interrupted by signals. Expanding the file in smaller + // steps avoids this problem. + return split_and_fallocate(punch_hole, offset, length); + } + + return err; +} + +bool ZBackingFile::commit_inner(size_t offset, size_t length) { + log_trace(gc, heap)("Committing memory: " SIZE_FORMAT "M-" SIZE_FORMAT "M (" SIZE_FORMAT "M)", + offset / M, (offset + length) / M, length / M); + +retry: + const ZErrno err = fallocate(false /* punch_hole */, offset, length); + if (err) { + if (err == ENOSPC && !is_init_completed() && is_hugetlbfs() && z_fallocate_hugetlbfs_attempts-- > 0) { + // If we fail to allocate during initialization, due to lack of space on + // the hugetlbfs filesystem, then we wait and retry a few times before + // giving up. Otherwise there is a risk that running JVMs back-to-back + // will fail, since there is a delay between process termination and the + // huge pages owned by that process being returned to the huge page pool + // and made available for new allocations. + log_debug(gc, init)("Failed to commit memory (%s), retrying", err.to_string()); + + // Wait and retry in one second, in the hope that huge pages will be + // available by then. + sleep(1); + goto retry; + } + + // Failed + log_error(gc)("Failed to commit memory (%s)", err.to_string()); + return false; + } + + // Success + return true; +} + +size_t ZBackingFile::commit(size_t offset, size_t length) { + // Try to commit the whole region + if (commit_inner(offset, length)) { // Success - return end; + return length; } - // Failed, try to expand as much as possible + // Failed, try to commit as much as possible + size_t start = offset; + size_t end = offset + length; + for (;;) { - length = align_down((end - start) / 2, alignment); - if (length < alignment) { - // Done, don't expand more - return start; + length = align_down((end - start) / 2, ZGranuleSize); + if (length < ZGranuleSize) { + // Done, don't commit more + return start - offset; } - if (try_expand_tmpfs_or_hugetlbfs(start, length, alignment)) { - // Success, try expand more + if (commit_inner(start, length)) { + // Success, try commit more start += length; } else { - // Failed, try expand less + // Failed, try commit less end -= length; } } } + +size_t ZBackingFile::uncommit(size_t offset, size_t length) { + log_trace(gc, heap)("Uncommitting memory: " SIZE_FORMAT "M-" SIZE_FORMAT "M (" SIZE_FORMAT "M)", + offset / M, (offset + length) / M, length / M); + + const ZErrno err = fallocate(true /* punch_hole */, offset, length); + if (err) { + log_error(gc)("Failed to uncommit memory (%s)", err.to_string()); + return 0; + } + + return length; +} --- old/src/hotspot/os_cpu/linux_x86/gc/z/zBackingFile_linux_x86.hpp 2019-05-02 09:57:42.955832570 +0200 +++ new/src/hotspot/os_cpu/linux_x86/gc/z/zBackingFile_linux_x86.hpp 2019-05-02 09:57:42.709824445 +0200 @@ -26,12 +26,14 @@ #include "memory/allocation.hpp" +class ZErrno; + class ZBackingFile { private: - static bool _hugetlbfs_mmap_retry; - int _fd; + size_t _size; uint64_t _filesystem; + size_t _block_size; size_t _available; bool _initialized; @@ -43,11 +45,17 @@ bool is_hugetlbfs() const; bool tmpfs_supports_transparent_huge_pages() const; - bool try_split_and_expand_tmpfs(size_t offset, size_t length, size_t alignment) const; - bool try_expand_tmpfs(size_t offset, size_t length, size_t alignment) const; - bool try_expand_tmpfs(size_t offset, size_t length) const; - bool try_expand_hugetlbfs(size_t offset, size_t length) const; - bool try_expand_tmpfs_or_hugetlbfs(size_t offset, size_t length, size_t alignment) const; + ZErrno fallocate_compat_ftruncate(size_t size) const; + ZErrno fallocate_compat_mmap(size_t offset, size_t length, bool reserve_only) const; + ZErrno fallocate_compat_pwrite(size_t offset, size_t length) const; + ZErrno fallocate_fill_hole_compat(size_t offset, size_t length); + ZErrno fallocate_fill_hole_syscall(size_t offset, size_t length); + ZErrno fallocate_fill_hole(size_t offset, size_t length); + ZErrno fallocate_punch_hole(size_t offset, size_t length); + ZErrno split_and_fallocate(bool punch_hole, size_t offset, size_t length); + ZErrno fallocate(bool punch_hole, size_t offset, size_t length); + + bool commit_inner(size_t offset, size_t length); public: ZBackingFile(); @@ -55,9 +63,11 @@ bool is_initialized() const; int fd() const; + size_t size() const; size_t available() const; - size_t try_expand(size_t offset, size_t length, size_t alignment) const; + size_t commit(size_t offset, size_t length); + size_t uncommit(size_t offset, size_t length); }; #endif // OS_CPU_LINUX_X86_GC_Z_ZBACKINGFILE_LINUX_X86_HPP --- old/src/hotspot/os_cpu/linux_x86/gc/z/zBackingPath_linux_x86.cpp 2019-05-02 09:57:43.278843237 +0200 +++ new/src/hotspot/os_cpu/linux_x86/gc/z/zBackingPath_linux_x86.cpp 2019-05-02 09:57:43.056835905 +0200 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016, 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2016, 2019, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -72,7 +72,7 @@ FILE* fd = fopen(PROC_SELF_MOUNTINFO, "r"); if (fd == NULL) { ZErrno err; - log_error(gc, init)("Failed to open %s: %s", PROC_SELF_MOUNTINFO, err.to_string()); + log_error(gc)("Failed to open %s: %s", PROC_SELF_MOUNTINFO, err.to_string()); return; } @@ -113,10 +113,10 @@ } // Preferred mount point not found - log_error(gc, init)("More than one %s filesystem found:", filesystem); + log_error(gc)("More than one %s filesystem found:", filesystem); ZArrayIterator iter2(mountpoints); for (char* mountpoint; iter2.next(&mountpoint);) { - log_error(gc, init)(" %s", mountpoint); + log_error(gc)(" %s", mountpoint); } return NULL; @@ -130,7 +130,7 @@ if (mountpoints.size() == 0) { // No mount point found - log_error(gc, init)("Failed to find an accessible %s filesystem", filesystem); + log_error(gc)("Failed to find an accessible %s filesystem", filesystem); } else if (mountpoints.size() == 1) { // One mount point found path = strdup(mountpoints.at(0)); --- old/src/hotspot/os_cpu/linux_x86/gc/z/zPhysicalMemoryBacking_linux_x86.cpp 2019-05-02 09:57:43.596853739 +0200 +++ new/src/hotspot/os_cpu/linux_x86/gc/z/zPhysicalMemoryBacking_linux_x86.cpp 2019-05-02 09:57:43.375846440 +0200 @@ -32,6 +32,7 @@ #include "gc/z/zPhysicalMemory.inline.hpp" #include "gc/z/zPhysicalMemoryBacking_linux_x86.hpp" #include "logging/log.hpp" +#include "runtime/init.hpp" #include "runtime/os.hpp" #include "utilities/align.hpp" #include "utilities/debug.hpp" @@ -40,7 +41,11 @@ #include #include +// // Support for building on older Linux systems +// + +// madvise(2) flags #ifndef MADV_HUGEPAGE #define MADV_HUGEPAGE 14 #endif @@ -48,22 +53,37 @@ // Proc file entry for max map mount #define ZFILENAME_PROC_MAX_MAP_COUNT "/proc/sys/vm/max_map_count" -ZPhysicalMemoryBacking::ZPhysicalMemoryBacking(size_t max_capacity) : - _manager(), - _file() { +bool ZPhysicalMemoryBacking::is_initialized() const { + return _file.is_initialized(); +} - if (!_file.is_initialized()) { +void ZPhysicalMemoryBacking::warn_available_space(size_t max) const { + // Note that the available space on a tmpfs or a hugetlbfs filesystem + // will be zero if no size limit was specified when it was mounted. + const size_t available = _file.available(); + if (available == 0) { + // No size limit set, skip check + log_info(gc, init)("Available space on backing filesystem: N/A"); return; } - // Check and warn if max map count is too low - check_max_map_count(max_capacity); + log_info(gc, init)("Available space on backing filesystem: " SIZE_FORMAT "M", available / M); - // Check and warn if available space on filesystem is too low - check_available_space_on_filesystem(max_capacity); + // Warn if the filesystem doesn't currently have enough space available to hold + // the max heap size. The max heap size will be capped if we later hit this limit + // when trying to expand the heap. + if (available < max) { + log_warning(gc)("***** WARNING! INCORRECT SYSTEM CONFIGURATION DETECTED! *****"); + log_warning(gc)("Not enough space available on the backing filesystem to hold the current max Java heap"); + log_warning(gc)("size (" SIZE_FORMAT "M). Please adjust the size of the backing filesystem accordingly " + "(available", max / M); + log_warning(gc)("space is currently " SIZE_FORMAT "M). Continuing execution with the current filesystem " + "size could", available / M); + log_warning(gc)("lead to a premature OutOfMemoryError being thrown, due to failure to map memory."); + } } -void ZPhysicalMemoryBacking::check_max_map_count(size_t max_capacity) const { +void ZPhysicalMemoryBacking::warn_max_map_count(size_t max) const { const char* const filename = ZFILENAME_PROC_MAX_MAP_COUNT; FILE* const file = fopen(filename, "r"); if (file == NULL) { @@ -86,62 +106,101 @@ // However, ZGC tends to create the most mappings and dominate the total count. // In the worst cases, ZGC will map each granule three times, i.e. once per heap view. // We speculate that we need another 20% to allow for non-ZGC subsystems to map memory. - const size_t required_max_map_count = (max_capacity / ZGranuleSize) * 3 * 1.2; + const size_t required_max_map_count = (max / ZGranuleSize) * 3 * 1.2; if (actual_max_map_count < required_max_map_count) { - log_warning(gc, init)("***** WARNING! INCORRECT SYSTEM CONFIGURATION DETECTED! *****"); - log_warning(gc, init)("The system limit on number of memory mappings per process might be too low " - "for the given"); - log_warning(gc, init)("max Java heap size (" SIZE_FORMAT "M). Please adjust %s to allow for at", - max_capacity / M, filename); - log_warning(gc, init)("least " SIZE_FORMAT " mappings (current limit is " SIZE_FORMAT "). Continuing " - "execution with the current", required_max_map_count, actual_max_map_count); - log_warning(gc, init)("limit could lead to a fatal error, due to failure to map memory."); + log_warning(gc)("***** WARNING! INCORRECT SYSTEM CONFIGURATION DETECTED! *****"); + log_warning(gc)("The system limit on number of memory mappings per process might be too low for the given"); + log_warning(gc)("max Java heap size (" SIZE_FORMAT "M). Please adjust %s to allow for at", + max / M, filename); + log_warning(gc)("least " SIZE_FORMAT " mappings (current limit is " SIZE_FORMAT "). Continuing execution " + "with the current", required_max_map_count, actual_max_map_count); + log_warning(gc)("limit could lead to a fatal error, due to failure to map memory."); } } -void ZPhysicalMemoryBacking::check_available_space_on_filesystem(size_t max_capacity) const { - // Note that the available space on a tmpfs or a hugetlbfs filesystem - // will be zero if no size limit was specified when it was mounted. - const size_t available = _file.available(); - if (available == 0) { - // No size limit set, skip check - log_info(gc, init)("Available space on backing filesystem: N/A"); - return; - } +void ZPhysicalMemoryBacking::warn_commit_limits(size_t max) const { + // Warn if available space is too low + warn_available_space(max); - log_info(gc, init)("Available space on backing filesystem: " SIZE_FORMAT "M", - available / M); + // Warn if max map count is too low + warn_max_map_count(max); +} - // Warn if the filesystem doesn't currently have enough space available to hold - // the max heap size. The max heap size will be capped if we later hit this limit - // when trying to expand the heap. - if (available < max_capacity) { - log_warning(gc, init)("***** WARNING! INCORRECT SYSTEM CONFIGURATION DETECTED! *****"); - log_warning(gc, init)("Not enough space available on the backing filesystem to hold the current " - "max Java heap"); - log_warning(gc, init)("size (" SIZE_FORMAT "M). Please adjust the size of the backing filesystem " - "accordingly (available", max_capacity / M); - log_warning(gc, init)("space is currently " SIZE_FORMAT "M). Continuing execution with the current " - "filesystem size could", available / M); - log_warning(gc, init)("lead to a premature OutOfMemoryError being thrown, due to failure to map " - "memory."); - } +bool ZPhysicalMemoryBacking::supports_uncommit() { + assert(!is_init_completed(), "Invalid state"); + assert(_file.size() >= ZGranuleSize, "Invalid size"); + + // Test if uncommit is supported by uncommitting and then re-committing a granule + return commit(uncommit(ZGranuleSize)) == ZGranuleSize; } -bool ZPhysicalMemoryBacking::is_initialized() const { - return _file.is_initialized(); +size_t ZPhysicalMemoryBacking::commit(size_t size) { + size_t committed = 0; + + // Fill holes in the backing file + while (committed < size) { + size_t allocated = 0; + const size_t remaining = size - committed; + const uintptr_t start = _uncommitted.alloc_from_front_at_most(remaining, &allocated); + if (start == UINTPTR_MAX) { + // No holes to commit + break; + } + + // Try commit hole + const size_t filled = _file.commit(start, allocated); + if (filled > 0) { + // Successful or partialy successful + _committed.free(start, filled); + committed += filled; + } + if (filled < allocated) { + // Failed or partialy failed + _uncommitted.free(start + filled, allocated - filled); + return committed; + } + } + + // Expand backing file + if (committed < size) { + const size_t remaining = size - committed; + const uintptr_t start = _file.size(); + const size_t expanded = _file.commit(start, remaining); + if (expanded > 0) { + // Successful or partialy successful + _committed.free(start, expanded); + committed += expanded; + } + } + + return committed; } -size_t ZPhysicalMemoryBacking::try_expand(size_t old_capacity, size_t new_capacity) { - assert(old_capacity < new_capacity, "Invalid old/new capacity"); +size_t ZPhysicalMemoryBacking::uncommit(size_t size) { + size_t uncommitted = 0; + + // Punch holes in backing file + while (uncommitted < size) { + size_t allocated = 0; + const size_t remaining = size - uncommitted; + const uintptr_t start = _committed.alloc_from_back_at_most(remaining, &allocated); + assert(start != UINTPTR_MAX, "Allocation should never fail"); - const size_t capacity = _file.try_expand(old_capacity, new_capacity - old_capacity, ZGranuleSize); - if (capacity > old_capacity) { - // Add expanded capacity to free list - _manager.free(old_capacity, capacity - old_capacity); + // Try punch hole + const size_t punched = _file.uncommit(start, allocated); + if (punched > 0) { + // Successful or partialy successful + _uncommitted.free(start, punched); + uncommitted += punched; + } + if (punched < allocated) { + // Failed or partialy failed + _committed.free(start + punched, allocated - punched); + return uncommitted; + } } - return capacity; + return uncommitted; } ZPhysicalMemory ZPhysicalMemoryBacking::alloc(size_t size) { @@ -151,7 +210,7 @@ // Allocate segments for (size_t allocated = 0; allocated < size; allocated += ZGranuleSize) { - const uintptr_t start = _manager.alloc_from_front(ZGranuleSize); + const uintptr_t start = _committed.alloc_from_front(ZGranuleSize); assert(start != UINTPTR_MAX, "Allocation should never fail"); pmem.add_segment(ZPhysicalMemorySegment(start, ZGranuleSize)); } @@ -159,13 +218,13 @@ return pmem; } -void ZPhysicalMemoryBacking::free(ZPhysicalMemory pmem) { +void ZPhysicalMemoryBacking::free(const ZPhysicalMemory& pmem) { const size_t nsegments = pmem.nsegments(); // Free segments for (size_t i = 0; i < nsegments; i++) { - const ZPhysicalMemorySegment segment = pmem.segment(i); - _manager.free(segment.start(), segment.size()); + const ZPhysicalMemorySegment& segment = pmem.segment(i); + _committed.free(segment.start(), segment.size()); } } @@ -178,10 +237,10 @@ } } -void ZPhysicalMemoryBacking::advise_view(uintptr_t addr, size_t size) const { - if (madvise((void*)addr, size, MADV_HUGEPAGE) == -1) { +void ZPhysicalMemoryBacking::advise_view(uintptr_t addr, size_t size, int advice) const { + if (madvise((void*)addr, size, advice) == -1) { ZErrno err; - log_error(gc)("Failed to advise use of transparent huge pages (%s)", err.to_string()); + log_error(gc)("Failed to advise on memory (advice %d, %s)", advice, err.to_string()); } } @@ -190,41 +249,42 @@ os::pretouch_memory((void*)addr, (void*)(addr + size), page_size); } -void ZPhysicalMemoryBacking::map_view(ZPhysicalMemory pmem, uintptr_t addr, bool pretouch) const { +void ZPhysicalMemoryBacking::map_view(const ZPhysicalMemory& pmem, uintptr_t addr, bool pretouch) const { const size_t nsegments = pmem.nsegments(); + size_t size = 0; // Map segments for (size_t i = 0; i < nsegments; i++) { - const ZPhysicalMemorySegment segment = pmem.segment(i); - const size_t size = segment.size(); - const void* const res = mmap((void*)addr, size, PROT_READ|PROT_WRITE, MAP_FIXED|MAP_SHARED, _file.fd(), segment.start()); + const ZPhysicalMemorySegment& segment = pmem.segment(i); + const uintptr_t segment_addr = addr + size; + const void* const res = mmap((void*)segment_addr, segment.size(), PROT_READ|PROT_WRITE, MAP_FIXED|MAP_SHARED, _file.fd(), segment.start()); if (res == MAP_FAILED) { ZErrno err; map_failed(err); } - // Advise on use of transparent huge pages before touching it - if (ZLargePages::is_transparent()) { - advise_view(addr, size); - } + size += segment.size(); + } - // NUMA interleave memory before touching it - ZNUMA::memory_interleave(addr, size); + // Advise on use of transparent huge pages before touching it + if (ZLargePages::is_transparent()) { + advise_view(addr, size, MADV_HUGEPAGE); + } - if (pretouch) { - pretouch_view(addr, size); - } + // NUMA interleave memory before touching it + ZNUMA::memory_interleave(addr, size); - addr += size; + // Pre-touch memory + if (pretouch) { + pretouch_view(addr, size); } } -void ZPhysicalMemoryBacking::unmap_view(ZPhysicalMemory pmem, uintptr_t addr) const { +void ZPhysicalMemoryBacking::unmap_view(const ZPhysicalMemory& pmem, uintptr_t addr) const { // Note that we must keep the address space reservation intact and just detach // the backing memory. For this reason we map a new anonymous, non-accessible // and non-reserved page over the mapping instead of actually unmapping. - const size_t size = pmem.size(); - const void* const res = mmap((void*)addr, size, PROT_NONE, MAP_FIXED|MAP_ANONYMOUS|MAP_PRIVATE|MAP_NORESERVE, -1, 0); + const void* const res = mmap((void*)addr, pmem.size(), PROT_NONE, MAP_FIXED|MAP_ANONYMOUS|MAP_PRIVATE|MAP_NORESERVE, -1, 0); if (res == MAP_FAILED) { ZErrno err; map_failed(err); @@ -232,11 +292,11 @@ } uintptr_t ZPhysicalMemoryBacking::nmt_address(uintptr_t offset) const { - // From an NMT point of view we treat the first heap mapping (marked0) as committed + // From an NMT point of view we treat the first heap view (marked0) as committed return ZAddress::marked0(offset); } -void ZPhysicalMemoryBacking::map(ZPhysicalMemory pmem, uintptr_t offset) const { +void ZPhysicalMemoryBacking::map(const ZPhysicalMemory& pmem, uintptr_t offset) const { if (ZVerifyViews) { // Map good view map_view(pmem, ZAddress::good(offset), AlwaysPreTouch); @@ -248,7 +308,7 @@ } } -void ZPhysicalMemoryBacking::unmap(ZPhysicalMemory pmem, uintptr_t offset) const { +void ZPhysicalMemoryBacking::unmap(const ZPhysicalMemory& pmem, uintptr_t offset) const { if (ZVerifyViews) { // Unmap good view unmap_view(pmem, ZAddress::good(offset)); @@ -260,13 +320,13 @@ } } -void ZPhysicalMemoryBacking::debug_map(ZPhysicalMemory pmem, uintptr_t offset) const { +void ZPhysicalMemoryBacking::debug_map(const ZPhysicalMemory& pmem, uintptr_t offset) const { // Map good view assert(ZVerifyViews, "Should be enabled"); map_view(pmem, ZAddress::good(offset), false /* pretouch */); } -void ZPhysicalMemoryBacking::debug_unmap(ZPhysicalMemory pmem, uintptr_t offset) const { +void ZPhysicalMemoryBacking::debug_unmap(const ZPhysicalMemory& pmem, uintptr_t offset) const { // Unmap good view assert(ZVerifyViews, "Should be enabled"); unmap_view(pmem, ZAddress::good(offset)); --- old/src/hotspot/os_cpu/linux_x86/gc/z/zPhysicalMemoryBacking_linux_x86.hpp 2019-05-02 09:57:43.946865298 +0200 +++ new/src/hotspot/os_cpu/linux_x86/gc/z/zPhysicalMemoryBacking_linux_x86.hpp 2019-05-02 09:57:43.702857240 +0200 @@ -32,35 +32,39 @@ class ZPhysicalMemoryBacking { private: - ZMemoryManager _manager; ZBackingFile _file; + ZMemoryManager _committed; + ZMemoryManager _uncommitted; + + void warn_available_space(size_t max) const; + void warn_max_map_count(size_t max) const; - void check_max_map_count(size_t max_capacity) const; - void check_available_space_on_filesystem(size_t max_capacity) const; void map_failed(ZErrno err) const; - void advise_view(uintptr_t addr, size_t size) const; + void advise_view(uintptr_t addr, size_t size, int advice) const; void pretouch_view(uintptr_t addr, size_t size) const; - void map_view(ZPhysicalMemory pmem, uintptr_t addr, bool pretouch) const; - void unmap_view(ZPhysicalMemory pmem, uintptr_t addr) const; + void map_view(const ZPhysicalMemory& pmem, uintptr_t addr, bool pretouch) const; + void unmap_view(const ZPhysicalMemory& pmem, uintptr_t addr) const; public: - ZPhysicalMemoryBacking(size_t max_capacity); - bool is_initialized() const; - size_t try_expand(size_t old_capacity, size_t new_capacity); + void warn_commit_limits(size_t max) const; + bool supports_uncommit(); + + size_t commit(size_t size); + size_t uncommit(size_t size); ZPhysicalMemory alloc(size_t size); - void free(ZPhysicalMemory pmem); + void free(const ZPhysicalMemory& pmem); uintptr_t nmt_address(uintptr_t offset) const; - void map(ZPhysicalMemory pmem, uintptr_t offset) const; - void unmap(ZPhysicalMemory pmem, uintptr_t offset) const; + void map(const ZPhysicalMemory& pmem, uintptr_t offset) const; + void unmap(const ZPhysicalMemory& pmem, uintptr_t offset) const; - void debug_map(ZPhysicalMemory pmem, uintptr_t offset) const; - void debug_unmap(ZPhysicalMemory pmem, uintptr_t offset) const; + void debug_map(const ZPhysicalMemory& pmem, uintptr_t offset) const; + void debug_unmap(const ZPhysicalMemory& pmem, uintptr_t offset) const; }; #endif // OS_CPU_LINUX_X86_GC_Z_ZPHYSICALMEMORYBACKING_LINUX_X86_HPP --- old/src/hotspot/share/gc/z/vmStructs_z.hpp 2019-05-02 09:57:44.292876725 +0200 +++ new/src/hotspot/share/gc/z/vmStructs_z.hpp 2019-05-02 09:57:44.046868601 +0200 @@ -30,7 +30,6 @@ #include "gc/z/zGranuleMap.hpp" #include "gc/z/zHeap.hpp" #include "gc/z/zPageAllocator.hpp" -#include "gc/z/zPhysicalMemory.hpp" #include "utilities/macros.hpp" // Expose some ZGC globals to the SA agent. @@ -77,20 +76,18 @@ nonstatic_field(ZPage, _virtual, const ZVirtualMemory) \ volatile_nonstatic_field(ZPage, _top, uintptr_t) \ \ - nonstatic_field(ZPageAllocator, _physical, ZPhysicalMemoryManager) \ + nonstatic_field(ZPageAllocator, _max_capacity, const size_t) \ + nonstatic_field(ZPageAllocator, _capacity, size_t) \ nonstatic_field(ZPageAllocator, _used, size_t) \ \ nonstatic_field(ZPageTable, _map, ZGranuleMapForPageTable) \ \ nonstatic_field(ZGranuleMapForPageTable, _map, ZPage** const) \ \ - nonstatic_field(ZVirtualMemory, _start, uintptr_t) \ - nonstatic_field(ZVirtualMemory, _end, uintptr_t) \ + nonstatic_field(ZVirtualMemory, _start, const uintptr_t) \ + nonstatic_field(ZVirtualMemory, _end, const uintptr_t) \ \ - nonstatic_field(ZForwarding, _entries, const ZAttachedArrayForForwarding) \ - \ - nonstatic_field(ZPhysicalMemoryManager, _max_capacity, const size_t) \ - nonstatic_field(ZPhysicalMemoryManager, _capacity, size_t) + nonstatic_field(ZForwarding, _entries, const ZAttachedArrayForForwarding) #define VM_INT_CONSTANTS_ZGC(declare_constant, declare_constant_with_value) \ declare_constant(ZPhaseRelocate) \ --- old/src/hotspot/share/gc/z/zCollectedHeap.cpp 2019-05-02 09:57:44.611887261 +0200 +++ new/src/hotspot/share/gc/z/zCollectedHeap.cpp 2019-05-02 09:57:44.389879929 +0200 @@ -48,6 +48,7 @@ _heap(), _director(new ZDirector()), _driver(new ZDriver()), + _uncommitter(new ZUncommitter()), _stat(new ZStat()), _runtime_workers() {} @@ -77,6 +78,7 @@ void ZCollectedHeap::stop() { _director->stop(); _driver->stop(); + _uncommitter->stop(); _stat->stop(); } @@ -276,6 +278,7 @@ void ZCollectedHeap::gc_threads_do(ThreadClosure* tc) const { tc->do_thread(_director); tc->do_thread(_driver); + tc->do_thread(_uncommitter); tc->do_thread(_stat); _heap.worker_threads_do(tc); _runtime_workers.threads_do(tc); @@ -335,6 +338,8 @@ st->cr(); _driver->print_on(st); st->cr(); + _uncommitter->print_on(st); + st->cr(); _stat->print_on(st); st->cr(); _heap.print_worker_threads_on(st); --- old/src/hotspot/share/gc/z/zCollectedHeap.hpp 2019-05-02 09:57:44.952898523 +0200 +++ new/src/hotspot/share/gc/z/zCollectedHeap.hpp 2019-05-02 09:57:44.709890497 +0200 @@ -30,10 +30,11 @@ #include "gc/z/zCollectorPolicy.hpp" #include "gc/z/zDirector.hpp" #include "gc/z/zDriver.hpp" -#include "gc/z/zInitialize.hpp" #include "gc/z/zHeap.hpp" +#include "gc/z/zInitialize.hpp" #include "gc/z/zRuntimeWorkers.hpp" #include "gc/z/zStat.hpp" +#include "gc/z/zUncommitter.hpp" class ZCollectedHeap : public CollectedHeap { friend class VMStructs; @@ -46,6 +47,7 @@ ZHeap _heap; ZDirector* _director; ZDriver* _driver; + ZUncommitter* _uncommitter; ZStat* _stat; ZRuntimeWorkers _runtime_workers; --- old/src/hotspot/share/gc/z/zHeap.cpp 2019-05-02 09:57:45.274909157 +0200 +++ new/src/hotspot/share/gc/z/zHeap.cpp 2019-05-02 09:57:45.051901792 +0200 @@ -45,6 +45,7 @@ #include "logging/log.hpp" #include "memory/resourceArea.hpp" #include "oops/oop.inline.hpp" +#include "runtime/arguments.hpp" #include "runtime/safepoint.hpp" #include "runtime/thread.hpp" #include "utilities/align.hpp" @@ -62,7 +63,7 @@ ZHeap::ZHeap() : _workers(), _object_allocator(_workers.nworkers()), - _page_allocator(heap_min_size(), heap_max_size(), heap_max_reserve_size()), + _page_allocator(heap_min_size(), heap_initial_size(), heap_max_size(), heap_max_reserve_size()), _page_table(), _forwarding_table(), _mark(&_workers, &_page_table), @@ -81,8 +82,13 @@ } size_t ZHeap::heap_min_size() const { - const size_t aligned_min_size = align_up(InitialHeapSize, ZGranuleSize); - return MIN2(aligned_min_size, heap_max_size()); + const size_t aligned_min_size = align_up(Arguments::min_heap_size(), ZGranuleSize); + return MAX2(MIN2(aligned_min_size, heap_max_size()), heap_max_reserve_size()); +} + +size_t ZHeap::heap_initial_size() const { + const size_t aligned_initial_size = align_up(InitialHeapSize, ZGranuleSize); + return MAX2(MIN2(aligned_initial_size, heap_max_size()), heap_min_size()); } size_t ZHeap::heap_max_size() const { @@ -102,7 +108,7 @@ } size_t ZHeap::min_capacity() const { - return heap_min_size(); + return _page_allocator.min_capacity(); } size_t ZHeap::max_capacity() const { @@ -250,6 +256,10 @@ _page_allocator.free_page(page, reclaimed); } +uint64_t ZHeap::uncommit(uint64_t delay) { + return _page_allocator.uncommit(delay); +} + void ZHeap::before_flip() { if (ZVerifyViews) { // Unmap all pages --- old/src/hotspot/share/gc/z/zHeap.hpp 2019-05-02 09:57:45.622920650 +0200 +++ new/src/hotspot/share/gc/z/zHeap.hpp 2019-05-02 09:57:45.377912559 +0200 @@ -66,6 +66,7 @@ ZServiceability _serviceability; size_t heap_min_size() const; + size_t heap_initial_size() const; size_t heap_max_size() const; size_t heap_max_reserve_size() const; @@ -129,6 +130,9 @@ void undo_alloc_page(ZPage* page); void free_page(ZPage* page, bool reclaimed); + // Uncommit memory + uint64_t uncommit(uint64_t delay); + // Object allocation uintptr_t alloc_tlab(size_t size); uintptr_t alloc_object(size_t size); --- old/src/hotspot/share/gc/z/zLiveMap.cpp 2019-05-02 09:57:45.980932474 +0200 +++ new/src/hotspot/share/gc/z/zLiveMap.cpp 2019-05-02 09:57:45.722923953 +0200 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015, 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -34,15 +34,19 @@ static const ZStatCounter ZCounterMarkSeqNumResetContention("Contention", "Mark SeqNum Reset Contention", ZStatUnitOpsPerSecond); static const ZStatCounter ZCounterMarkSegmentResetContention("Contention", "Mark Segment Reset Contention", ZStatUnitOpsPerSecond); +static size_t bitmap_size(uint32_t size, size_t nsegments) { + // We need at least one bit per segment + return MAX2(size, nsegments) * 2; +} + ZLiveMap::ZLiveMap(uint32_t size) : _seqnum(0), _live_objects(0), _live_bytes(0), _segment_live_bits(0), _segment_claim_bits(0), - // We need at least one bit per segment. - _bitmap(MAX2(size, nsegments) * 2), - _shift(exact_log2(segment_size())) {} + _bitmap(bitmap_size(size, nsegments)), + _segment_shift(exact_log2(segment_size())) {} void ZLiveMap::reset(size_t index) { const uint32_t seqnum_initializing = (uint32_t)-1; @@ -121,3 +125,11 @@ const bool success = set_segment_live_atomic(segment); assert(success, "Should never fail"); } + +void ZLiveMap::resize(uint32_t size) { + const size_t new_bitmap_size = bitmap_size(size, nsegments); + if (_bitmap.size() != new_bitmap_size) { + _bitmap.reinitialize(new_bitmap_size, false /* clear */); + _segment_shift = exact_log2(segment_size()); + } +} --- old/src/hotspot/share/gc/z/zLiveMap.hpp 2019-05-02 09:57:46.326943901 +0200 +++ new/src/hotspot/share/gc/z/zLiveMap.hpp 2019-05-02 09:57:46.081935809 +0200 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015, 2017, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -35,13 +35,13 @@ private: static const size_t nsegments = 64; - volatile uint32_t _seqnum; // Mark sequence number - volatile uint32_t _live_objects; // Number of live objects - volatile size_t _live_bytes; // Number of live bytes - BitMap::bm_word_t _segment_live_bits; // Segment live bits - BitMap::bm_word_t _segment_claim_bits; // Segment claim bits - ZBitMap _bitmap; // Mark bitmap - const size_t _shift; // Segment shift + volatile uint32_t _seqnum; + volatile uint32_t _live_objects; + volatile size_t _live_bytes; + BitMap::bm_word_t _segment_live_bits; + BitMap::bm_word_t _segment_claim_bits; + ZBitMap _bitmap; + size_t _segment_shift; const BitMapView segment_live_bits() const; const BitMapView segment_claim_bits() const; @@ -72,6 +72,7 @@ ZLiveMap(uint32_t size); void reset(); + void resize(uint32_t size); bool is_marked() const; --- old/src/hotspot/share/gc/z/zLiveMap.inline.hpp 2019-05-02 09:57:46.670955262 +0200 +++ new/src/hotspot/share/gc/z/zLiveMap.inline.hpp 2019-05-02 09:57:46.426947203 +0200 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015, 2017, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -92,7 +92,7 @@ } inline BitMap::idx_t ZLiveMap::index_to_segment(BitMap::idx_t index) const { - return index >> _shift; + return index >> _segment_shift; } inline bool ZLiveMap::get(size_t index) const { --- old/src/hotspot/share/gc/z/zMemory.cpp 2019-05-02 09:57:47.023966920 +0200 +++ new/src/hotspot/share/gc/z/zMemory.cpp 2019-05-02 09:57:46.777958795 +0200 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015, 2017, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -49,6 +49,30 @@ return UINTPTR_MAX; } +uintptr_t ZMemoryManager::alloc_from_front_at_most(size_t size, size_t* allocated) { + ZMemory* area = _freelist.first(); + if (area != NULL) { + if (area->size() <= size) { + // Smaller than or equal to requested, remove area + const uintptr_t start = area->start(); + *allocated = area->size(); + _freelist.remove(area); + delete area; + return start; + } else { + // Larger than requested, shrink area + const uintptr_t start = area->start(); + area->shrink_from_front(size); + *allocated = size; + return start; + } + } + + // Out of memory + *allocated = 0; + return UINTPTR_MAX; +} + uintptr_t ZMemoryManager::alloc_from_back(size_t size) { ZListReverseIterator iter(&_freelist); for (ZMemory* area; iter.next(&area);) { @@ -71,6 +95,29 @@ return UINTPTR_MAX; } +uintptr_t ZMemoryManager::alloc_from_back_at_most(size_t size, size_t* allocated) { + ZMemory* area = _freelist.last(); + if (area != NULL) { + if (area->size() <= size) { + // Smaller than or equal to requested, remove area + const uintptr_t start = area->start(); + *allocated = area->size(); + _freelist.remove(area); + delete area; + return start; + } else { + // Larger than requested, shrink area + area->shrink_from_back(size); + *allocated = size; + return area->end(); + } + } + + // Out of memory + *allocated = 0; + return UINTPTR_MAX; +} + void ZMemoryManager::free(uintptr_t start, size_t size) { assert(start != UINTPTR_MAX, "Invalid address"); const uintptr_t end = start + size; --- old/src/hotspot/share/gc/z/zMemory.hpp 2019-05-02 09:57:47.376978578 +0200 +++ new/src/hotspot/share/gc/z/zMemory.hpp 2019-05-02 09:57:47.131970487 +0200 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015, 2017, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -54,7 +54,9 @@ public: uintptr_t alloc_from_front(size_t size); + uintptr_t alloc_from_front_at_most(size_t size, size_t* allocated); uintptr_t alloc_from_back(size_t size); + uintptr_t alloc_from_back_at_most(size_t size, size_t* allocated); void free(uintptr_t start, size_t size); }; --- old/src/hotspot/share/gc/z/zPage.cpp 2019-05-02 09:57:47.722990005 +0200 +++ new/src/hotspot/share/gc/z/zPage.cpp 2019-05-02 09:57:47.477981914 +0200 @@ -28,30 +28,72 @@ #include "utilities/align.hpp" #include "utilities/debug.hpp" -ZPage::ZPage(uint8_t type, ZVirtualMemory vmem, ZPhysicalMemory pmem) : +ZPage::ZPage(const ZVirtualMemory& vmem, const ZPhysicalMemory& pmem) : + _type(type_from_size(vmem.size())), + _numa_id((uint8_t)-1), + _seqnum(0), + _virtual(vmem), + _top(start()), + _livemap(object_max_count()), + _last_used(0), + _physical(pmem) { + assert_initialized(); +} + +ZPage::ZPage(uint8_t type, const ZVirtualMemory& vmem, const ZPhysicalMemory& pmem) : _type(type), _numa_id((uint8_t)-1), _seqnum(0), _virtual(vmem), _top(start()), _livemap(object_max_count()), + _last_used(0), _physical(pmem) { + assert_initialized(); +} + +void ZPage::assert_initialized() const { assert(!_physical.is_null(), "Should not be null"); assert(!_virtual.is_null(), "Should not be null"); - assert((type == ZPageTypeSmall && size() == ZPageSizeSmall) || - (type == ZPageTypeMedium && size() == ZPageSizeMedium) || - (type == ZPageTypeLarge && is_aligned(size(), ZGranuleSize)), + assert((_type == ZPageTypeSmall && size() == ZPageSizeSmall) || + (_type == ZPageTypeMedium && size() == ZPageSizeMedium) || + (_type == ZPageTypeLarge && is_aligned(size(), ZGranuleSize)), "Page type/size mismatch"); } -ZPage::~ZPage() { - assert(_physical.is_null(), "Should be null"); -} - void ZPage::reset() { _seqnum = ZGlobalSeqNum; _top = start(); _livemap.reset(); + _last_used = 0; +} + +ZPage* ZPage::retype(uint8_t type) { + assert(_type != type, "Invalid retype"); + _type = type; + _livemap.resize(object_max_count()); + return this; +} + +ZPage* ZPage::split(size_t size) { + return split(type_from_size(size), size); +} + +ZPage* ZPage::split(uint8_t type, size_t size) { + assert(_virtual.size() > size, "Invalid split"); + + // Resize this page, keep _numa_id, _seqnum, and _last_used + const ZVirtualMemory vmem = _virtual.split(size); + const ZPhysicalMemory pmem = _physical.split(size); + _type = type_from_size(_virtual.size()); + _top = start(); + _livemap.resize(object_max_count()); + + // Create new page, inherit _seqnum and _last_used + ZPage* const page = new ZPage(type, vmem, pmem); + page->_seqnum = _seqnum; + page->_last_used = _last_used; + return page; } void ZPage::print_on(outputStream* out) const { --- old/src/hotspot/share/gc/z/zPage.hpp 2019-05-02 09:57:48.066001333 +0200 +++ new/src/hotspot/share/gc/z/zPage.hpp 2019-05-02 09:57:47.820993242 +0200 @@ -35,26 +35,27 @@ friend class ZList; private: - // Always hot - const uint8_t _type; // Page type - uint8_t _numa_id; // NUMA node affinity - uint32_t _seqnum; // Allocation sequence number - const ZVirtualMemory _virtual; // Virtual start/end address - volatile uintptr_t _top; // Virtual top address - ZLiveMap _livemap; // Live map - - // Hot when relocated and cached - ZPhysicalMemory _physical; // Physical memory for page - ZListNode _node; // Page list node + uint8_t _type; + uint8_t _numa_id; + uint32_t _seqnum; + ZVirtualMemory _virtual; + volatile uintptr_t _top; + ZLiveMap _livemap; + uint64_t _last_used; + ZPhysicalMemory _physical; + ZListNode _node; + void assert_initialized() const; + + uint8_t type_from_size(size_t size) const; const char* type_to_string() const; bool is_object_marked(uintptr_t addr) const; bool is_object_strongly_marked(uintptr_t addr) const; public: - ZPage(uint8_t type, ZVirtualMemory vmem, ZPhysicalMemory pmem); - ~ZPage(); + ZPage(const ZVirtualMemory& vmem, const ZPhysicalMemory& pmem); + ZPage(uint8_t type, const ZVirtualMemory& vmem, const ZPhysicalMemory& pmem); uint32_t object_max_count() const; size_t object_alignment_shift() const; @@ -69,11 +70,18 @@ uint8_t numa_id(); - ZPhysicalMemory& physical_memory(); + uint64_t last_used() const; + void set_last_used(); + + const ZPhysicalMemory& physical_memory() const; const ZVirtualMemory& virtual_memory() const; void reset(); + ZPage* retype(uint8_t type); + ZPage* split(size_t size); + ZPage* split(uint8_t type, size_t size); + bool is_in(uintptr_t addr) const; uintptr_t block_start(uintptr_t addr) const; --- old/src/hotspot/share/gc/z/zPage.inline.hpp 2019-05-02 09:57:48.406012562 +0200 +++ new/src/hotspot/share/gc/z/zPage.inline.hpp 2019-05-02 09:57:48.164004570 +0200 @@ -34,10 +34,23 @@ #include "gc/z/zVirtualMemory.inline.hpp" #include "oops/oop.inline.hpp" #include "runtime/atomic.hpp" -#include "runtime/orderAccess.hpp" +#include "runtime/os.hpp" #include "utilities/align.hpp" #include "utilities/debug.hpp" +inline uint8_t ZPage::type_from_size(size_t size) const { + switch (size) { + case ZPageSizeSmall: + return ZPageTypeSmall; + + case ZPageSizeMedium: + return ZPageTypeMedium; + + default: + return ZPageTypeLarge; + } +} + inline const char* ZPage::type_to_string() const { switch (type()) { case ZPageTypeSmall: @@ -116,7 +129,7 @@ return end() - top(); } -inline ZPhysicalMemory& ZPage::physical_memory() { +inline const ZPhysicalMemory& ZPage::physical_memory() const { return _physical; } @@ -124,6 +137,14 @@ return _virtual; } +inline uint64_t ZPage::last_used() const { + return _last_used; +} + +inline void ZPage::set_last_used() { + _last_used = os::elapsedTime(); +} + inline uint8_t ZPage::numa_id() { if (_numa_id == (uint8_t)-1) { _numa_id = (uint8_t)ZNUMA::memory_id(ZAddress::good(start())); --- old/src/hotspot/share/gc/z/zPageAllocator.cpp 2019-05-02 09:57:48.730023263 +0200 +++ new/src/hotspot/share/gc/z/zPageAllocator.cpp 2019-05-02 09:57:48.509015964 +0200 @@ -30,14 +30,16 @@ #include "gc/z/zPage.inline.hpp" #include "gc/z/zPageAllocator.hpp" #include "gc/z/zPageCache.inline.hpp" -#include "gc/z/zPreMappedMemory.inline.hpp" #include "gc/z/zSafeDelete.inline.hpp" #include "gc/z/zStat.hpp" #include "gc/z/zTracer.inline.hpp" #include "runtime/init.hpp" +#include "runtime/java.hpp" +#include "utilities/debug.hpp" static const ZStatCounter ZCounterAllocationRate("Memory", "Allocation Rate", ZStatUnitBytesPerSecond); -static const ZStatCounter ZCounterPageCacheEvict("Memory", "Page Cache Evict", ZStatUnitBytesPerSecond); +static const ZStatCounter ZCounterPageCacheFlush("Memory", "Page Cache Flush", ZStatUnitBytesPerSecond); +static const ZStatCounter ZCounterUncommit("Memory", "Uncommit", ZStatUnitBytesPerSecond); static const ZStatCriticalPhase ZCriticalPhaseAllocationStall("Allocation Stall"); class ZPageAllocRequest : public StackObj { @@ -85,37 +87,105 @@ ZPage* const ZPageAllocator::gc_marker = (ZPage*)-1; -ZPageAllocator::ZPageAllocator(size_t min_capacity, size_t max_capacity, size_t max_reserve) : +ZPageAllocator::ZPageAllocator(size_t min_capacity, + size_t initial_capacity, + size_t max_capacity, + size_t max_reserve) : _lock(), _virtual(), - _physical(max_capacity), + _physical(), _cache(), + _min_capacity(min_capacity), + _max_capacity(max_capacity), _max_reserve(max_reserve), - _pre_mapped(_virtual, _physical, try_ensure_unused_for_pre_mapped(min_capacity)), + _current_max_capacity(max_capacity), + _capacity(0), _used_high(0), _used_low(0), _used(0), _allocated(0), _reclaimed(0), _queue(), - _safe_delete() {} + _safe_delete(), + _uncommit(false), + _initialized(false) { + + if (!_virtual.is_initialized() || !_physical.is_initialized()) { + return; + } + + log_info(gc, init)("Min Capacity: " SIZE_FORMAT "M", min_capacity / M); + log_info(gc, init)("Initial Capacity: " SIZE_FORMAT "M", initial_capacity / M); + log_info(gc, init)("Max Capacity: " SIZE_FORMAT "M", max_capacity / M); + log_info(gc, init)("Max Reserve: " SIZE_FORMAT "M", max_reserve / M); + log_info(gc, init)("Pre-touch: %s", AlwaysPreTouch ? "Enabled" : "Disabled"); + + // Warn if system limits could stop us from reaching max capacity + _physical.warn_commit_limits(max_capacity); + + // Commit initial capacity + _capacity = _physical.commit(initial_capacity); + if (_capacity != initial_capacity) { + log_error(gc)("Failed to allocate initial Java heap (" SIZE_FORMAT "M)", initial_capacity / M); + return; + } + + // If uncommit is not explicitly disabled, max capacity is greater than + // min capacity, and uncommit is supported by the platform, then we will + // try to uncommit unused memory. + _uncommit = ZUncommit && (max_capacity > min_capacity) && _physical.supports_uncommit(); + if (_uncommit) { + log_info(gc, init)("Uncommit: Enabled, Delay: " UINTX_FORMAT "s", ZUncommitDelay); + } else { + log_info(gc, init)("Uncommit: Disabled"); + } + + // Pre-map initial capacity + prime_cache(initial_capacity); + + // Successfully initialized + _initialized = true; +} + +void ZPageAllocator::prime_cache(size_t size) { + // Allocate physical memory + const ZPhysicalMemory pmem = _physical.alloc(size); + guarantee(!pmem.is_null(), "Invalid size"); + + // Allocate virtual memory + const ZVirtualMemory vmem = _virtual.alloc(size, true /* alloc_from_front */); + guarantee(!vmem.is_null(), "Invalid size"); + + // Allocate page + ZPage* const page = new ZPage(vmem, pmem); + + // Map page + map_page(page); + page->set_pre_mapped(); + + // Add page to cache + page->set_last_used(); + _cache.free_page(page); +} bool ZPageAllocator::is_initialized() const { - return _physical.is_initialized() && - _virtual.is_initialized() && - _pre_mapped.is_initialized(); + return _initialized; +} + +size_t ZPageAllocator::min_capacity() const { + return _min_capacity; } size_t ZPageAllocator::max_capacity() const { - return _physical.max_capacity(); + return _max_capacity; } size_t ZPageAllocator::current_max_capacity() const { - return _physical.current_max_capacity(); + return _current_max_capacity; } size_t ZPageAllocator::capacity() const { - return _physical.capacity(); + return _capacity; } size_t ZPageAllocator::max_reserve() const { @@ -135,7 +205,7 @@ } size_t ZPageAllocator::unused() const { - const ssize_t unused = (ssize_t)_physical.capacity() - (ssize_t)_used - (ssize_t)_max_reserve; + const ssize_t unused = (ssize_t)_capacity - (ssize_t)_used - (ssize_t)_max_reserve; return unused > 0 ? (size_t)unused : 0; } @@ -181,77 +251,34 @@ } } -size_t ZPageAllocator::max_available(bool no_reserve) const { - size_t available = current_max_capacity() - used(); - - if (no_reserve) { - // The reserve should not be considered available - available -= MIN2(available, max_reserve()); - } - - return available; -} - -size_t ZPageAllocator::try_ensure_unused(size_t size, bool no_reserve) { - // Ensure that we always have space available for the reserve. This - // is needed to avoid losing the reserve because of failure to map - // more memory before reaching max capacity. - _physical.try_ensure_unused_capacity(size + max_reserve()); - - size_t unused = _physical.unused_capacity(); - - if (no_reserve) { - // The reserve should not be considered unused - unused -= MIN2(unused, max_reserve()); - } - - return MIN2(size, unused); -} - -size_t ZPageAllocator::try_ensure_unused_for_pre_mapped(size_t size) { - // This function is called during construction, where the - // physical memory manager might have failed to initialied. - if (!_physical.is_initialized()) { - return 0; - } - - return try_ensure_unused(size, true /* no_reserve */); -} - ZPage* ZPageAllocator::create_page(uint8_t type, size_t size) { - // Allocate physical memory - const ZPhysicalMemory pmem = _physical.alloc(size); - if (pmem.is_null()) { - // Out of memory - return NULL; - } - // Allocate virtual memory const ZVirtualMemory vmem = _virtual.alloc(size); if (vmem.is_null()) { // Out of address space - _physical.free(pmem); return NULL; } + // Allocate physical memory + const ZPhysicalMemory pmem = _physical.alloc(size); + assert(!pmem.is_null(), "Invalid size"); + // Allocate page return new ZPage(type, vmem, pmem); } -void ZPageAllocator::flush_pre_mapped() { - if (_pre_mapped.available() == 0) { - return; - } +void ZPageAllocator::destroy_page(ZPage* page) { + const ZVirtualMemory& vmem = page->virtual_memory(); + const ZPhysicalMemory& pmem = page->physical_memory(); - // Detach the memory mapping. - detach_memory(_pre_mapped.virtual_memory(), _pre_mapped.physical_memory()); + // Unmap memory + _physical.unmap(pmem, vmem.start()); - _pre_mapped.clear(); -} + // Free physical memory + _physical.free(pmem); -void ZPageAllocator::destroy_page(ZPage* page) { - // Detach virtual and physical memory - detach_memory(page->virtual_memory(), page->physical_memory()); + // Free virtual memory + _virtual.free(vmem); // Delete page safely _safe_delete(page); @@ -267,56 +294,96 @@ } void ZPageAllocator::unmap_all_pages() { - ZPhysicalMemory pmem(ZPhysicalMemorySegment(0 /* start */, ZAddressOffsetMax)); - _physical.debug_unmap(pmem, 0 /* offset */); - pmem.clear(); + // Unmap all physical memory + _physical.debug_unmap(ZPhysicalMemorySegment(0 /* start */, ZAddressOffsetMax), 0 /* offset */); } -void ZPageAllocator::check_out_of_memory_during_initialization() { - if (!is_init_completed()) { - vm_exit_during_initialization("java.lang.OutOfMemoryError", "Java heap too small"); +size_t ZPageAllocator::max_available(bool no_reserve) const { + size_t available = _current_max_capacity - _used; + + if (no_reserve) { + // The reserve should not be considered available + available -= MIN2(available, _max_reserve); } + + return available; } -ZPage* ZPageAllocator::alloc_page_common_inner(uint8_t type, size_t size, ZAllocationFlags flags) { - const size_t max = max_available(flags.no_reserve()); - if (max < size) { +bool ZPageAllocator::ensure_available(size_t size, bool no_reserve) { + if (max_available(no_reserve) < size) { // Not enough free memory - return NULL; + return false; } - // Try allocating from the page cache - ZPage* const cached_page = _cache.alloc_page(type, size); - if (cached_page != NULL) { - return cached_page; - } - - // Try allocate from the pre-mapped memory - ZPage* const pre_mapped_page = _pre_mapped.alloc_page(type, size); - if (pre_mapped_page != NULL) { - return pre_mapped_page; - } - - // Flush any remaining pre-mapped memory so that - // subsequent allocations can use the physical memory. - flush_pre_mapped(); - - // Try ensure that physical memory is available - const size_t unused = try_ensure_unused(size, flags.no_reserve()); - if (unused < size) { - // Try evict pages from the cache - const size_t needed = size - unused; - if (_cache.available() >= needed) { - evict_cache(needed); + // We add the max_reserve to the requested size to avoid losing + // the reserve because of failure to increase capacity before + // reaching max capacity. + size += _max_reserve; + + // Don't try to increase capacity if enough unused capacity + // is available or if current max capacity has been reached. + const size_t available = _capacity - _used; + if (available < size && _capacity < _current_max_capacity) { + // Try to increase capacity + const size_t commit = MIN2(size - available, _current_max_capacity - _capacity); + const size_t committed = _physical.commit(commit); + _capacity += committed; + + log_trace(gc, heap)("Make Available: Size: " SIZE_FORMAT "M, NoReserve: %s, " + "Available: " SIZE_FORMAT "M, Commit: " SIZE_FORMAT "M, " + "Committed: " SIZE_FORMAT "M, Capacity: " SIZE_FORMAT "M", + size / M, no_reserve ? "True" : "False", available / M, + commit / M, committed / M, _capacity / M); + + if (committed != commit) { + // Failed, or partly failed, to increase capacity. Adjust current + // max capacity to avoid further attempts to increase capacity. + log_error(gc)("Forced to lower max Java heap size from " + SIZE_FORMAT "M(%.0lf%%) to " SIZE_FORMAT "M(%.0lf%%)", + _current_max_capacity / M, percent_of(_current_max_capacity, _max_capacity), + _capacity / M, percent_of(_capacity, _max_capacity)); + + _current_max_capacity = _capacity; } } - // Create new page and allocate physical memory + if (!no_reserve) { + size -= _max_reserve; + } + + const size_t new_available = _capacity - _used; + return new_available >= size; +} + +void ZPageAllocator::ensure_uncached_available(size_t size) { + assert(_capacity - _used >= size, "Invalid size"); + const size_t uncached_available = _capacity - _used - _cache.available(); + if (size > uncached_available) { + flush_cache_for_allocation(size - uncached_available); + } +} + +ZPage* ZPageAllocator::alloc_page_common_inner(uint8_t type, size_t size, bool no_reserve) { + if (!ensure_available(size, no_reserve)) { + // Not enough free memory + return NULL; + } + + // Try allocate page from the cache + ZPage* const page = _cache.alloc_page(type, size); + if (page != NULL) { + return page; + } + + // Try flush pages from the cache + ensure_uncached_available(size); + + // Create new page return create_page(type, size); } ZPage* ZPageAllocator::alloc_page_common(uint8_t type, size_t size, ZAllocationFlags flags) { - ZPage* const page = alloc_page_common_inner(type, size, flags); + ZPage* const page = alloc_page_common_inner(type, size, flags.no_reserve()); if (page == NULL) { // Out of memory return NULL; @@ -326,11 +393,17 @@ increase_used(size, flags.relocation()); // Send trace event - ZTracer::tracer()->report_page_alloc(size, used(), max_available(flags.no_reserve()), _cache.available(), flags); + ZTracer::tracer()->report_page_alloc(size, _used, max_available(flags.no_reserve()), _cache.available(), flags); return page; } +void ZPageAllocator::check_out_of_memory_during_initialization() { + if (!is_init_completed()) { + vm_exit_during_initialization("java.lang.OutOfMemoryError", "Java heap too small"); + } +} + ZPage* ZPageAllocator::alloc_page_blocking(uint8_t type, size_t size, ZAllocationFlags flags) { // Prepare to block ZPageAllocRequest request(type, size, flags, ZCollectedHeap::heap()->total_collections()); @@ -433,28 +506,15 @@ } } -void ZPageAllocator::detach_memory(const ZVirtualMemory& vmem, ZPhysicalMemory& pmem) { - const uintptr_t addr = vmem.start(); - - // Free virtual memory - _virtual.free(vmem); - - // Unmap physical memory - _physical.unmap(pmem, addr); - - // Free physical memory - _physical.free(pmem); - - // Clear physical mapping - pmem.clear(); -} - void ZPageAllocator::free_page(ZPage* page, bool reclaimed) { ZLocker locker(&_lock); // Update used statistics decrease_used(page->size(), reclaimed); + // Set time when last used + page->set_last_used(); + // Cache page _cache.free_page(page); @@ -462,59 +522,156 @@ satisfy_alloc_queue(); } -void ZPageAllocator::flush_cache(ZPageCacheFlushClosure* cl) { +size_t ZPageAllocator::flush_cache(ZPageCacheFlushClosure* cl) { ZList list; + // Flush pages _cache.flush(cl, &list); + const size_t overflushed = cl->overflushed(); + if (overflushed > 0) { + // Overflushed, keep part of last page + ZPage* const page = list.last()->split(overflushed); + _cache.free_page(page); + } + + // Destroy pages + size_t flushed = 0; for (ZPage* page = list.remove_first(); page != NULL; page = list.remove_first()) { + flushed += page->size(); destroy_page(page); } -} -class ZPageCacheEvictClosure : public ZPageCacheFlushClosure { -private: - const size_t _requested; - size_t _evicted; + return flushed; +} +class ZPageCacheFlushForAllocationClosure : public ZPageCacheFlushClosure { public: - ZPageCacheEvictClosure(size_t requested) : - _requested(requested), - _evicted(0) {} + ZPageCacheFlushForAllocationClosure(size_t requested) : + ZPageCacheFlushClosure(requested) {} virtual bool do_page(const ZPage* page) { - if (_evicted < _requested) { - // Evict page - _evicted += page->size(); + if (_flushed < _requested) { + // Flush page + _flushed += page->size(); return true; } - // Don't evict page + // Don't flush page return false; } - - size_t evicted() const { - return _evicted; - } }; -void ZPageAllocator::evict_cache(size_t requested) { - // Evict pages - ZPageCacheEvictClosure cl(requested); - flush_cache(&cl); +void ZPageAllocator::flush_cache_for_allocation(size_t requested) { + assert(requested <= _cache.available(), "Invalid request"); + + // Flush pages + ZPageCacheFlushForAllocationClosure cl(requested); + const size_t flushed = flush_cache(&cl); + + assert(requested == flushed, "Failed to flush"); - const size_t evicted = cl.evicted(); const size_t cached_after = _cache.available(); - const size_t cached_before = cached_after + evicted; + const size_t cached_before = cached_after + flushed; log_info(gc, heap)("Page Cache: " SIZE_FORMAT "M(%.0lf%%)->" SIZE_FORMAT "M(%.0lf%%), " - "Evicted: " SIZE_FORMAT "M, Requested: " SIZE_FORMAT "M", + "Flushed: " SIZE_FORMAT "M", cached_before / M, percent_of(cached_before, max_capacity()), cached_after / M, percent_of(cached_after, max_capacity()), - evicted / M, requested / M); + flushed / M); // Update statistics - ZStatInc(ZCounterPageCacheEvict, evicted); + ZStatInc(ZCounterPageCacheFlush, flushed); +} + +class ZPageCacheFlushForUncommitClosure : public ZPageCacheFlushClosure { +private: + const uint64_t _now; + const uint64_t _delay; + uint64_t _timeout; + +public: + ZPageCacheFlushForUncommitClosure(size_t requested, uint64_t delay) : + ZPageCacheFlushClosure(requested), + _now(os::elapsedTime()), + _delay(delay), + _timeout(_delay) {} + + virtual bool do_page(const ZPage* page) { + const uint64_t expires = page->last_used() + _delay; + const uint64_t timeout = expires - MIN2(expires, _now); + + if (_flushed < _requested && timeout == 0) { + // Flush page + _flushed += page->size(); + return true; + } + + // Record shortest non-expired timeout + _timeout = MIN2(_timeout, timeout); + + // Don't flush page + return false; + } + + uint64_t timeout() const { + return _timeout; + } +}; + +uint64_t ZPageAllocator::uncommit(uint64_t delay) { + // Set the default timeout, when no pages are found in the + // cache or when uncommit is disabled, equal to the delay. + uint64_t timeout = delay; + + if (!_uncommit) { + // Disabled + return timeout; + } + + size_t capacity_before; + size_t capacity_after; + size_t uncommitted; + + { + ZLocker locker(&_lock); + + // Don't flush more than we will uncommit. Never uncommit + // the reserve, and never uncommit below min capacity. + const size_t needed = MIN2(_used + _max_reserve, _current_max_capacity); + const size_t guarded = MAX2(needed, _min_capacity); + const size_t uncommittable = _capacity - guarded; + const size_t uncached_available = _capacity - _used - _cache.available(); + size_t uncommit = MIN2(uncommittable, uncached_available); + const size_t flush = uncommittable - uncommit; + + if (flush > 0) { + // Flush pages to uncommit + ZPageCacheFlushForUncommitClosure cl(flush, delay); + uncommit += flush_cache(&cl); + timeout = cl.timeout(); + } + + // Uncommit + uncommitted = _physical.uncommit(uncommit); + _capacity -= uncommitted; + + capacity_after = _capacity; + capacity_before = capacity_after + uncommitted; + } + + if (uncommitted > 0) { + log_info(gc, heap)("Capacity: " SIZE_FORMAT "M(%.0lf%%)->" SIZE_FORMAT "M(%.0lf%%), " + "Uncommitted: " SIZE_FORMAT "M", + capacity_before / M, percent_of(capacity_before, max_capacity()), + capacity_after / M, percent_of(capacity_after, max_capacity()), + uncommitted / M); + + // Update statistics + ZStatInc(ZCounterUncommit, uncommitted); + } + + return timeout; } void ZPageAllocator::enable_deferred_delete() const { --- old/src/hotspot/share/gc/z/zPageAllocator.hpp 2019-05-02 09:57:49.079034789 +0200 +++ new/src/hotspot/share/gc/z/zPageAllocator.hpp 2019-05-02 09:57:48.856027424 +0200 @@ -29,7 +29,6 @@ #include "gc/z/zLock.hpp" #include "gc/z/zPageCache.hpp" #include "gc/z/zPhysicalMemory.hpp" -#include "gc/z/zPreMappedMemory.hpp" #include "gc/z/zSafeDelete.hpp" #include "gc/z/zVirtualMemory.hpp" #include "memory/allocation.hpp" @@ -44,8 +43,11 @@ ZVirtualMemoryManager _virtual; ZPhysicalMemoryManager _physical; ZPageCache _cache; + const size_t _min_capacity; + const size_t _max_capacity; const size_t _max_reserve; - ZPreMappedMemory _pre_mapped; + size_t _current_max_capacity; + size_t _capacity; size_t _used_high; size_t _used_low; size_t _used; @@ -53,39 +55,44 @@ ssize_t _reclaimed; ZList _queue; mutable ZSafeDelete _safe_delete; + bool _uncommit; + bool _initialized; static ZPage* const gc_marker; + void prime_cache(size_t size); + void increase_used(size_t size, bool relocation); void decrease_used(size_t size, bool reclaimed); - size_t max_available(bool no_reserve) const; - size_t try_ensure_unused(size_t size, bool no_reserve); - size_t try_ensure_unused_for_pre_mapped(size_t size); - ZPage* create_page(uint8_t type, size_t size); void destroy_page(ZPage* page); - void flush_pre_mapped(); - void flush_cache(ZPageCacheFlushClosure* cl); - void evict_cache(size_t requested); + size_t max_available(bool no_reserve) const; + bool ensure_available(size_t size, bool no_reserve); + void ensure_uncached_available(size_t size); void check_out_of_memory_during_initialization(); - ZPage* alloc_page_common_inner(uint8_t type, size_t size, ZAllocationFlags flags); + ZPage* alloc_page_common_inner(uint8_t type, size_t size, bool no_reserve); ZPage* alloc_page_common(uint8_t type, size_t size, ZAllocationFlags flags); ZPage* alloc_page_blocking(uint8_t type, size_t size, ZAllocationFlags flags); ZPage* alloc_page_nonblocking(uint8_t type, size_t size, ZAllocationFlags flags); - void satisfy_alloc_queue(); + size_t flush_cache(ZPageCacheFlushClosure* cl); + void flush_cache_for_allocation(size_t requested); - void detach_memory(const ZVirtualMemory& vmem, ZPhysicalMemory& pmem); + void satisfy_alloc_queue(); public: - ZPageAllocator(size_t min_capacity, size_t max_capacity, size_t max_reserve); + ZPageAllocator(size_t min_capacity, + size_t initial_capacity, + size_t max_capacity, + size_t max_reserve); bool is_initialized() const; + size_t min_capacity() const; size_t max_capacity() const; size_t current_max_capacity() const; size_t capacity() const; @@ -102,6 +109,8 @@ ZPage* alloc_page(uint8_t type, size_t size, ZAllocationFlags flags); void free_page(ZPage* page, bool reclaimed); + uint64_t uncommit(uint64_t delay); + void enable_deferred_delete() const; void disable_deferred_delete() const; --- old/src/hotspot/share/gc/z/zPageCache.cpp 2019-05-02 09:57:49.428046315 +0200 +++ new/src/hotspot/share/gc/z/zPageCache.cpp 2019-05-02 09:57:49.177038025 +0200 @@ -31,8 +31,17 @@ static const ZStatCounter ZCounterPageCacheHitL1("Memory", "Page Cache Hit L1", ZStatUnitOpsPerSecond); static const ZStatCounter ZCounterPageCacheHitL2("Memory", "Page Cache Hit L2", ZStatUnitOpsPerSecond); +static const ZStatCounter ZCounterPageCacheHitL3("Memory", "Page Cache Hit L3", ZStatUnitOpsPerSecond); static const ZStatCounter ZCounterPageCacheMiss("Memory", "Page Cache Miss", ZStatUnitOpsPerSecond); +ZPageCacheFlushClosure::ZPageCacheFlushClosure(size_t requested) : + _requested(requested), + _flushed(0) {} + +size_t ZPageCacheFlushClosure::overflushed() const { + return _flushed > _requested ? _flushed - _requested : 0; +} + ZPageCache::ZPageCache() : _available(0), _small(), @@ -67,40 +76,73 @@ remote_numa_id++; } - ZStatInc(ZCounterPageCacheMiss); return NULL; } ZPage* ZPageCache::alloc_medium_page() { - ZPage* const l1_page = _medium.remove_first(); - if (l1_page != NULL) { + ZPage* const page = _medium.remove_first(); + if (page != NULL) { ZStatInc(ZCounterPageCacheHitL1); - return l1_page; + return page; } - ZStatInc(ZCounterPageCacheMiss); return NULL; } ZPage* ZPageCache::alloc_large_page(size_t size) { // Find a page with the right size ZListIterator iter(&_large); - for (ZPage* l1_page; iter.next(&l1_page);) { - if (l1_page->size() == size) { + for (ZPage* page; iter.next(&page);) { + if (size == page->size()) { // Page found - _large.remove(l1_page); + _large.remove(page); ZStatInc(ZCounterPageCacheHitL1); - return l1_page; + return page; + } + } + + return NULL; +} + +ZPage* ZPageCache::alloc_oversized_medium_page(size_t size) { + if (size <= ZPageSizeMedium) { + return _medium.remove_first(); + } + + return NULL; +} + +ZPage* ZPageCache::alloc_oversized_large_page(size_t size) { + // Find a page that is large enough + ZListIterator iter(&_large); + for (ZPage* page; iter.next(&page);) { + if (size <= page->size()) { + // Page found + _large.remove(page); + return page; } } - ZStatInc(ZCounterPageCacheMiss); return NULL; } +ZPage* ZPageCache::alloc_oversized_page(size_t size) { + ZPage* page = alloc_oversized_large_page(size); + if (page == NULL) { + page = alloc_oversized_medium_page(size); + } + + if (page != NULL) { + ZStatInc(ZCounterPageCacheHitL3); + } + + return page; +} + ZPage* ZPageCache::alloc_page(uint8_t type, size_t size) { ZPage* page; + // Try allocate exact page if (type == ZPageTypeSmall) { page = alloc_small_page(); } else if (type == ZPageTypeMedium) { @@ -109,14 +151,33 @@ page = alloc_large_page(size); } + if (page == NULL) { + // Try allocate potentially oversized page + ZPage* const oversized = alloc_oversized_page(size); + if (oversized != NULL) { + if (size < oversized->size()) { + // Split oversized page + page = oversized->split(type, size); + + // Cache remainder + free_page_inner(oversized); + } else { + // Re-type correctly sized page + page = oversized->retype(type); + } + } + } + if (page != NULL) { _available -= page->size(); + } else { + ZStatInc(ZCounterPageCacheMiss); } return page; } -void ZPageCache::free_page(ZPage* page) { +void ZPageCache::free_page_inner(ZPage* page) { const uint8_t type = page->type(); if (type == ZPageTypeSmall) { _small.get(page->numa_id()).insert_first(page); @@ -125,7 +186,10 @@ } else { _large.insert_first(page); } +} +void ZPageCache::free_page(ZPage* page) { + free_page_inner(page); _available += page->size(); } --- old/src/hotspot/share/gc/z/zPageCache.hpp 2019-05-02 09:57:49.784058072 +0200 +++ new/src/hotspot/share/gc/z/zPageCache.hpp 2019-05-02 09:57:49.539049981 +0200 @@ -30,7 +30,13 @@ #include "memory/allocation.hpp" class ZPageCacheFlushClosure : public StackObj { +protected: + const size_t _requested; + size_t _flushed; + public: + ZPageCacheFlushClosure(size_t requested); + size_t overflushed() const; virtual bool do_page(const ZPage* page) = 0; }; @@ -45,6 +51,12 @@ ZPage* alloc_medium_page(); ZPage* alloc_large_page(size_t size); + ZPage* alloc_oversized_medium_page(size_t size); + ZPage* alloc_oversized_large_page(size_t size); + ZPage* alloc_oversized_page(size_t size); + + void free_page_inner(ZPage* page); + bool flush_list_inner(ZPageCacheFlushClosure* cl, ZList* from, ZList* to); void flush_list(ZPageCacheFlushClosure* cl, ZList* from, ZList* to); void flush_per_numa_lists(ZPageCacheFlushClosure* cl, ZPerNUMA >* from, ZList* to); --- old/src/hotspot/share/gc/z/zPhysicalMemory.cpp 2019-05-02 09:57:50.131069532 +0200 +++ new/src/hotspot/share/gc/z/zPhysicalMemory.cpp 2019-05-02 09:57:49.887061474 +0200 @@ -33,16 +33,40 @@ _nsegments(0), _segments(NULL) {} -ZPhysicalMemory::ZPhysicalMemory(size_t size) : +ZPhysicalMemory::ZPhysicalMemory(const ZPhysicalMemorySegment& segment) : _nsegments(0), _segments(NULL) { - add_segment(ZPhysicalMemorySegment(0, size)); + add_segment(segment); } -ZPhysicalMemory::ZPhysicalMemory(const ZPhysicalMemorySegment& segment) : +ZPhysicalMemory::ZPhysicalMemory(const ZPhysicalMemory& pmem) : _nsegments(0), _segments(NULL) { - add_segment(segment); + + // Copy segments + for (size_t i = 0; i < pmem.nsegments(); i++) { + add_segment(pmem.segment(i)); + } +} + +const ZPhysicalMemory& ZPhysicalMemory::operator=(const ZPhysicalMemory& pmem) { + // Free segments + delete [] _segments; + _segments = NULL; + _nsegments = 0; + + // Copy segments + for (size_t i = 0; i < pmem.nsegments(); i++) { + add_segment(pmem.segment(i)); + } + + return *this; +} + +ZPhysicalMemory::~ZPhysicalMemory() { + delete [] _segments; + _segments = NULL; + _nsegments = 0; } size_t ZPhysicalMemory::size() const { @@ -55,134 +79,114 @@ return size; } -void ZPhysicalMemory::add_segment(ZPhysicalMemorySegment segment) { +void ZPhysicalMemory::add_segment(const ZPhysicalMemorySegment& segment) { // Try merge with last segment if (_nsegments > 0) { ZPhysicalMemorySegment& last = _segments[_nsegments - 1]; assert(last.end() <= segment.start(), "Segments added out of order"); if (last.end() == segment.start()) { - // Merge - last.expand(segment.size()); + last = ZPhysicalMemorySegment(last.start(), last.size() + segment.size()); return; } } - // Make room for a new segment - const size_t size = sizeof(ZPhysicalMemorySegment) * (_nsegments + 1); - _segments = (ZPhysicalMemorySegment*)ReallocateHeap((char*)_segments, size, mtGC); + // Resize array + ZPhysicalMemorySegment* const old_segments = _segments; + _segments = new ZPhysicalMemorySegment[_nsegments + 1]; + for (size_t i = 0; i < _nsegments; i++) { + _segments[i] = old_segments[i]; + } + delete [] old_segments; // Add new segment _segments[_nsegments] = segment; _nsegments++; } -ZPhysicalMemory ZPhysicalMemory::split(size_t split_size) { - // Only splitting of single-segment instances have been implemented. - assert(nsegments() == 1, "Can only have one segment"); - assert(split_size <= size(), "Invalid size"); - return ZPhysicalMemory(_segments[0].split(split_size)); -} - -void ZPhysicalMemory::clear() { - if (_segments != NULL) { - FreeHeap(_segments); - _segments = NULL; - _nsegments = 0; +ZPhysicalMemory ZPhysicalMemory::split(size_t size) { + ZPhysicalMemory pmem; + size_t nsegments = 0; + + for (size_t i = 0; i < _nsegments; i++) { + const ZPhysicalMemorySegment& segment = _segments[i]; + if (pmem.size() < size) { + if (pmem.size() + segment.size() <= size) { + // Transfer segment + pmem.add_segment(segment); + } else { + // Split segment + const size_t split_size = size - pmem.size(); + pmem.add_segment(ZPhysicalMemorySegment(segment.start(), split_size)); + _segments[nsegments++] = ZPhysicalMemorySegment(segment.start() + split_size, segment.size() - split_size); + } + } else { + // Keep segment + _segments[nsegments++] = segment; + } } -} -ZPhysicalMemoryManager::ZPhysicalMemoryManager(size_t max_capacity) : - _backing(max_capacity), - _max_capacity(max_capacity), - _current_max_capacity(max_capacity), - _capacity(0), - _used(0) {} + _nsegments = nsegments; + + return pmem; +} bool ZPhysicalMemoryManager::is_initialized() const { return _backing.is_initialized(); } -void ZPhysicalMemoryManager::try_ensure_unused_capacity(size_t size) { - const size_t unused = unused_capacity(); - if (unused >= size) { - // Don't try to expand, enough unused capacity available - return; - } - - const size_t current_max = current_max_capacity(); - if (_capacity == current_max) { - // Don't try to expand, current max capacity reached - return; - } - - // Try to expand - const size_t old_capacity = capacity(); - const size_t new_capacity = MIN2(old_capacity + size - unused, current_max); - _capacity = _backing.try_expand(old_capacity, new_capacity); - - if (_capacity != new_capacity) { - // Failed, or partly failed, to expand - log_error(gc, init)("Not enough space available on the backing filesystem to hold the current max"); - log_error(gc, init)("Java heap size (" SIZE_FORMAT "M). Forcefully lowering max Java heap size to " - SIZE_FORMAT "M (%.0lf%%).", current_max / M, _capacity / M, - percent_of(_capacity, current_max)); +void ZPhysicalMemoryManager::warn_commit_limits(size_t max) const { + _backing.warn_commit_limits(max); +} - // Adjust current max capacity to avoid further expand attempts - _current_max_capacity = _capacity; - } +bool ZPhysicalMemoryManager::supports_uncommit() { + return _backing.supports_uncommit(); } -void ZPhysicalMemoryManager::nmt_commit(ZPhysicalMemory pmem, uintptr_t offset) { +void ZPhysicalMemoryManager::nmt_commit(const ZPhysicalMemory& pmem, uintptr_t offset) { const uintptr_t addr = _backing.nmt_address(offset); const size_t size = pmem.size(); MemTracker::record_virtual_memory_commit((void*)addr, size, CALLER_PC); } -void ZPhysicalMemoryManager::nmt_uncommit(ZPhysicalMemory pmem, uintptr_t offset) { +void ZPhysicalMemoryManager::nmt_uncommit(const ZPhysicalMemory& pmem, uintptr_t offset) { if (MemTracker::tracking_level() > NMT_minimal) { const uintptr_t addr = _backing.nmt_address(offset); const size_t size = pmem.size(); - Tracker tracker(Tracker::uncommit); tracker.record((address)addr, size); } } -ZPhysicalMemory ZPhysicalMemoryManager::alloc(size_t size) { - if (unused_capacity() < size) { - // Not enough memory available - return ZPhysicalMemory(); - } +size_t ZPhysicalMemoryManager::commit(size_t size) { + return _backing.commit(size); +} - _used += size; +size_t ZPhysicalMemoryManager::uncommit(size_t size) { + return _backing.uncommit(size); +} + +ZPhysicalMemory ZPhysicalMemoryManager::alloc(size_t size) { return _backing.alloc(size); } -void ZPhysicalMemoryManager::free(ZPhysicalMemory pmem) { +void ZPhysicalMemoryManager::free(const ZPhysicalMemory& pmem) { _backing.free(pmem); - _used -= pmem.size(); } -void ZPhysicalMemoryManager::map(ZPhysicalMemory pmem, uintptr_t offset) { - // Map page +void ZPhysicalMemoryManager::map(const ZPhysicalMemory& pmem, uintptr_t offset) { _backing.map(pmem, offset); - - // Update native memory tracker nmt_commit(pmem, offset); } -void ZPhysicalMemoryManager::unmap(ZPhysicalMemory pmem, uintptr_t offset) { - // Update native memory tracker +void ZPhysicalMemoryManager::unmap(const ZPhysicalMemory& pmem, uintptr_t offset) { nmt_uncommit(pmem, offset); - - // Unmap page _backing.unmap(pmem, offset); } -void ZPhysicalMemoryManager::debug_map(ZPhysicalMemory pmem, uintptr_t offset) { +void ZPhysicalMemoryManager::debug_map(const ZPhysicalMemory& pmem, uintptr_t offset) { _backing.debug_map(pmem, offset); } -void ZPhysicalMemoryManager::debug_unmap(ZPhysicalMemory pmem, uintptr_t offset) { +void ZPhysicalMemoryManager::debug_unmap(const ZPhysicalMemory& pmem, uintptr_t offset) { _backing.debug_unmap(pmem, offset); } --- old/src/hotspot/share/gc/z/zPhysicalMemory.hpp 2019-05-02 09:57:50.483081157 +0200 +++ new/src/hotspot/share/gc/z/zPhysicalMemory.hpp 2019-05-02 09:57:50.239073099 +0200 @@ -27,20 +27,18 @@ #include "memory/allocation.hpp" #include OS_CPU_HEADER(gc/z/zPhysicalMemoryBacking) -class ZPhysicalMemorySegment { +class ZPhysicalMemorySegment : public CHeapObj { private: uintptr_t _start; uintptr_t _end; public: + ZPhysicalMemorySegment(); ZPhysicalMemorySegment(uintptr_t start, size_t size); uintptr_t start() const; uintptr_t end() const; size_t size() const; - - void expand(size_t size); - ZPhysicalMemorySegment split(size_t size); }; class ZPhysicalMemory { @@ -50,53 +48,45 @@ public: ZPhysicalMemory(); - ZPhysicalMemory(size_t size); ZPhysicalMemory(const ZPhysicalMemorySegment& segment); + ZPhysicalMemory(const ZPhysicalMemory& pmem); + const ZPhysicalMemory& operator=(const ZPhysicalMemory& pmem); + ~ZPhysicalMemory(); bool is_null() const; size_t size() const; size_t nsegments() const; - ZPhysicalMemorySegment segment(size_t index) const; - void add_segment(ZPhysicalMemorySegment segment); + const ZPhysicalMemorySegment& segment(size_t index) const; + void add_segment(const ZPhysicalMemorySegment& segment); ZPhysicalMemory split(size_t size); - void clear(); }; class ZPhysicalMemoryManager { - friend class VMStructs; - private: ZPhysicalMemoryBacking _backing; - const size_t _max_capacity; - size_t _current_max_capacity; - size_t _capacity; - size_t _used; - void nmt_commit(ZPhysicalMemory pmem, uintptr_t offset); - void nmt_uncommit(ZPhysicalMemory pmem, uintptr_t offset); + void nmt_commit(const ZPhysicalMemory& pmem, uintptr_t offset); + void nmt_uncommit(const ZPhysicalMemory& pmem, uintptr_t offset); public: - ZPhysicalMemoryManager(size_t max_capacity); - bool is_initialized() const; - size_t max_capacity() const; - size_t current_max_capacity() const; - size_t capacity() const; - size_t unused_capacity() const; + void warn_commit_limits(size_t max) const; + bool supports_uncommit(); - void try_ensure_unused_capacity(size_t size); + size_t commit(size_t size); + size_t uncommit(size_t size); ZPhysicalMemory alloc(size_t size); - void free(ZPhysicalMemory pmem); + void free(const ZPhysicalMemory& pmem); - void map(ZPhysicalMemory pmem, uintptr_t offset); - void unmap(ZPhysicalMemory pmem, uintptr_t offset); + void map(const ZPhysicalMemory& pmem, uintptr_t offset); + void unmap(const ZPhysicalMemory& pmem, uintptr_t offset); - void debug_map(ZPhysicalMemory pmem, uintptr_t offset); - void debug_unmap(ZPhysicalMemory pmem, uintptr_t offset); + void debug_map(const ZPhysicalMemory& pmem, uintptr_t offset); + void debug_unmap(const ZPhysicalMemory& pmem, uintptr_t offset); }; #endif // SHARE_GC_Z_ZPHYSICALMEMORY_HPP --- old/src/hotspot/share/gc/z/zPhysicalMemory.inline.hpp 2019-05-02 09:57:50.828092551 +0200 +++ new/src/hotspot/share/gc/z/zPhysicalMemory.inline.hpp 2019-05-02 09:57:50.579084328 +0200 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015, 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -27,6 +27,10 @@ #include "gc/z/zPhysicalMemory.hpp" #include "utilities/debug.hpp" +inline ZPhysicalMemorySegment::ZPhysicalMemorySegment() : + _start(UINTPTR_MAX), + _end(UINTPTR_MAX) {} + inline ZPhysicalMemorySegment::ZPhysicalMemorySegment(uintptr_t start, size_t size) : _start(start), _end(start + size) {} @@ -40,18 +44,7 @@ } inline size_t ZPhysicalMemorySegment::size() const { - return end() - start(); -} - -inline void ZPhysicalMemorySegment::expand(size_t size) { - _end += size; -} - -inline ZPhysicalMemorySegment ZPhysicalMemorySegment::split(size_t split_size) { - assert(split_size <= size(), "Invalid size"); - ZPhysicalMemorySegment segment(_start, split_size); - _start += split_size; - return segment; + return _end - _start; } inline bool ZPhysicalMemory::is_null() const { @@ -62,25 +55,9 @@ return _nsegments; } -inline ZPhysicalMemorySegment ZPhysicalMemory::segment(size_t index) const { +inline const ZPhysicalMemorySegment& ZPhysicalMemory::segment(size_t index) const { assert(index < _nsegments, "Invalid segment index"); return _segments[index]; } -inline size_t ZPhysicalMemoryManager::max_capacity() const { - return _max_capacity; -} - -inline size_t ZPhysicalMemoryManager::current_max_capacity() const { - return _current_max_capacity; -} - -inline size_t ZPhysicalMemoryManager::capacity() const { - return _capacity; -} - -inline size_t ZPhysicalMemoryManager::unused_capacity() const { - return _capacity - _used; -} - #endif // SHARE_GC_Z_ZPHYSICALMEMORY_INLINE_HPP --- old/src/hotspot/share/gc/z/zVirtualMemory.cpp 2019-05-02 09:57:51.159103483 +0200 +++ new/src/hotspot/share/gc/z/zVirtualMemory.cpp 2019-05-02 09:57:50.922095656 +0200 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015, 2017, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -72,6 +72,6 @@ return ZVirtualMemory(start, size); } -void ZVirtualMemoryManager::free(ZVirtualMemory vmem) { +void ZVirtualMemoryManager::free(const ZVirtualMemory& vmem) { _manager.free(vmem.start(), vmem.size()); } --- old/src/hotspot/share/gc/z/zVirtualMemory.hpp 2019-05-02 09:57:51.474113886 +0200 +++ new/src/hotspot/share/gc/z/zVirtualMemory.hpp 2019-05-02 09:57:51.252106554 +0200 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015, 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -25,7 +25,6 @@ #define SHARE_GC_Z_ZVIRTUALMEMORY_HPP #include "gc/z/zMemory.hpp" -#include "memory/allocation.hpp" class ZVirtualMemory { friend class VMStructs; @@ -42,8 +41,8 @@ uintptr_t start() const; uintptr_t end() const; size_t size() const; + ZVirtualMemory split(size_t size); - void clear(); }; class ZVirtualMemoryManager { @@ -60,7 +59,7 @@ bool is_initialized() const; ZVirtualMemory alloc(size_t size, bool alloc_from_front = false); - void free(ZVirtualMemory vmem); + void free(const ZVirtualMemory& vmem); }; #endif // SHARE_GC_Z_ZVIRTUALMEMORY_HPP --- old/src/hotspot/share/gc/z/zVirtualMemory.inline.hpp 2019-05-02 09:57:51.817125214 +0200 +++ new/src/hotspot/share/gc/z/zVirtualMemory.inline.hpp 2019-05-02 09:57:51.570117057 +0200 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015, 2017, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -51,16 +51,9 @@ return _end - _start; } -inline ZVirtualMemory ZVirtualMemory::split(size_t split_size) { - assert(split_size <= size(), "precondition"); - ZVirtualMemory mem(_start, split_size); - _start += split_size; - return mem; -} - -inline void ZVirtualMemory::clear() { - _start = UINTPTR_MAX; - _end = UINTPTR_MAX; +inline ZVirtualMemory ZVirtualMemory::split(size_t size) { + _start += size; + return ZVirtualMemory(_start - size, size); } #endif // SHARE_GC_Z_ZVIRTUALMEMORY_INLINE_HPP --- old/src/hotspot/share/gc/z/z_globals.hpp 2019-05-02 09:57:52.164136674 +0200 +++ new/src/hotspot/share/gc/z/z_globals.hpp 2019-05-02 09:57:51.915128451 +0200 @@ -56,6 +56,13 @@ experimental(uint, ZCollectionInterval, 0, \ "Force GC at a fixed time interval (in seconds)") \ \ + experimental(bool, ZUncommit, true, \ + "Uncommit unused memory") \ + \ + experimental(uintx, ZUncommitDelay, 5 * 60, \ + "Uncommit memory if it has been unused for the specified " \ + "amount of time (in seconds)") \ + \ diagnostic(uint, ZStatisticsInterval, 10, \ "Time between statistics print outs (in seconds)") \ range(1, (uint)-1) \ --- old/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZPageAllocator.java 2019-05-02 09:57:52.486147309 +0200 +++ new/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZPageAllocator.java 2019-05-02 09:57:52.262139911 +0200 @@ -27,8 +27,6 @@ import sun.jvm.hotspot.debugger.Address; import sun.jvm.hotspot.runtime.VM; import sun.jvm.hotspot.runtime.VMObject; -import sun.jvm.hotspot.runtime.VMObjectFactory; -import sun.jvm.hotspot.types.AddressField; import sun.jvm.hotspot.types.CIntegerField; import sun.jvm.hotspot.types.Type; import sun.jvm.hotspot.types.TypeDataBase; @@ -37,7 +35,8 @@ public class ZPageAllocator extends VMObject { - private static long physicalFieldOffset; + private static CIntegerField maxCapacityField; + private static CIntegerField capacityField; private static CIntegerField usedField; static { @@ -47,21 +46,17 @@ static private synchronized void initialize(TypeDataBase db) { Type type = db.lookupType("ZPageAllocator"); - physicalFieldOffset = type.getAddressField("_physical").getOffset(); + maxCapacityField = type.getCIntegerField("_max_capacity"); + capacityField = type.getCIntegerField("_capacity"); usedField = type.getCIntegerField("_used"); } - private ZPhysicalMemoryManager physical() { - Address physicalAddr = addr.addOffsetTo(physicalFieldOffset); - return (ZPhysicalMemoryManager)VMObjectFactory.newObject(ZPhysicalMemoryManager.class, physicalAddr); - } - public long maxCapacity() { - return physical().maxCapacity(); + return maxCapacityField.getValue(addr); } public long capacity() { - return physical().capacity(); + return capacityField.getValue(addr); } public long used() { --- old/test/hotspot/gtest/gc/z/test_zForwarding.cpp 2019-05-02 09:57:52.774156820 +0200 +++ new/test/hotspot/gtest/gc/z/test_zForwarding.cpp 2019-05-02 09:57:52.581150446 +0200 @@ -169,9 +169,6 @@ // Teardown forwarding ZForwarding::destroy(forwarding); - - // Teardown page - page.physical_memory().clear(); } // Run the given function with a few different input values. --- old/test/hotspot/gtest/gc/z/test_zPhysicalMemory.cpp 2019-05-02 09:57:53.063166365 +0200 +++ new/test/hotspot/gtest/gc/z/test_zPhysicalMemory.cpp 2019-05-02 09:57:52.871160024 +0200 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016, 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2016, 2019, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -22,59 +22,93 @@ */ #include "precompiled.hpp" -#include "gc/z/zGlobals.hpp" #include "gc/z/zPhysicalMemory.inline.hpp" -#include "utilities/debug.hpp" #include "unittest.hpp" -#if defined(AMD64) - -TEST(ZPhysicalMemorySegmentTest, split) { - ZPhysicalMemorySegment seg(0, 10 * ZGranuleSize); - - ZPhysicalMemorySegment seg_split0 = seg.split(0 * ZGranuleSize); - EXPECT_EQ(seg_split0.size(), 0 * ZGranuleSize); - EXPECT_EQ( seg.size(), 10 * ZGranuleSize); - - ZPhysicalMemorySegment seg_split1 = seg.split(5 * ZGranuleSize); - EXPECT_EQ(seg_split1.size(), 5 * ZGranuleSize); - EXPECT_EQ( seg.size(), 5 * ZGranuleSize); - - ZPhysicalMemorySegment seg_split2 = seg.split(5 * ZGranuleSize); - EXPECT_EQ(seg_split2.size(), 5 * ZGranuleSize); - EXPECT_EQ( seg.size(), 0 * ZGranuleSize); - - ZPhysicalMemorySegment seg_split3 = seg.split(0 * ZGranuleSize); - EXPECT_EQ(seg_split3.size(), 0 * ZGranuleSize); - EXPECT_EQ( seg.size(), 0 * ZGranuleSize); +TEST(ZPhysicalMemoryTest, copy) { + const ZPhysicalMemorySegment seg0(0, 100); + const ZPhysicalMemorySegment seg1(200, 100); + + ZPhysicalMemory pmem0; + pmem0.add_segment(seg0); + EXPECT_EQ(pmem0.nsegments(), 1u) << "wrong number of segments"; + EXPECT_EQ(pmem0.segment(0).size(), 100u) << "wrong segment size"; + + ZPhysicalMemory pmem1; + pmem1.add_segment(seg0); + pmem1.add_segment(seg1); + EXPECT_EQ(pmem1.nsegments(), 2u) << "wrong number of segments"; + EXPECT_EQ(pmem1.segment(0).size(), 100u) << "wrong segment size"; + EXPECT_EQ(pmem1.segment(1).size(), 100u) << "wrong segment size"; + + ZPhysicalMemory pmem2(pmem0); + EXPECT_EQ(pmem2.nsegments(), 1u) << "wrong number of segments"; + EXPECT_EQ(pmem2.segment(0).size(), 100u) << "wrong segment size"; + + pmem2 = pmem1; + EXPECT_EQ(pmem2.nsegments(), 2u) << "wrong number of segments"; + EXPECT_EQ(pmem2.segment(0).size(), 100u) << "wrong segment size"; + EXPECT_EQ(pmem2.segment(1).size(), 100u) << "wrong segment size"; } -TEST(ZPhysicalMemoryTest, split) { - ZPhysicalMemoryManager pmem_manager(10 * ZGranuleSize); - - pmem_manager.try_ensure_unused_capacity(10 * ZGranuleSize); - EXPECT_EQ(pmem_manager.unused_capacity(), 10 * ZGranuleSize); - - ZPhysicalMemory pmem = pmem_manager.alloc(8 * ZGranuleSize); - EXPECT_EQ(pmem.nsegments(), 1u) << "wrong number of segments"; - - ZPhysicalMemory split0_pmem = pmem.split(ZGranuleSize); - EXPECT_EQ(split0_pmem.nsegments(), 1u); - EXPECT_EQ( pmem.nsegments(), 1u); - EXPECT_EQ(split0_pmem.size(), 1 * ZGranuleSize); - EXPECT_EQ( pmem.size(), 7 * ZGranuleSize); - - ZPhysicalMemory split1_pmem = pmem.split(2 * ZGranuleSize); - EXPECT_EQ(split1_pmem.nsegments(), 1u); - EXPECT_EQ( pmem.nsegments(), 1u); - EXPECT_EQ(split1_pmem.size(), 2 * ZGranuleSize); - EXPECT_EQ( pmem.size(), 5 * ZGranuleSize); - - ZPhysicalMemory split2_pmem = pmem.split(5 * ZGranuleSize); - EXPECT_EQ(split2_pmem.nsegments(), 1u); - EXPECT_EQ( pmem.nsegments(), 1u); - EXPECT_EQ(split2_pmem.size(), 5 * ZGranuleSize); - EXPECT_EQ( pmem.size(), 0 * ZGranuleSize); +TEST(ZPhysicalMemoryTest, segments) { + const ZPhysicalMemorySegment seg0(0, 1); + const ZPhysicalMemorySegment seg1(1, 1); + const ZPhysicalMemorySegment seg2(2, 1); + const ZPhysicalMemorySegment seg3(3, 1); + const ZPhysicalMemorySegment seg4(4, 1); + const ZPhysicalMemorySegment seg5(5, 1); + const ZPhysicalMemorySegment seg6(6, 1); + + ZPhysicalMemory pmem0; + EXPECT_EQ(pmem0.nsegments(), 0u) << "wrong number of segments"; + EXPECT_EQ(pmem0.is_null(), true) << "should be null"; + + ZPhysicalMemory pmem1; + pmem1.add_segment(seg0); + pmem1.add_segment(seg1); + pmem1.add_segment(seg2); + pmem1.add_segment(seg3); + pmem1.add_segment(seg4); + pmem1.add_segment(seg5); + pmem1.add_segment(seg6); + EXPECT_EQ(pmem1.nsegments(), 1u) << "wrong number of segments"; + EXPECT_EQ(pmem1.segment(0).size(), 7u) << "wrong segment size"; + EXPECT_EQ(pmem1.is_null(), false) << "should not be null"; + + ZPhysicalMemory pmem2; + pmem2.add_segment(seg0); + pmem2.add_segment(seg1); + pmem2.add_segment(seg2); + pmem2.add_segment(seg4); + pmem2.add_segment(seg5); + pmem2.add_segment(seg6); + EXPECT_EQ(pmem2.nsegments(), 2u) << "wrong number of segments"; + EXPECT_EQ(pmem2.segment(0).size(), 3u) << "wrong segment size"; + EXPECT_EQ(pmem2.segment(1).size(), 3u) << "wrong segment size"; + EXPECT_EQ(pmem2.is_null(), false) << "should not be null"; + + ZPhysicalMemory pmem3; + pmem3.add_segment(seg0); + pmem3.add_segment(seg2); + pmem3.add_segment(seg3); + pmem3.add_segment(seg4); + pmem3.add_segment(seg6); + EXPECT_EQ(pmem3.nsegments(), 3u) << "wrong number of segments"; + EXPECT_EQ(pmem3.segment(0).size(), 1u) << "wrong segment size"; + EXPECT_EQ(pmem3.segment(1).size(), 3u) << "wrong segment size"; + EXPECT_EQ(pmem3.segment(2).size(), 1u) << "wrong segment size"; + EXPECT_EQ(pmem3.is_null(), false) << "should not be null"; + + ZPhysicalMemory pmem4; + pmem4.add_segment(seg0); + pmem4.add_segment(seg2); + pmem4.add_segment(seg4); + pmem4.add_segment(seg6); + EXPECT_EQ(pmem4.nsegments(), 4u) << "wrong number of segments"; + EXPECT_EQ(pmem4.segment(0).size(), 1u) << "wrong segment size"; + EXPECT_EQ(pmem4.segment(1).size(), 1u) << "wrong segment size"; + EXPECT_EQ(pmem4.segment(2).size(), 1u) << "wrong segment size"; + EXPECT_EQ(pmem4.segment(3).size(), 1u) << "wrong segment size"; + EXPECT_EQ(pmem4.is_null(), false) << "should not be null"; } - -#endif --- old/test/hotspot/jtreg/ProblemList-zgc.txt 2019-05-02 09:57:53.357176074 +0200 +++ new/test/hotspot/jtreg/ProblemList-zgc.txt 2019-05-02 09:57:53.168169832 +0200 @@ -37,6 +37,5 @@ serviceability/sa/TestClhsdbJstackLock.java 8220624 generic-all serviceability/sa/TestHeapDumpForInvokeDynamic.java 8220624 generic-all serviceability/sa/TestHeapDumpForLargeArray.java 8220624 generic-all -serviceability/sa/TestUniverse.java 8220624 generic-all serviceability/sa/TestJmapCore.java 8220624 generic-all serviceability/sa/TestJmapCoreMetaspace.java 8219443 generic-all --- /dev/null 2019-04-18 12:37:53.502576822 +0200 +++ new/src/hotspot/share/gc/z/zUncommitter.cpp 2019-05-02 09:57:53.449179113 +0200 @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#include "precompiled.hpp" +#include "gc/z/zHeap.inline.hpp" +#include "gc/z/zUncommitter.hpp" +#include "runtime/mutexLocker.hpp" +#include "runtime/os.hpp" + +ZUncommitter::ZUncommitter() : + _monitor(Monitor::leaf, "ZUncommitter", false, Monitor::_safepoint_check_never), + _stop(false) { + set_name("ZUncommitter"); + create_and_start(); +} + +bool ZUncommitter::idle(uint64_t timeout) { + // Idle for at least one second + const uint64_t expires = os::elapsedTime() + MAX2(timeout, 1ul); + + for (;;) { + // We might wake up spuriously from wait, so always recalculate + // the timeout after a wakeup to see if we need to wait again. + const uint64_t now = os::elapsedTime(); + const uint64_t remaining = expires - MIN2(expires, now); + + MonitorLocker ml(&_monitor, Monitor::_no_safepoint_check_flag); + if (remaining > 0 && !_stop) { + ml.wait(remaining * MILLIUNITS); + } else { + return !_stop; + } + } +} + +void ZUncommitter::run_service() { + for (;;) { + // Try uncommit unused memory + const uint64_t timeout = ZHeap::heap()->uncommit(ZUncommitDelay); + + log_trace(gc, heap)("Uncommit Timeout: " UINT64_FORMAT "s", timeout); + + // Idle until next attempt + if (!idle(timeout)) { + return; + } + } +} + +void ZUncommitter::stop_service() { + MonitorLocker ml(&_monitor, Monitor::_no_safepoint_check_flag); + _stop = true; + ml.notify(); +} --- /dev/null 2019-04-18 12:37:53.502576822 +0200 +++ new/src/hotspot/share/gc/z/zUncommitter.hpp 2019-05-02 09:57:53.752189120 +0200 @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_ZUNCOMMITTER_HPP +#define SHARE_GC_Z_ZUNCOMMITTER_HPP + +#include "gc/shared/concurrentGCThread.hpp" +#include "runtime/mutex.hpp" + +class ZUncommitter : public ConcurrentGCThread { +private: + Monitor _monitor; + bool _stop; + + bool idle(uint64_t timeout); + +protected: + virtual void run_service(); + virtual void stop_service(); + +public: + ZUncommitter(); +}; + +#endif // SHARE_GC_Z_ZUNCOMMITTER_HPP --- /dev/null 2019-04-18 12:37:53.502576822 +0200 +++ new/test/hotspot/jtreg/gc/z/TestUncommit.java 2019-05-02 09:57:54.057199193 +0200 @@ -0,0 +1,129 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +package gc.z; + +/* + * @test TestUncommit + * @requires vm.gc.Z + * @summary Test ZGC uncommit unused memory + * @run main/othervm -XX:+UnlockExperimentalVMOptions -XX:+UseZGC -Xms128M -Xmx512M -XX:ZUncommitDelay=10 gc.z.TestUncommit true 3 + * @run main/othervm -XX:+UnlockExperimentalVMOptions -XX:+UseZGC -Xms512M -Xmx512M -XX:ZUncommitDelay=10 gc.z.TestUncommit false 1 + * @run main/othervm -XX:+UnlockExperimentalVMOptions -XX:+UseZGC -Xms128M -Xmx512M -XX:ZUncommitDelay=10 -XX:-ZUncommit gc.z.TestUncommit false 1 + */ + +import java.util.ArrayList; + +public class TestUncommit { + private static final int delay = 10; // seconds + private static final int allocSize = 200 * 1024 * 1024; // 200M + private static final int smallObjectSize = 4 * 1024; // 4K + private static final int mediumObjectSize = 2 * 1024 * 1024; // 2M + private static final int largeObjectSize = allocSize; + + private static volatile ArrayList keepAlive; + + private static long capacity() { + return Runtime.getRuntime().totalMemory(); + } + + private static void allocate(int objectSize) { + keepAlive = new ArrayList<>(); + for (int i = 0; i < allocSize; i+= objectSize) { + keepAlive.add(new byte[objectSize]); + } + } + + private static void reclaim() { + keepAlive = null; + System.gc(); + } + + private static void test(boolean enabled, int objectSize) throws Exception { + final var beforeAlloc = capacity(); + + // Allocate memory + allocate(objectSize); + + final var afterAlloc = capacity(); + + // Reclaim memory + reclaim(); + + // Wait shorter than the uncommit delay + Thread.sleep(delay * 1000 / 2); + + final var beforeUncommit = capacity(); + + // Wait longer than the uncommit delay + Thread.sleep(delay * 1000); + + final var afterUncommit = capacity(); + + System.out.println(" Uncommit Enabled: " + enabled); + System.out.println(" Uncommit Delay: " + delay); + System.out.println(" Object Size: " + objectSize); + System.out.println(" Alloc Size: " + allocSize); + System.out.println(" Before Alloc: " + beforeAlloc); + System.out.println(" After Alloc: " + afterAlloc); + System.out.println(" Before Uncommit: " + beforeUncommit); + System.out.println(" After Uncommit: " + afterUncommit); + System.out.println(); + + // Verify + if (enabled) { + if (beforeUncommit == beforeAlloc) { + throw new Exception("Uncommitted too fast"); + } + + if (afterUncommit >= afterAlloc) { + throw new Exception("Uncommitted too slow"); + } + + if (afterUncommit < beforeAlloc) { + throw new Exception("Uncommitted too much"); + } + + if (afterUncommit > beforeAlloc) { + throw new Exception("Uncommitted too little"); + } + } else { + if (afterAlloc > beforeUncommit || + afterAlloc > afterUncommit) { + throw new Exception("Should not uncommit"); + } + } + } + + public static void main(String[] args) throws Exception { + final boolean enabled = Boolean.parseBoolean(args[0]); + final int iterations = Integer.parseInt(args[1]); + + for (int i = 0; i < iterations; i++) { + System.out.println("Iteration " + i); + test(enabled, smallObjectSize); + test(enabled, mediumObjectSize); + test(enabled, largeObjectSize); + } + } +} --- old/src/hotspot/share/gc/z/zPreMappedMemory.cpp 2019-05-02 09:57:54.649218744 +0200 +++ /dev/null 2019-04-18 12:37:53.502576822 +0200 @@ -1,87 +0,0 @@ -/* - * Copyright (c) 2016, 2018, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - */ - -#include "precompiled.hpp" -#include "gc/z/zPage.inline.hpp" -#include "gc/z/zPhysicalMemory.inline.hpp" -#include "gc/z/zPreMappedMemory.inline.hpp" -#include "gc/z/zVirtualMemory.inline.hpp" -#include "logging/log.hpp" - -ZPreMappedMemory::ZPreMappedMemory(ZVirtualMemoryManager &vmm, ZPhysicalMemoryManager &pmm, size_t size) : - _vmem(), - _pmem(), - _initialized(false) { - if (!vmm.is_initialized() || !pmm.is_initialized()) { - // Not initialized - return; - } - - // Pre-mapping and pre-touching memory can take a long time. Log a message - // to help the user understand why the JVM might seem slow to start. - log_info(gc, init)("Pre-touching: %s", AlwaysPreTouch ? "Enabled" : "Disabled"); - log_info(gc, init)("Pre-mapping: " SIZE_FORMAT "M", size / M); - - if (size > 0) { - _pmem = pmm.alloc(size); - if (_pmem.is_null()) { - // Out of memory - log_error(gc, init)("Failed to pre-map Java heap (Cannot allocate physical memory)"); - return; - } - - _vmem = vmm.alloc(size, true /* alloc_from_front */); - if (_vmem.is_null()) { - // Out of address space - log_error(gc, init)("Failed to pre-map Java heap (Cannot allocate virtual memory)"); - pmm.free(_pmem); - return; - } - - // Map physical memory - pmm.map(_pmem, _vmem.start()); - } - - _initialized = true; -} - -ZPage* ZPreMappedMemory::alloc_page(uint8_t type, size_t size) { - if (size > available()) { - // Not enough pre-mapped memory - return NULL; - } - - // Take a chunk of the pre-mapped memory - const ZPhysicalMemory pmem = _pmem.split(size); - const ZVirtualMemory vmem = _vmem.split(size); - - ZPage* const page = new ZPage(type, vmem, pmem); - page->set_pre_mapped(); - - return page; -} - -void ZPreMappedMemory::clear() { - assert(_pmem.is_null(), "Should be detached"); - _vmem.clear(); -} --- old/src/hotspot/share/gc/z/zPreMappedMemory.hpp 2019-05-02 09:57:54.939228322 +0200 +++ /dev/null 2019-04-18 12:37:53.502576822 +0200 @@ -1,54 +0,0 @@ -/* - * Copyright (c) 2016, 2017, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - */ - -#ifndef SHARE_GC_Z_ZPREMAPPEDMEMORY_HPP -#define SHARE_GC_Z_ZPREMAPPEDMEMORY_HPP - -#include "gc/z/zPhysicalMemory.hpp" -#include "gc/z/zVirtualMemory.hpp" -#include "memory/allocation.hpp" - -class ZPage; - -class ZPreMappedMemory { -private: - ZVirtualMemory _vmem; - ZPhysicalMemory _pmem; - bool _initialized; - -public: - ZPreMappedMemory(ZVirtualMemoryManager &vmm, ZPhysicalMemoryManager &pmm, size_t size); - - bool is_initialized() const; - - ZPhysicalMemory& physical_memory(); - const ZVirtualMemory& virtual_memory() const; - - size_t available() const; - - ZPage* alloc_page(uint8_t type, size_t size); - - void clear(); -}; - -#endif // SHARE_GC_Z_ZPREMAPPEDMEMORY_HPP --- old/src/hotspot/share/gc/z/zPreMappedMemory.inline.hpp 2019-05-02 09:57:55.211237305 +0200 +++ /dev/null 2019-04-18 12:37:53.502576822 +0200 @@ -1,45 +0,0 @@ -/* - * Copyright (c) 2016, 2017, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - */ - -#ifndef SHARE_GC_Z_ZPREMAPPEDMEMORY_INLINE_HPP -#define SHARE_GC_Z_ZPREMAPPEDMEMORY_INLINE_HPP - -#include "gc/z/zPreMappedMemory.hpp" - -inline bool ZPreMappedMemory::is_initialized() const { - return _initialized; -} - -inline ZPhysicalMemory& ZPreMappedMemory::physical_memory() { - return _pmem; -} - -inline const ZVirtualMemory& ZPreMappedMemory::virtual_memory() const { - return _vmem; -} - -inline size_t ZPreMappedMemory::available() const { - return _vmem.size(); -} - -#endif // SHARE_GC_Z_ZPREMAPPEDMEMORY_INLINE_HPP --- old/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZPhysicalMemoryManager.java 2019-05-02 09:57:55.464245660 +0200 +++ /dev/null 2019-04-18 12:37:53.502576822 +0200 @@ -1,64 +0,0 @@ -/* - * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -package sun.jvm.hotspot.gc.z; - -import sun.jvm.hotspot.debugger.Address; -import sun.jvm.hotspot.runtime.VM; -import sun.jvm.hotspot.runtime.VMObject; -import sun.jvm.hotspot.types.CIntegerField; -import sun.jvm.hotspot.types.Type; -import sun.jvm.hotspot.types.TypeDataBase; - -// Mirror class for ZPhysicalMemoryManager - -public class ZPhysicalMemoryManager extends VMObject { - - private static CIntegerField capacityField; - - private static CIntegerField maxCapacityField; - - static { - VM.registerVMInitializedObserver((o, d) -> initialize(VM.getVM().getTypeDataBase())); - } - - private static synchronized void initialize(TypeDataBase db) { - Type type = db.lookupType("ZPhysicalMemoryManager"); - - capacityField = type.getCIntegerField("_capacity"); - maxCapacityField = type.getCIntegerField("_max_capacity"); - } - - public long capacity() { - return capacityField.getValue(addr); - } - - public long maxCapacity() { - return maxCapacityField.getValue(addr); - } - - public ZPhysicalMemoryManager(Address addr) { - super(addr); - } -} --- old/test/hotspot/gtest/gc/z/test_zVirtualMemory.cpp 2019-05-02 09:57:55.719254082 +0200 +++ /dev/null 2019-04-18 12:37:53.502576822 +0200 @@ -1,48 +0,0 @@ -/* - * Copyright (c) 2016, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - */ - -#include "precompiled.hpp" -#include "gc/z/zVirtualMemory.inline.hpp" -#include "utilities/debug.hpp" -#include "unittest.hpp" - -TEST(ZVirtualMemory, split) { - const size_t PageSize = 2 * M; - - ZVirtualMemory mem(0, 10 * PageSize); - - ZVirtualMemory mem_split0 = mem.split(0 * PageSize); - EXPECT_EQ(mem_split0.size(), 0 * PageSize); - EXPECT_EQ( mem.size(), 10 * PageSize); - - ZVirtualMemory mem_split1 = mem.split(5u * PageSize); - EXPECT_EQ(mem_split1.size(), 5 * PageSize); - EXPECT_EQ( mem.size(), 5 * PageSize); - - ZVirtualMemory mem_split2 = mem.split(5u * PageSize); - EXPECT_EQ(mem_split2.size(), 5 * PageSize); - EXPECT_EQ( mem.size(), 0 * PageSize); - - ZVirtualMemory mem_split3 = mem.split(0 * PageSize); - EXPECT_EQ(mem_split3.size(), 0 * PageSize); -}