1 /* 2 * Copyright (c) 2015, 2020, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 */ 23 24 #include "precompiled.hpp" 25 #include "gc/z/zArray.inline.hpp" 26 #include "gc/z/zErrno.hpp" 27 #include "gc/z/zGlobals.hpp" 28 #include "gc/z/zLargePages.inline.hpp" 29 #include "gc/z/zMountPoint_linux.hpp" 30 #include "gc/z/zNUMA.inline.hpp" 31 #include "gc/z/zPhysicalMemoryBacking_linux.hpp" 32 #include "gc/z/zSyscall_linux.hpp" 33 #include "logging/log.hpp" 34 #include "runtime/init.hpp" 35 #include "runtime/os.hpp" 36 #include "utilities/align.hpp" 37 #include "utilities/debug.hpp" 38 #include "utilities/growableArray.hpp" 39 40 #include <fcntl.h> 41 #include <stdio.h> 42 #include <sys/mman.h> 43 #include <sys/stat.h> 44 #include <sys/statfs.h> 45 #include <sys/types.h> 46 #include <unistd.h> 47 48 // 49 // Support for building on older Linux systems 50 // 51 52 // memfd_create(2) flags 53 #ifndef MFD_CLOEXEC 54 #define MFD_CLOEXEC 0x0001U 55 #endif 56 #ifndef MFD_HUGETLB 57 #define MFD_HUGETLB 0x0004U 58 #endif 59 60 // open(2) flags 61 #ifndef O_CLOEXEC 62 #define O_CLOEXEC 02000000 63 #endif 64 #ifndef O_TMPFILE 65 #define O_TMPFILE (020000000 | O_DIRECTORY) 66 #endif 67 68 // fallocate(2) flags 69 #ifndef FALLOC_FL_KEEP_SIZE 70 #define FALLOC_FL_KEEP_SIZE 0x01 71 #endif 72 #ifndef FALLOC_FL_PUNCH_HOLE 73 #define FALLOC_FL_PUNCH_HOLE 0x02 74 #endif 75 76 // Filesystem types, see statfs(2) 77 #ifndef TMPFS_MAGIC 78 #define TMPFS_MAGIC 0x01021994 79 #endif 80 #ifndef HUGETLBFS_MAGIC 81 #define HUGETLBFS_MAGIC 0x958458f6 82 #endif 83 84 // Filesystem names 85 #define ZFILESYSTEM_TMPFS "tmpfs" 86 #define ZFILESYSTEM_HUGETLBFS "hugetlbfs" 87 88 // Proc file entry for max map mount 89 #define ZFILENAME_PROC_MAX_MAP_COUNT "/proc/sys/vm/max_map_count" 90 91 // Sysfs file for transparent huge page on tmpfs 92 #define ZFILENAME_SHMEM_ENABLED "/sys/kernel/mm/transparent_hugepage/shmem_enabled" 93 94 // Java heap filename 95 #define ZFILENAME_HEAP "java_heap" 96 97 // Preferred tmpfs mount points, ordered by priority 98 static const char* z_preferred_tmpfs_mountpoints[] = { 99 "/dev/shm", 100 "/run/shm", 101 NULL 102 }; 103 104 // Preferred hugetlbfs mount points, ordered by priority 105 static const char* z_preferred_hugetlbfs_mountpoints[] = { 106 "/dev/hugepages", 107 "/hugepages", 108 NULL 109 }; 110 111 static int z_fallocate_hugetlbfs_attempts = 3; 112 static bool z_fallocate_supported = true; 113 114 ZPhysicalMemoryBacking::ZPhysicalMemoryBacking() : 115 _fd(-1), 116 _size(0), 117 _filesystem(0), 118 _block_size(0), 119 _available(0), 120 _initialized(false) { 121 122 // Create backing file 123 _fd = create_fd(ZFILENAME_HEAP); 124 if (_fd == -1) { 125 return; 126 } 127 128 // Get filesystem statistics 129 struct statfs buf; 130 if (fstatfs(_fd, &buf) == -1) { 131 ZErrno err; 132 log_error(gc)("Failed to determine filesystem type for backing file (%s)", err.to_string()); 133 return; 134 } 135 136 _filesystem = buf.f_type; 137 _block_size = buf.f_bsize; 138 _available = buf.f_bavail * _block_size; 139 140 log_info(gc, init)("Heap Backing Filesystem: %s (0x" UINT64_FORMAT_X ")", 141 is_tmpfs() ? ZFILESYSTEM_TMPFS : is_hugetlbfs() ? ZFILESYSTEM_HUGETLBFS : "other", _filesystem); 142 143 // Make sure the filesystem type matches requested large page type 144 if (ZLargePages::is_transparent() && !is_tmpfs()) { 145 log_error(gc)("-XX:+UseTransparentHugePages can only be enabled when using a %s filesystem", 146 ZFILESYSTEM_TMPFS); 147 return; 148 } 149 150 if (ZLargePages::is_transparent() && !tmpfs_supports_transparent_huge_pages()) { 151 log_error(gc)("-XX:+UseTransparentHugePages on a %s filesystem not supported by kernel", 152 ZFILESYSTEM_TMPFS); 153 return; 154 } 155 156 if (ZLargePages::is_explicit() && !is_hugetlbfs()) { 157 log_error(gc)("-XX:+UseLargePages (without -XX:+UseTransparentHugePages) can only be enabled " 158 "when using a %s filesystem", ZFILESYSTEM_HUGETLBFS); 159 return; 160 } 161 162 if (!ZLargePages::is_explicit() && is_hugetlbfs()) { 163 log_error(gc)("-XX:+UseLargePages must be enabled when using a %s filesystem", 164 ZFILESYSTEM_HUGETLBFS); 165 return; 166 } 167 168 if (ZLargePages::is_explicit() && os::large_page_size() != ZGranuleSize) { 169 log_error(gc)("Incompatible large page size configured " SIZE_FORMAT " (expected " SIZE_FORMAT ")", 170 os::large_page_size(), ZGranuleSize); 171 return; 172 } 173 174 // Make sure the filesystem block size is compatible 175 if (ZGranuleSize % _block_size != 0) { 176 log_error(gc)("Filesystem backing the heap has incompatible block size (" SIZE_FORMAT ")", 177 _block_size); 178 return; 179 } 180 181 if (is_hugetlbfs() && _block_size != ZGranuleSize) { 182 log_error(gc)("%s filesystem has unexpected block size " SIZE_FORMAT " (expected " SIZE_FORMAT ")", 183 ZFILESYSTEM_HUGETLBFS, _block_size, ZGranuleSize); 184 return; 185 } 186 187 // Successfully initialized 188 _initialized = true; 189 } 190 191 int ZPhysicalMemoryBacking::create_mem_fd(const char* name) const { 192 // Create file name 193 char filename[PATH_MAX]; 194 snprintf(filename, sizeof(filename), "%s%s", name, ZLargePages::is_explicit() ? ".hugetlb" : ""); 195 196 // Create file 197 const int extra_flags = ZLargePages::is_explicit() ? MFD_HUGETLB : 0; 198 const int fd = ZSyscall::memfd_create(filename, MFD_CLOEXEC | extra_flags); 199 if (fd == -1) { 200 ZErrno err; 201 log_debug(gc, init)("Failed to create memfd file (%s)", 202 ((ZLargePages::is_explicit() && err == EINVAL) ? "Hugepages not supported" : err.to_string())); 203 return -1; 204 } 205 206 log_info(gc, init)("Heap Backing File: /memfd:%s", filename); 207 208 return fd; 209 } 210 211 int ZPhysicalMemoryBacking::create_file_fd(const char* name) const { 212 const char* const filesystem = ZLargePages::is_explicit() 213 ? ZFILESYSTEM_HUGETLBFS 214 : ZFILESYSTEM_TMPFS; 215 const char** const preferred_mountpoints = ZLargePages::is_explicit() 216 ? z_preferred_hugetlbfs_mountpoints 217 : z_preferred_tmpfs_mountpoints; 218 219 // Find mountpoint 220 ZMountPoint mountpoint(filesystem, preferred_mountpoints); 221 if (mountpoint.get() == NULL) { 222 log_error(gc)("Use -XX:AllocateHeapAt to specify the path to a %s filesystem", filesystem); 223 return -1; 224 } 225 226 // Try to create an anonymous file using the O_TMPFILE flag. Note that this 227 // flag requires kernel >= 3.11. If this fails we fall back to open/unlink. 228 const int fd_anon = os::open(mountpoint.get(), O_TMPFILE|O_EXCL|O_RDWR|O_CLOEXEC, S_IRUSR|S_IWUSR); 229 if (fd_anon == -1) { 230 ZErrno err; 231 log_debug(gc, init)("Failed to create anonymous file in %s (%s)", mountpoint.get(), 232 (err == EINVAL ? "Not supported" : err.to_string())); 233 } else { 234 // Get inode number for anonymous file 235 struct stat stat_buf; 236 if (fstat(fd_anon, &stat_buf) == -1) { 237 ZErrno err; 238 log_error(gc)("Failed to determine inode number for anonymous file (%s)", err.to_string()); 239 return -1; 240 } 241 242 log_info(gc, init)("Heap Backing File: %s/#" UINT64_FORMAT, mountpoint.get(), (uint64_t)stat_buf.st_ino); 243 244 return fd_anon; 245 } 246 247 log_debug(gc, init)("Falling back to open/unlink"); 248 249 // Create file name 250 char filename[PATH_MAX]; 251 snprintf(filename, sizeof(filename), "%s/%s.%d", mountpoint.get(), name, os::current_process_id()); 252 253 // Create file 254 const int fd = os::open(filename, O_CREAT|O_EXCL|O_RDWR|O_CLOEXEC, S_IRUSR|S_IWUSR); 255 if (fd == -1) { 256 ZErrno err; 257 log_error(gc)("Failed to create file %s (%s)", filename, err.to_string()); 258 return -1; 259 } 260 261 // Unlink file 262 if (unlink(filename) == -1) { 263 ZErrno err; 264 log_error(gc)("Failed to unlink file %s (%s)", filename, err.to_string()); 265 return -1; 266 } 267 268 log_info(gc, init)("Heap Backing File: %s", filename); 269 270 return fd; 271 } 272 273 int ZPhysicalMemoryBacking::create_fd(const char* name) const { 274 if (AllocateHeapAt == NULL) { 275 // If the path is not explicitly specified, then we first try to create a memfd file 276 // instead of looking for a tmpfd/hugetlbfs mount point. Note that memfd_create() might 277 // not be supported at all (requires kernel >= 3.17), or it might not support large 278 // pages (requires kernel >= 4.14). If memfd_create() fails, then we try to create a 279 // file on an accessible tmpfs or hugetlbfs mount point. 280 const int fd = create_mem_fd(name); 281 if (fd != -1) { 282 return fd; 283 } 284 285 log_debug(gc, init)("Falling back to searching for an accessible mount point"); 286 } 287 288 return create_file_fd(name); 289 } 290 291 bool ZPhysicalMemoryBacking::is_initialized() const { 292 return _initialized; 293 } 294 295 void ZPhysicalMemoryBacking::warn_available_space(size_t max) const { 296 // Note that the available space on a tmpfs or a hugetlbfs filesystem 297 // will be zero if no size limit was specified when it was mounted. 298 if (_available == 0) { 299 // No size limit set, skip check 300 log_info(gc, init)("Available space on backing filesystem: N/A"); 301 return; 302 } 303 304 log_info(gc, init)("Available space on backing filesystem: " SIZE_FORMAT "M", _available / M); 305 306 // Warn if the filesystem doesn't currently have enough space available to hold 307 // the max heap size. The max heap size will be capped if we later hit this limit 308 // when trying to expand the heap. 309 if (_available < max) { 310 log_warning(gc)("***** WARNING! INCORRECT SYSTEM CONFIGURATION DETECTED! *****"); 311 log_warning(gc)("Not enough space available on the backing filesystem to hold the current max Java heap"); 312 log_warning(gc)("size (" SIZE_FORMAT "M). Please adjust the size of the backing filesystem accordingly " 313 "(available", max / M); 314 log_warning(gc)("space is currently " SIZE_FORMAT "M). Continuing execution with the current filesystem " 315 "size could", _available / M); 316 log_warning(gc)("lead to a premature OutOfMemoryError being thrown, due to failure to map memory."); 317 } 318 } 319 320 void ZPhysicalMemoryBacking::warn_max_map_count(size_t max) const { 321 const char* const filename = ZFILENAME_PROC_MAX_MAP_COUNT; 322 FILE* const file = fopen(filename, "r"); 323 if (file == NULL) { 324 // Failed to open file, skip check 325 log_debug(gc, init)("Failed to open %s", filename); 326 return; 327 } 328 329 size_t actual_max_map_count = 0; 330 const int result = fscanf(file, SIZE_FORMAT, &actual_max_map_count); 331 fclose(file); 332 if (result != 1) { 333 // Failed to read file, skip check 334 log_debug(gc, init)("Failed to read %s", filename); 335 return; 336 } 337 338 // The required max map count is impossible to calculate exactly since subsystems 339 // other than ZGC are also creating memory mappings, and we have no control over that. 340 // However, ZGC tends to create the most mappings and dominate the total count. 341 // In the worst cases, ZGC will map each granule three times, i.e. once per heap view. 342 // We speculate that we need another 20% to allow for non-ZGC subsystems to map memory. 343 const size_t required_max_map_count = (max / ZGranuleSize) * 3 * 1.2; 344 if (actual_max_map_count < required_max_map_count) { 345 log_warning(gc)("***** WARNING! INCORRECT SYSTEM CONFIGURATION DETECTED! *****"); 346 log_warning(gc)("The system limit on number of memory mappings per process might be too low for the given"); 347 log_warning(gc)("max Java heap size (" SIZE_FORMAT "M). Please adjust %s to allow for at", 348 max / M, filename); 349 log_warning(gc)("least " SIZE_FORMAT " mappings (current limit is " SIZE_FORMAT "). Continuing execution " 350 "with the current", required_max_map_count, actual_max_map_count); 351 log_warning(gc)("limit could lead to a fatal error, due to failure to map memory."); 352 } 353 } 354 355 void ZPhysicalMemoryBacking::warn_commit_limits(size_t max) const { 356 // Warn if available space is too low 357 warn_available_space(max); 358 359 // Warn if max map count is too low 360 warn_max_map_count(max); 361 } 362 363 size_t ZPhysicalMemoryBacking::size() const { 364 return _size; 365 } 366 367 bool ZPhysicalMemoryBacking::is_tmpfs() const { 368 return _filesystem == TMPFS_MAGIC; 369 } 370 371 bool ZPhysicalMemoryBacking::is_hugetlbfs() const { 372 return _filesystem == HUGETLBFS_MAGIC; 373 } 374 375 bool ZPhysicalMemoryBacking::tmpfs_supports_transparent_huge_pages() const { 376 // If the shmem_enabled file exists and is readable then we 377 // know the kernel supports transparent huge pages for tmpfs. 378 return access(ZFILENAME_SHMEM_ENABLED, R_OK) == 0; 379 } 380 381 ZErrno ZPhysicalMemoryBacking::fallocate_compat_ftruncate(size_t size) const { 382 while (ftruncate(_fd, size) == -1) { 383 if (errno != EINTR) { 384 // Failed 385 return errno; 386 } 387 } 388 389 // Success 390 return 0; 391 } 392 393 ZErrno ZPhysicalMemoryBacking::fallocate_compat_mmap(size_t offset, size_t length, bool touch) const { 394 // On hugetlbfs, mapping a file segment will fail immediately, without 395 // the need to touch the mapped pages first, if there aren't enough huge 396 // pages available to back the mapping. 397 void* const addr = mmap(0, length, PROT_READ|PROT_WRITE, MAP_SHARED, _fd, offset); 398 if (addr == MAP_FAILED) { 399 // Failed 400 return errno; 401 } 402 403 // Once mapped, the huge pages are only reserved. We need to touch them 404 // to associate them with the file segment. Note that we can not punch 405 // hole in file segments which only have reserved pages. 406 if (touch) { 407 char* const start = (char*)addr; 408 char* const end = start + length; 409 os::pretouch_memory(start, end, _block_size); 410 } 411 412 // Unmap again. From now on, the huge pages that were mapped are allocated 413 // to this file. There's no risk in getting SIGBUS when touching them. 414 if (munmap(addr, length) == -1) { 415 // Failed 416 return errno; 417 } 418 419 // Success 420 return 0; 421 } 422 423 ZErrno ZPhysicalMemoryBacking::fallocate_compat_pwrite(size_t offset, size_t length) const { 424 uint8_t data = 0; 425 426 // Allocate backing memory by writing to each block 427 for (size_t pos = offset; pos < offset + length; pos += _block_size) { 428 if (pwrite(_fd, &data, sizeof(data), pos) == -1) { 429 // Failed 430 return errno; 431 } 432 } 433 434 // Success 435 return 0; 436 } 437 438 ZErrno ZPhysicalMemoryBacking::fallocate_fill_hole_compat(size_t offset, size_t length) { 439 // fallocate(2) is only supported by tmpfs since Linux 3.5, and by hugetlbfs 440 // since Linux 4.3. When fallocate(2) is not supported we emulate it using 441 // ftruncate/pwrite (for tmpfs) or ftruncate/mmap/munmap (for hugetlbfs). 442 443 const size_t end = offset + length; 444 if (end > _size) { 445 // Increase file size 446 const ZErrno err = fallocate_compat_ftruncate(end); 447 if (err) { 448 // Failed 449 return err; 450 } 451 } 452 453 // Allocate backing memory 454 const ZErrno err = is_hugetlbfs() ? fallocate_compat_mmap(offset, length, false /* touch */) 455 : fallocate_compat_pwrite(offset, length); 456 if (err) { 457 if (end > _size) { 458 // Restore file size 459 fallocate_compat_ftruncate(_size); 460 } 461 462 // Failed 463 return err; 464 } 465 466 if (end > _size) { 467 // Record new file size 468 _size = end; 469 } 470 471 // Success 472 return 0; 473 } 474 475 ZErrno ZPhysicalMemoryBacking::fallocate_fill_hole_syscall(size_t offset, size_t length) { 476 const int mode = 0; // Allocate 477 const int res = ZSyscall::fallocate(_fd, mode, offset, length); 478 if (res == -1) { 479 // Failed 480 return errno; 481 } 482 483 const size_t end = offset + length; 484 if (end > _size) { 485 // Record new file size 486 _size = end; 487 } 488 489 // Success 490 return 0; 491 } 492 493 ZErrno ZPhysicalMemoryBacking::fallocate_fill_hole(size_t offset, size_t length) { 494 // Using compat mode is more efficient when allocating space on hugetlbfs. 495 // Note that allocating huge pages this way will only reserve them, and not 496 // associate them with segments of the file. We must guarantee that we at 497 // some point touch these segments, otherwise we can not punch hole in them. 498 if (z_fallocate_supported && !is_hugetlbfs()) { 499 const ZErrno err = fallocate_fill_hole_syscall(offset, length); 500 if (!err) { 501 // Success 502 return 0; 503 } 504 505 if (err != ENOSYS && err != EOPNOTSUPP) { 506 // Failed 507 return err; 508 } 509 510 // Not supported 511 log_debug(gc)("Falling back to fallocate() compatibility mode"); 512 z_fallocate_supported = false; 513 } 514 515 return fallocate_fill_hole_compat(offset, length); 516 } 517 518 ZErrno ZPhysicalMemoryBacking::fallocate_punch_hole(size_t offset, size_t length) { 519 if (is_hugetlbfs()) { 520 // We can only punch hole in pages that have been touched. Non-touched 521 // pages are only reserved, and not associated with any specific file 522 // segment. We don't know which pages have been previously touched, so 523 // we always touch them here to guarantee that we can punch hole. 524 const ZErrno err = fallocate_compat_mmap(offset, length, true /* touch */); 525 if (err) { 526 // Failed 527 return err; 528 } 529 } 530 531 const int mode = FALLOC_FL_PUNCH_HOLE|FALLOC_FL_KEEP_SIZE; 532 if (ZSyscall::fallocate(_fd, mode, offset, length) == -1) { 533 // Failed 534 return errno; 535 } 536 537 // Success 538 return 0; 539 } 540 541 ZErrno ZPhysicalMemoryBacking::split_and_fallocate(bool punch_hole, size_t offset, size_t length) { 542 // Try first half 543 const size_t offset0 = offset; 544 const size_t length0 = align_up(length / 2, _block_size); 545 const ZErrno err0 = fallocate(punch_hole, offset0, length0); 546 if (err0) { 547 return err0; 548 } 549 550 // Try second half 551 const size_t offset1 = offset0 + length0; 552 const size_t length1 = length - length0; 553 const ZErrno err1 = fallocate(punch_hole, offset1, length1); 554 if (err1) { 555 return err1; 556 } 557 558 // Success 559 return 0; 560 } 561 562 ZErrno ZPhysicalMemoryBacking::fallocate(bool punch_hole, size_t offset, size_t length) { 563 assert(is_aligned(offset, _block_size), "Invalid offset"); 564 assert(is_aligned(length, _block_size), "Invalid length"); 565 566 const ZErrno err = punch_hole ? fallocate_punch_hole(offset, length) : fallocate_fill_hole(offset, length); 567 if (err == EINTR && length > _block_size) { 568 // Calling fallocate(2) with a large length can take a long time to 569 // complete. When running profilers, such as VTune, this syscall will 570 // be constantly interrupted by signals. Expanding the file in smaller 571 // steps avoids this problem. 572 return split_and_fallocate(punch_hole, offset, length); 573 } 574 575 return err; 576 } 577 578 bool ZPhysicalMemoryBacking::commit_inner(size_t offset, size_t length) { 579 log_trace(gc, heap)("Committing memory: " SIZE_FORMAT "M-" SIZE_FORMAT "M (" SIZE_FORMAT "M)", 580 offset / M, (offset + length) / M, length / M); 581 582 retry: 583 const ZErrno err = fallocate(false /* punch_hole */, offset, length); 584 if (err) { 585 if (err == ENOSPC && !is_init_completed() && is_hugetlbfs() && z_fallocate_hugetlbfs_attempts-- > 0) { 586 // If we fail to allocate during initialization, due to lack of space on 587 // the hugetlbfs filesystem, then we wait and retry a few times before 588 // giving up. Otherwise there is a risk that running JVMs back-to-back 589 // will fail, since there is a delay between process termination and the 590 // huge pages owned by that process being returned to the huge page pool 591 // and made available for new allocations. 592 log_debug(gc, init)("Failed to commit memory (%s), retrying", err.to_string()); 593 594 // Wait and retry in one second, in the hope that huge pages will be 595 // available by then. 596 sleep(1); 597 goto retry; 598 } 599 600 // Failed 601 log_error(gc)("Failed to commit memory (%s)", err.to_string()); 602 return false; 603 } 604 605 // Success 606 return true; 607 } 608 609 static int offset_to_node(size_t offset) { 610 const GrowableArray<int>* mapping = os::Linux::numa_nindex_to_node(); 611 const size_t nindex = (offset >> ZGranuleSizeShift) % mapping->length(); 612 return mapping->at((int)nindex); 613 } 614 615 size_t ZPhysicalMemoryBacking::commit_numa_interleaved(size_t offset, size_t length) { 616 size_t committed = 0; 617 618 // Commit one granule at a time, so that each granule 619 // can be allocated from a different preferred node. 620 while (committed < length) { 621 const size_t granule_offset = offset + committed; 622 623 // Setup NUMA policy to allocate memory from a preferred node 624 os::Linux::numa_set_preferred(offset_to_node(granule_offset)); 625 626 if (!commit_inner(granule_offset, ZGranuleSize)) { 627 // Failed 628 break; 629 } 630 631 committed += ZGranuleSize; 632 } 633 634 // Restore NUMA policy 635 os::Linux::numa_set_preferred(-1); 636 637 return committed; 638 } 639 640 size_t ZPhysicalMemoryBacking::commit_default(size_t offset, size_t length) { 641 // Try to commit the whole region 642 if (commit_inner(offset, length)) { 643 // Success 644 return length; 645 } 646 647 // Failed, try to commit as much as possible 648 size_t start = offset; 649 size_t end = offset + length; 650 651 for (;;) { 652 length = align_down((end - start) / 2, ZGranuleSize); 653 if (length < ZGranuleSize) { 654 // Done, don't commit more 655 return start - offset; 656 } 657 658 if (commit_inner(start, length)) { 659 // Success, try commit more 660 start += length; 661 } else { 662 // Failed, try commit less 663 end -= length; 664 } 665 } 666 } 667 668 size_t ZPhysicalMemoryBacking::commit(size_t offset, size_t length) { 669 if (ZNUMA::is_enabled() && !ZLargePages::is_explicit()) { 670 // To get granule-level NUMA interleaving when using non-large pages, 671 // we must explicitly interleave the memory at commit/fallocate time. 672 return commit_numa_interleaved(offset, length); 673 } 674 675 return commit_default(offset, length); 676 } 677 678 size_t ZPhysicalMemoryBacking::uncommit(size_t offset, size_t length) { 679 log_trace(gc, heap)("Uncommitting memory: " SIZE_FORMAT "M-" SIZE_FORMAT "M (" SIZE_FORMAT "M)", 680 offset / M, (offset + length) / M, length / M); 681 682 const ZErrno err = fallocate(true /* punch_hole */, offset, length); 683 if (err) { 684 log_error(gc)("Failed to uncommit memory (%s)", err.to_string()); 685 return 0; 686 } 687 688 return length; 689 } 690 691 void ZPhysicalMemoryBacking::map(uintptr_t addr, size_t size, uintptr_t offset) const { 692 const void* const res = mmap((void*)addr, size, PROT_READ|PROT_WRITE, MAP_FIXED|MAP_SHARED, _fd, offset); 693 if (res == MAP_FAILED) { 694 ZErrno err; 695 fatal("Failed to map memory (%s)", err.to_string()); 696 } 697 } 698 699 void ZPhysicalMemoryBacking::unmap(uintptr_t addr, size_t size) const { 700 // Note that we must keep the address space reservation intact and just detach 701 // the backing memory. For this reason we map a new anonymous, non-accessible 702 // and non-reserved page over the mapping instead of actually unmapping. 703 const void* const res = mmap((void*)addr, size, PROT_NONE, MAP_FIXED | MAP_ANONYMOUS | MAP_PRIVATE | MAP_NORESERVE, -1, 0); 704 if (res == MAP_FAILED) { 705 ZErrno err; 706 fatal("Failed to map memory (%s)", err.to_string()); 707 } 708 }