1 /*
2 * Copyright (c) 2015, 2020, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23
24 #include "precompiled.hpp"
25 #include "gc/z/zArray.inline.hpp"
26 #include "gc/z/zErrno.hpp"
27 #include "gc/z/zGlobals.hpp"
28 #include "gc/z/zLargePages.inline.hpp"
29 #include "gc/z/zMountPoint_linux.hpp"
30 #include "gc/z/zNUMA.inline.hpp"
31 #include "gc/z/zPhysicalMemoryBacking_linux.hpp"
32 #include "gc/z/zSyscall_linux.hpp"
33 #include "logging/log.hpp"
34 #include "runtime/init.hpp"
35 #include "runtime/os.hpp"
36 #include "runtime/stubRoutines.hpp"
37 #include "utilities/align.hpp"
38 #include "utilities/debug.hpp"
39 #include "utilities/growableArray.hpp"
40
41 #include <fcntl.h>
42 #include <stdio.h>
43 #include <sys/mman.h>
44 #include <sys/stat.h>
45 #include <sys/statfs.h>
46 #include <sys/types.h>
47 #include <unistd.h>
48
49 //
50 // Support for building on older Linux systems
51 //
52
53 // memfd_create(2) flags
54 #ifndef MFD_CLOEXEC
55 #define MFD_CLOEXEC 0x0001U
56 #endif
57 #ifndef MFD_HUGETLB
58 #define MFD_HUGETLB 0x0004U
59 #endif
60
61 // open(2) flags
62 #ifndef O_CLOEXEC
63 #define O_CLOEXEC 02000000
64 #endif
65 #ifndef O_TMPFILE
66 #define O_TMPFILE (020000000 | O_DIRECTORY)
67 #endif
68
69 // fallocate(2) flags
70 #ifndef FALLOC_FL_KEEP_SIZE
71 #define FALLOC_FL_KEEP_SIZE 0x01
72 #endif
73 #ifndef FALLOC_FL_PUNCH_HOLE
74 #define FALLOC_FL_PUNCH_HOLE 0x02
75 #endif
76
77 // Filesystem types, see statfs(2)
78 #ifndef TMPFS_MAGIC
79 #define TMPFS_MAGIC 0x01021994
80 #endif
81 #ifndef HUGETLBFS_MAGIC
82 #define HUGETLBFS_MAGIC 0x958458f6
83 #endif
84
85 // Filesystem names
86 #define ZFILESYSTEM_TMPFS "tmpfs"
87 #define ZFILESYSTEM_HUGETLBFS "hugetlbfs"
88
89 // Proc file entry for max map mount
90 #define ZFILENAME_PROC_MAX_MAP_COUNT "/proc/sys/vm/max_map_count"
91
92 // Sysfs file for transparent huge page on tmpfs
93 #define ZFILENAME_SHMEM_ENABLED "/sys/kernel/mm/transparent_hugepage/shmem_enabled"
94
95 // Java heap filename
96 #define ZFILENAME_HEAP "java_heap"
97
98 // Preferred tmpfs mount points, ordered by priority
99 static const char* z_preferred_tmpfs_mountpoints[] = {
100 "/dev/shm",
101 "/run/shm",
102 NULL
103 };
104
105 // Preferred hugetlbfs mount points, ordered by priority
106 static const char* z_preferred_hugetlbfs_mountpoints[] = {
107 "/dev/hugepages",
108 "/hugepages",
109 NULL
110 };
111
112 static int z_fallocate_hugetlbfs_attempts = 3;
113 static bool z_fallocate_supported = true;
114
115 ZPhysicalMemoryBacking::ZPhysicalMemoryBacking(size_t max_capacity) :
116 _fd(-1),
117 _filesystem(0),
118 _block_size(0),
119 _available(0),
120 _initialized(false) {
121
122 // Create backing file
123 _fd = create_fd(ZFILENAME_HEAP);
124 if (_fd == -1) {
125 return;
126 }
127
128 // Truncate backing file
129 if (ftruncate(_fd, max_capacity) == -1) {
130 ZErrno err;
131 log_error(gc)("Failed to truncate backing file (%s)", err.to_string());
132 return;
133 }
134
135 // Get filesystem statistics
136 struct statfs buf;
137 if (fstatfs(_fd, &buf) == -1) {
138 ZErrno err;
139 log_error(gc)("Failed to determine filesystem type for backing file (%s)", err.to_string());
140 return;
141 }
142
143 _filesystem = buf.f_type;
144 _block_size = buf.f_bsize;
145 _available = buf.f_bavail * _block_size;
146
147 log_info(gc, init)("Heap Backing Filesystem: %s (0x" UINT64_FORMAT_X ")",
148 is_tmpfs() ? ZFILESYSTEM_TMPFS : is_hugetlbfs() ? ZFILESYSTEM_HUGETLBFS : "other", _filesystem);
149
150 // Make sure the filesystem type matches requested large page type
151 if (ZLargePages::is_transparent() && !is_tmpfs()) {
152 log_error(gc)("-XX:+UseTransparentHugePages can only be enabled when using a %s filesystem",
153 ZFILESYSTEM_TMPFS);
154 return;
155 }
156
157 if (ZLargePages::is_transparent() && !tmpfs_supports_transparent_huge_pages()) {
158 log_error(gc)("-XX:+UseTransparentHugePages on a %s filesystem not supported by kernel",
159 ZFILESYSTEM_TMPFS);
160 return;
161 }
162
163 if (ZLargePages::is_explicit() && !is_hugetlbfs()) {
164 log_error(gc)("-XX:+UseLargePages (without -XX:+UseTransparentHugePages) can only be enabled "
165 "when using a %s filesystem", ZFILESYSTEM_HUGETLBFS);
166 return;
167 }
168
169 if (!ZLargePages::is_explicit() && is_hugetlbfs()) {
170 log_error(gc)("-XX:+UseLargePages must be enabled when using a %s filesystem",
171 ZFILESYSTEM_HUGETLBFS);
172 return;
173 }
174
175 if (ZLargePages::is_explicit() && os::large_page_size() != ZGranuleSize) {
176 log_error(gc)("Incompatible large page size configured " SIZE_FORMAT " (expected " SIZE_FORMAT ")",
177 os::large_page_size(), ZGranuleSize);
178 return;
179 }
180
181 // Make sure the filesystem block size is compatible
182 if (ZGranuleSize % _block_size != 0) {
183 log_error(gc)("Filesystem backing the heap has incompatible block size (" SIZE_FORMAT ")",
184 _block_size);
185 return;
186 }
187
188 if (is_hugetlbfs() && _block_size != ZGranuleSize) {
189 log_error(gc)("%s filesystem has unexpected block size " SIZE_FORMAT " (expected " SIZE_FORMAT ")",
190 ZFILESYSTEM_HUGETLBFS, _block_size, ZGranuleSize);
191 return;
192 }
193
194 // Successfully initialized
195 _initialized = true;
196 }
197
198 int ZPhysicalMemoryBacking::create_mem_fd(const char* name) const {
199 // Create file name
200 char filename[PATH_MAX];
201 snprintf(filename, sizeof(filename), "%s%s", name, ZLargePages::is_explicit() ? ".hugetlb" : "");
202
203 // Create file
204 const int extra_flags = ZLargePages::is_explicit() ? MFD_HUGETLB : 0;
205 const int fd = ZSyscall::memfd_create(filename, MFD_CLOEXEC | extra_flags);
206 if (fd == -1) {
207 ZErrno err;
208 log_debug(gc, init)("Failed to create memfd file (%s)",
209 ((ZLargePages::is_explicit() && err == EINVAL) ? "Hugepages not supported" : err.to_string()));
210 return -1;
211 }
212
213 log_info(gc, init)("Heap Backing File: /memfd:%s", filename);
214
215 return fd;
216 }
217
218 int ZPhysicalMemoryBacking::create_file_fd(const char* name) const {
219 const char* const filesystem = ZLargePages::is_explicit()
220 ? ZFILESYSTEM_HUGETLBFS
221 : ZFILESYSTEM_TMPFS;
222 const char** const preferred_mountpoints = ZLargePages::is_explicit()
223 ? z_preferred_hugetlbfs_mountpoints
224 : z_preferred_tmpfs_mountpoints;
225
226 // Find mountpoint
227 ZMountPoint mountpoint(filesystem, preferred_mountpoints);
228 if (mountpoint.get() == NULL) {
229 log_error(gc)("Use -XX:AllocateHeapAt to specify the path to a %s filesystem", filesystem);
230 return -1;
231 }
232
233 // Try to create an anonymous file using the O_TMPFILE flag. Note that this
234 // flag requires kernel >= 3.11. If this fails we fall back to open/unlink.
235 const int fd_anon = os::open(mountpoint.get(), O_TMPFILE|O_EXCL|O_RDWR|O_CLOEXEC, S_IRUSR|S_IWUSR);
236 if (fd_anon == -1) {
237 ZErrno err;
238 log_debug(gc, init)("Failed to create anonymous file in %s (%s)", mountpoint.get(),
239 (err == EINVAL ? "Not supported" : err.to_string()));
240 } else {
241 // Get inode number for anonymous file
242 struct stat stat_buf;
243 if (fstat(fd_anon, &stat_buf) == -1) {
244 ZErrno err;
245 log_error(gc)("Failed to determine inode number for anonymous file (%s)", err.to_string());
246 return -1;
247 }
248
249 log_info(gc, init)("Heap Backing File: %s/#" UINT64_FORMAT, mountpoint.get(), (uint64_t)stat_buf.st_ino);
250
251 return fd_anon;
252 }
253
254 log_debug(gc, init)("Falling back to open/unlink");
255
256 // Create file name
257 char filename[PATH_MAX];
258 snprintf(filename, sizeof(filename), "%s/%s.%d", mountpoint.get(), name, os::current_process_id());
259
260 // Create file
261 const int fd = os::open(filename, O_CREAT|O_EXCL|O_RDWR|O_CLOEXEC, S_IRUSR|S_IWUSR);
262 if (fd == -1) {
263 ZErrno err;
264 log_error(gc)("Failed to create file %s (%s)", filename, err.to_string());
265 return -1;
266 }
267
268 // Unlink file
269 if (unlink(filename) == -1) {
270 ZErrno err;
271 log_error(gc)("Failed to unlink file %s (%s)", filename, err.to_string());
272 return -1;
273 }
274
275 log_info(gc, init)("Heap Backing File: %s", filename);
276
277 return fd;
278 }
279
280 int ZPhysicalMemoryBacking::create_fd(const char* name) const {
281 if (AllocateHeapAt == NULL) {
282 // If the path is not explicitly specified, then we first try to create a memfd file
283 // instead of looking for a tmpfd/hugetlbfs mount point. Note that memfd_create() might
284 // not be supported at all (requires kernel >= 3.17), or it might not support large
285 // pages (requires kernel >= 4.14). If memfd_create() fails, then we try to create a
286 // file on an accessible tmpfs or hugetlbfs mount point.
287 const int fd = create_mem_fd(name);
288 if (fd != -1) {
289 return fd;
290 }
291
292 log_debug(gc, init)("Falling back to searching for an accessible mount point");
293 }
294
295 return create_file_fd(name);
296 }
297
298 bool ZPhysicalMemoryBacking::is_initialized() const {
299 return _initialized;
300 }
301
302 void ZPhysicalMemoryBacking::warn_available_space(size_t max_capacity) const {
303 // Note that the available space on a tmpfs or a hugetlbfs filesystem
304 // will be zero if no size limit was specified when it was mounted.
305 if (_available == 0) {
306 // No size limit set, skip check
307 log_info(gc, init)("Available space on backing filesystem: N/A");
308 return;
309 }
310
311 log_info(gc, init)("Available space on backing filesystem: " SIZE_FORMAT "M", _available / M);
312
313 // Warn if the filesystem doesn't currently have enough space available to hold
314 // the max heap size. The max heap size will be capped if we later hit this limit
315 // when trying to expand the heap.
316 if (_available < max_capacity) {
317 log_warning(gc)("***** WARNING! INCORRECT SYSTEM CONFIGURATION DETECTED! *****");
318 log_warning(gc)("Not enough space available on the backing filesystem to hold the current max Java heap");
319 log_warning(gc)("size (" SIZE_FORMAT "M). Please adjust the size of the backing filesystem accordingly "
320 "(available", max_capacity / M);
321 log_warning(gc)("space is currently " SIZE_FORMAT "M). Continuing execution with the current filesystem "
322 "size could", _available / M);
323 log_warning(gc)("lead to a premature OutOfMemoryError being thrown, due to failure to commit memory.");
324 }
325 }
326
327 void ZPhysicalMemoryBacking::warn_max_map_count(size_t max_capacity) const {
328 const char* const filename = ZFILENAME_PROC_MAX_MAP_COUNT;
329 FILE* const file = fopen(filename, "r");
330 if (file == NULL) {
331 // Failed to open file, skip check
332 log_debug(gc, init)("Failed to open %s", filename);
333 return;
334 }
335
336 size_t actual_max_map_count = 0;
337 const int result = fscanf(file, SIZE_FORMAT, &actual_max_map_count);
338 fclose(file);
339 if (result != 1) {
340 // Failed to read file, skip check
341 log_debug(gc, init)("Failed to read %s", filename);
342 return;
343 }
344
345 // The required max map count is impossible to calculate exactly since subsystems
346 // other than ZGC are also creating memory mappings, and we have no control over that.
347 // However, ZGC tends to create the most mappings and dominate the total count.
348 // In the worst cases, ZGC will map each granule three times, i.e. once per heap view.
349 // We speculate that we need another 20% to allow for non-ZGC subsystems to map memory.
350 const size_t required_max_map_count = (max_capacity / ZGranuleSize) * 3 * 1.2;
351 if (actual_max_map_count < required_max_map_count) {
352 log_warning(gc)("***** WARNING! INCORRECT SYSTEM CONFIGURATION DETECTED! *****");
353 log_warning(gc)("The system limit on number of memory mappings per process might be too low for the given");
354 log_warning(gc)("max Java heap size (" SIZE_FORMAT "M). Please adjust %s to allow for at",
355 max_capacity / M, filename);
356 log_warning(gc)("least " SIZE_FORMAT " mappings (current limit is " SIZE_FORMAT "). Continuing execution "
357 "with the current", required_max_map_count, actual_max_map_count);
358 log_warning(gc)("limit could lead to a premature OutOfMemoryError being thrown, due to failure to map memory.");
359 }
360 }
361
362 void ZPhysicalMemoryBacking::warn_commit_limits(size_t max_capacity) const {
363 // Warn if available space is too low
364 warn_available_space(max_capacity);
365
366 // Warn if max map count is too low
367 warn_max_map_count(max_capacity);
368 }
369
370 bool ZPhysicalMemoryBacking::is_tmpfs() const {
371 return _filesystem == TMPFS_MAGIC;
372 }
373
374 bool ZPhysicalMemoryBacking::is_hugetlbfs() const {
375 return _filesystem == HUGETLBFS_MAGIC;
376 }
377
378 bool ZPhysicalMemoryBacking::tmpfs_supports_transparent_huge_pages() const {
379 // If the shmem_enabled file exists and is readable then we
380 // know the kernel supports transparent huge pages for tmpfs.
381 return access(ZFILENAME_SHMEM_ENABLED, R_OK) == 0;
382 }
383
384 ZErrno ZPhysicalMemoryBacking::fallocate_compat_mmap_hugetlbfs(size_t offset, size_t length, bool touch) const {
385 // On hugetlbfs, mapping a file segment will fail immediately, without
386 // the need to touch the mapped pages first, if there aren't enough huge
387 // pages available to back the mapping.
388 void* const addr = mmap(0, length, PROT_READ|PROT_WRITE, MAP_SHARED, _fd, offset);
389 if (addr == MAP_FAILED) {
390 // Failed
391 return errno;
392 }
393
394 // Once mapped, the huge pages are only reserved. We need to touch them
395 // to associate them with the file segment. Note that we can not punch
396 // hole in file segments which only have reserved pages.
397 if (touch) {
398 char* const start = (char*)addr;
399 char* const end = start + length;
400 os::pretouch_memory(start, end, _block_size);
401 }
402
403 // Unmap again. From now on, the huge pages that were mapped are allocated
404 // to this file. There's no risk of getting a SIGBUS when mapping and
405 // touching these pages again.
406 if (munmap(addr, length) == -1) {
407 // Failed
408 return errno;
409 }
410
411 // Success
412 return 0;
413 }
414
415 static bool safe_touch_mapping(void* addr, size_t length, size_t page_size) {
416 char* const start = (char*)addr;
417 char* const end = start + length;
418
419 // Touching a mapping that can't be backed by memory will generate a
420 // SIGBUS. By using SafeFetch32 any SIGBUS will be safely caught and
421 // handled. On tmpfs, doing a fetch (rather than a store) is enough
422 // to cause backing pages to be allocated (there's no zero-page to
423 // worry about).
424 for (char *p = start; p < end; p += page_size) {
425 if (SafeFetch32((int*)p, -1) == -1) {
426 // Failed
427 return false;
428 }
429 }
430
431 // Success
432 return true;
433 }
434
435 ZErrno ZPhysicalMemoryBacking::fallocate_compat_mmap_tmpfs(size_t offset, size_t length) const {
436 // On tmpfs, we need to touch the mapped pages to figure out
437 // if there are enough pages available to back the mapping.
438 void* const addr = mmap(0, length, PROT_READ|PROT_WRITE, MAP_SHARED, _fd, offset);
439 if (addr == MAP_FAILED) {
440 // Failed
441 return errno;
442 }
443
444 // Advise mapping to use transparent huge pages
445 os::realign_memory((char*)addr, length, os::large_page_size());
446
447 // Touch the mapping (safely) to make sure it's backed by memory
448 const bool backed = safe_touch_mapping(addr, length, _block_size);
449
450 // Unmap again. If successfully touched, the backing memory will
451 // be allocated to this file. There's no risk of getting a SIGBUS
452 // when mapping and touching these pages again.
453 if (munmap(addr, length) == -1) {
454 // Failed
455 return errno;
456 }
457
458 // Success
459 return backed ? 0 : ENOMEM;
460 }
461
462 ZErrno ZPhysicalMemoryBacking::fallocate_compat_pwrite(size_t offset, size_t length) const {
463 uint8_t data = 0;
464
465 // Allocate backing memory by writing to each block
466 for (size_t pos = offset; pos < offset + length; pos += _block_size) {
467 if (pwrite(_fd, &data, sizeof(data), pos) == -1) {
468 // Failed
469 return errno;
470 }
471 }
472
473 // Success
474 return 0;
475 }
476
477 ZErrno ZPhysicalMemoryBacking::fallocate_fill_hole_compat(size_t offset, size_t length) const {
478 // fallocate(2) is only supported by tmpfs since Linux 3.5, and by hugetlbfs
479 // since Linux 4.3. When fallocate(2) is not supported we emulate it using
480 // mmap/munmap (for hugetlbfs and tmpfs with transparent huge pages) or pwrite
481 // (for tmpfs without transparent huge pages and other filesystem types).
482 if (ZLargePages::is_explicit()) {
483 return fallocate_compat_mmap_hugetlbfs(offset, length, false /* touch */);
484 } else if (ZLargePages::is_transparent()) {
485 return fallocate_compat_mmap_tmpfs(offset, length);
486 } else {
487 return fallocate_compat_pwrite(offset, length);
488 }
489 }
490
491 ZErrno ZPhysicalMemoryBacking::fallocate_fill_hole_syscall(size_t offset, size_t length) const {
492 const int mode = 0; // Allocate
493 const int res = ZSyscall::fallocate(_fd, mode, offset, length);
494 if (res == -1) {
495 // Failed
496 return errno;
497 }
498
499 // Success
500 return 0;
501 }
502
503 ZErrno ZPhysicalMemoryBacking::fallocate_fill_hole(size_t offset, size_t length) const {
504 // Using compat mode is more efficient when allocating space on hugetlbfs.
505 // Note that allocating huge pages this way will only reserve them, and not
506 // associate them with segments of the file. We must guarantee that we at
507 // some point touch these segments, otherwise we can not punch hole in them.
508 // Also note that we need to use compat mode when using transparent huge pages,
509 // since we need to use madvise(2) on the mapping before the page is allocated.
510 if (z_fallocate_supported && !ZLargePages::is_enabled()) {
511 const ZErrno err = fallocate_fill_hole_syscall(offset, length);
512 if (!err) {
513 // Success
514 return 0;
515 }
516
517 if (err != ENOSYS && err != EOPNOTSUPP) {
518 // Failed
519 return err;
520 }
521
522 // Not supported
523 log_debug(gc)("Falling back to fallocate() compatibility mode");
524 z_fallocate_supported = false;
525 }
526
527 return fallocate_fill_hole_compat(offset, length);
528 }
529
530 ZErrno ZPhysicalMemoryBacking::fallocate_punch_hole(size_t offset, size_t length) const {
531 if (ZLargePages::is_explicit()) {
532 // We can only punch hole in pages that have been touched. Non-touched
533 // pages are only reserved, and not associated with any specific file
534 // segment. We don't know which pages have been previously touched, so
535 // we always touch them here to guarantee that we can punch hole.
536 const ZErrno err = fallocate_compat_mmap_hugetlbfs(offset, length, true /* touch */);
537 if (err) {
538 // Failed
539 return err;
540 }
541 }
542
543 const int mode = FALLOC_FL_PUNCH_HOLE|FALLOC_FL_KEEP_SIZE;
544 if (ZSyscall::fallocate(_fd, mode, offset, length) == -1) {
545 // Failed
546 return errno;
547 }
548
549 // Success
550 return 0;
551 }
552
553 ZErrno ZPhysicalMemoryBacking::split_and_fallocate(bool punch_hole, size_t offset, size_t length) const {
554 // Try first half
555 const size_t offset0 = offset;
556 const size_t length0 = align_up(length / 2, _block_size);
557 const ZErrno err0 = fallocate(punch_hole, offset0, length0);
558 if (err0) {
559 return err0;
560 }
561
562 // Try second half
563 const size_t offset1 = offset0 + length0;
564 const size_t length1 = length - length0;
565 const ZErrno err1 = fallocate(punch_hole, offset1, length1);
566 if (err1) {
567 return err1;
568 }
569
570 // Success
571 return 0;
572 }
573
574 ZErrno ZPhysicalMemoryBacking::fallocate(bool punch_hole, size_t offset, size_t length) const {
575 assert(is_aligned(offset, _block_size), "Invalid offset");
576 assert(is_aligned(length, _block_size), "Invalid length");
577
578 const ZErrno err = punch_hole ? fallocate_punch_hole(offset, length) : fallocate_fill_hole(offset, length);
579 if (err == EINTR && length > _block_size) {
580 // Calling fallocate(2) with a large length can take a long time to
581 // complete. When running profilers, such as VTune, this syscall will
582 // be constantly interrupted by signals. Expanding the file in smaller
583 // steps avoids this problem.
584 return split_and_fallocate(punch_hole, offset, length);
585 }
586
587 return err;
588 }
589
590 bool ZPhysicalMemoryBacking::commit_inner(size_t offset, size_t length) const {
591 log_trace(gc, heap)("Committing memory: " SIZE_FORMAT "M-" SIZE_FORMAT "M (" SIZE_FORMAT "M)",
592 offset / M, (offset + length) / M, length / M);
593
594 retry:
595 const ZErrno err = fallocate(false /* punch_hole */, offset, length);
596 if (err) {
597 if (err == ENOSPC && !is_init_completed() && ZLargePages::is_explicit() && z_fallocate_hugetlbfs_attempts-- > 0) {
598 // If we fail to allocate during initialization, due to lack of space on
599 // the hugetlbfs filesystem, then we wait and retry a few times before
600 // giving up. Otherwise there is a risk that running JVMs back-to-back
601 // will fail, since there is a delay between process termination and the
602 // huge pages owned by that process being returned to the huge page pool
603 // and made available for new allocations.
604 log_debug(gc, init)("Failed to commit memory (%s), retrying", err.to_string());
605
606 // Wait and retry in one second, in the hope that huge pages will be
607 // available by then.
608 sleep(1);
609 goto retry;
610 }
611
612 // Failed
613 log_error(gc)("Failed to commit memory (%s)", err.to_string());
614 return false;
615 }
616
617 // Success
618 return true;
619 }
620
621 static int offset_to_node(size_t offset) {
622 const GrowableArray<int>* mapping = os::Linux::numa_nindex_to_node();
623 const size_t nindex = (offset >> ZGranuleSizeShift) % mapping->length();
624 return mapping->at((int)nindex);
625 }
626
627 size_t ZPhysicalMemoryBacking::commit_numa_interleaved(size_t offset, size_t length) const {
628 size_t committed = 0;
629
630 // Commit one granule at a time, so that each granule
631 // can be allocated from a different preferred node.
632 while (committed < length) {
633 const size_t granule_offset = offset + committed;
634
635 // Setup NUMA policy to allocate memory from a preferred node
636 os::Linux::numa_set_preferred(offset_to_node(granule_offset));
637
638 if (!commit_inner(granule_offset, ZGranuleSize)) {
639 // Failed
640 break;
641 }
642
643 committed += ZGranuleSize;
644 }
645
646 // Restore NUMA policy
647 os::Linux::numa_set_preferred(-1);
648
649 return committed;
650 }
651
652 size_t ZPhysicalMemoryBacking::commit_default(size_t offset, size_t length) const {
653 // Try to commit the whole region
654 if (commit_inner(offset, length)) {
655 // Success
656 return length;
657 }
658
659 // Failed, try to commit as much as possible
660 size_t start = offset;
661 size_t end = offset + length;
662
663 for (;;) {
664 length = align_down((end - start) / 2, ZGranuleSize);
665 if (length < ZGranuleSize) {
666 // Done, don't commit more
667 return start - offset;
668 }
669
670 if (commit_inner(start, length)) {
671 // Success, try commit more
672 start += length;
673 } else {
674 // Failed, try commit less
675 end -= length;
676 }
677 }
678 }
679
680 size_t ZPhysicalMemoryBacking::commit(size_t offset, size_t length) const {
681 if (ZNUMA::is_enabled() && !ZLargePages::is_explicit()) {
682 // To get granule-level NUMA interleaving when using non-large pages,
683 // we must explicitly interleave the memory at commit/fallocate time.
684 return commit_numa_interleaved(offset, length);
685 }
686
687 return commit_default(offset, length);
688 }
689
690 size_t ZPhysicalMemoryBacking::uncommit(size_t offset, size_t length) const {
691 log_trace(gc, heap)("Uncommitting memory: " SIZE_FORMAT "M-" SIZE_FORMAT "M (" SIZE_FORMAT "M)",
692 offset / M, (offset + length) / M, length / M);
693
694 const ZErrno err = fallocate(true /* punch_hole */, offset, length);
695 if (err) {
696 log_error(gc)("Failed to uncommit memory (%s)", err.to_string());
697 return 0;
698 }
699
700 return length;
701 }
702
703 bool ZPhysicalMemoryBacking::map(uintptr_t addr, size_t size, uintptr_t offset) const {
704 const void* const res = mmap((void*)addr, size, PROT_READ|PROT_WRITE, MAP_FIXED|MAP_SHARED, _fd, offset);
705 if (res == MAP_FAILED) {
706 ZErrno err;
707 log_error(gc)("Failed to map memory (%s)", err.to_string());
708 return false;
709 }
710
711 return true;
712 }
713
714 void ZPhysicalMemoryBacking::unmap(uintptr_t addr, size_t size) const {
715 // Note that we must keep the address space reservation intact and just detach
716 // the backing memory. For this reason we map a new anonymous, non-accessible
717 // and non-reserved page over the mapping instead of actually unmapping.
718 const void* const res = mmap((void*)addr, size, PROT_NONE, MAP_FIXED | MAP_ANONYMOUS | MAP_PRIVATE | MAP_NORESERVE, -1, 0);
719 if (res == MAP_FAILED) {
720 ZErrno err;
721 log_error(gc)("Failed to map memory (%s)", err.to_string());
722 }
723 }