3069 char* const start_aligned = (char*) align_ptr_up(start, alignment);
3070 char* const end_aligned = start_aligned + bytes;
3071 char* const end = start + extra_size;
3072 if (start_aligned > start) {
3073 ::munmap(start, start_aligned - start);
3074 }
3075 if (end_aligned < end) {
3076 ::munmap(end_aligned, end - end_aligned);
3077 }
3078 start = start_aligned;
3079 }
3080 }
3081 return start;
3082 }
3083
3084 static int anon_munmap(char * addr, size_t size) {
3085 return ::munmap(addr, size) == 0;
3086 }
3087
3088 char* os::pd_reserve_memory(size_t bytes, char* requested_addr,
3089 size_t alignment_hint) {
3090 return anon_mmap(requested_addr, bytes, (requested_addr != NULL));
3091 }
3092
3093 bool os::pd_release_memory(char* addr, size_t size) {
3094 return anon_munmap(addr, size);
3095 }
3096
3097 static bool linux_mprotect(char* addr, size_t size, int prot) {
3098 // Linux wants the mprotect address argument to be page aligned.
3099 char* bottom = (char*)align_size_down((intptr_t)addr, os::Linux::page_size());
3100
3101 // According to SUSv3, mprotect() should only be used with mappings
3102 // established by mmap(), and mmap() always maps whole pages. Unaligned
3103 // 'addr' likely indicates problem in the VM (e.g. trying to change
3104 // protection of malloc'ed or statically allocated memory). Check the
3105 // caller if you hit this assert.
3106 assert(addr == bottom, "sanity check");
3107
3108 size = align_size_up(pointer_delta(addr, bottom, 1) + size, os::Linux::page_size());
3109 return ::mprotect(bottom, size, prot) == 0;
3524 bool exec) {
3525 assert(UseLargePages && UseHugeTLBFS, "only for Huge TLBFS large pages");
3526 assert(is_size_aligned(bytes, os::large_page_size()), "Unaligned size");
3527 assert(is_ptr_aligned(req_addr, os::large_page_size()), "Unaligned address");
3528
3529 int prot = exec ? PROT_READ|PROT_WRITE|PROT_EXEC : PROT_READ|PROT_WRITE;
3530 char* addr = (char*)::mmap(req_addr, bytes, prot,
3531 MAP_PRIVATE|MAP_ANONYMOUS|MAP_HUGETLB,
3532 -1, 0);
3533
3534 if (addr == MAP_FAILED) {
3535 warn_on_large_pages_failure(req_addr, bytes, errno);
3536 return NULL;
3537 }
3538
3539 assert(is_ptr_aligned(addr, os::large_page_size()), "Must be");
3540
3541 return addr;
3542 }
3543
3544 // Reserve memory using mmap(MAP_HUGETLB).
3545 // - bytes shall be a multiple of alignment.
3546 // - req_addr can be NULL. If not NULL, it must be a multiple of alignment.
3547 // - alignment sets the alignment at which memory shall be allocated.
3548 // It must be a multiple of allocation granularity.
3549 // Returns address of memory or NULL. If req_addr was not NULL, will only return
3550 // req_addr or NULL.
3551 char* os::Linux::reserve_memory_special_huge_tlbfs_mixed(size_t bytes,
3552 size_t alignment,
3553 char* req_addr,
3554 bool exec) {
3555 size_t large_page_size = os::large_page_size();
3556 assert(bytes >= large_page_size, "Shouldn't allocate large pages for small sizes");
3557
3558 assert(is_ptr_aligned(req_addr, alignment), "Must be");
3559 assert(is_size_aligned(bytes, alignment), "Must be");
3560
3561 // First reserve - but not commit - the address range in small pages.
3562 char* const start = anon_mmap_aligned(bytes, alignment, req_addr);
3563
3714 }
3715
3716 // With SysV SHM the entire memory region must be allocated as shared
3717 // memory.
3718 // HugeTLBFS allows application to commit large page memory on demand.
3719 // However, when committing memory with HugeTLBFS fails, the region
3720 // that was supposed to be committed will lose the old reservation
3721 // and allow other threads to steal that memory region. Because of this
3722 // behavior we can't commit HugeTLBFS memory.
3723 bool os::can_commit_large_page_memory() {
3724 return UseTransparentHugePages;
3725 }
3726
3727 bool os::can_execute_large_page_memory() {
3728 return UseTransparentHugePages || UseHugeTLBFS;
3729 }
3730
3731 // Reserve memory at an arbitrary address, only if that area is
3732 // available (and not reserved for something else).
3733
3734 char* os::pd_attempt_reserve_memory_at(size_t bytes, char* requested_addr) {
3735 const int max_tries = 10;
3736 char* base[max_tries];
3737 size_t size[max_tries];
3738 const size_t gap = 0x000000;
3739
3740 // Assert only that the size is a multiple of the page size, since
3741 // that's all that mmap requires, and since that's all we really know
3742 // about at this low abstraction level. If we need higher alignment,
3743 // we can either pass an alignment to this method or verify alignment
3744 // in one of the methods further up the call chain. See bug 5044738.
3745 assert(bytes % os::vm_page_size() == 0, "reserving unexpected size block");
3746
3747 // Repeatedly allocate blocks until the block is allocated at the
3748 // right spot.
3749
3750 // Linux mmap allows caller to pass an address as hint; give it a try first,
3751 // if kernel honors the hint then we can return immediately.
3752 char * addr = anon_mmap(requested_addr, bytes, false);
3753 if (addr == requested_addr) {
3754 return requested_addr;
3755 }
3756
3757 if (addr != NULL) {
3758 // mmap() is successful but it fails to reserve at the requested address
3759 anon_munmap(addr, bytes);
3760 }
3761
3762 int i;
3763 for (i = 0; i < max_tries; ++i) {
3764 base[i] = reserve_memory(bytes);
3765
3766 if (base[i] != NULL) {
3767 // Is this the block we wanted?
3768 if (base[i] == requested_addr) {
3769 size[i] = bytes;
3770 break;
3771 }
3772
3773 // Does this overlap the block we wanted? Give back the overlapped
3774 // parts and try again.
3775
3782 ptrdiff_t bottom_overlap = base[i] + bytes - requested_addr;
3783 if (bottom_overlap >= 0 && (size_t)bottom_overlap < bytes) {
3784 unmap_memory(requested_addr, bottom_overlap);
3785 size[i] = bytes - bottom_overlap;
3786 } else {
3787 size[i] = bytes;
3788 }
3789 }
3790 }
3791 }
3792
3793 // Give back the unused reserved pieces.
3794
3795 for (int j = 0; j < i; ++j) {
3796 if (base[j] != NULL) {
3797 unmap_memory(base[j], size[j]);
3798 }
3799 }
3800
3801 if (i < max_tries) {
3802 return requested_addr;
3803 } else {
3804 return NULL;
3805 }
3806 }
3807
3808 size_t os::read(int fd, void *buf, unsigned int nBytes) {
3809 return ::read(fd, buf, nBytes);
3810 }
3811
3812 size_t os::read_at(int fd, void *buf, unsigned int nBytes, jlong offset) {
3813 return ::pread(fd, buf, nBytes, offset);
3814 }
3815
3816 // Short sleep, direct OS call.
3817 //
3818 // Note: certain versions of Linux CFS scheduler (since 2.6.23) do not guarantee
3819 // sched_yield(2) will actually give up the CPU:
3820 //
3821 // * Alone on this pariticular CPU, keeps running.
|
3069 char* const start_aligned = (char*) align_ptr_up(start, alignment);
3070 char* const end_aligned = start_aligned + bytes;
3071 char* const end = start + extra_size;
3072 if (start_aligned > start) {
3073 ::munmap(start, start_aligned - start);
3074 }
3075 if (end_aligned < end) {
3076 ::munmap(end_aligned, end - end_aligned);
3077 }
3078 start = start_aligned;
3079 }
3080 }
3081 return start;
3082 }
3083
3084 static int anon_munmap(char * addr, size_t size) {
3085 return ::munmap(addr, size) == 0;
3086 }
3087
3088 char* os::pd_reserve_memory(size_t bytes, char* requested_addr,
3089 size_t alignment_hint, int file_desc) {
3090 if (file_desc != -1) {
3091 return map_memory_to_file(requested_addr, bytes, file_desc);
3092 }
3093 return anon_mmap(requested_addr, bytes, (requested_addr != NULL));
3094 }
3095
3096 bool os::pd_release_memory(char* addr, size_t size) {
3097 return anon_munmap(addr, size);
3098 }
3099
3100 static bool linux_mprotect(char* addr, size_t size, int prot) {
3101 // Linux wants the mprotect address argument to be page aligned.
3102 char* bottom = (char*)align_size_down((intptr_t)addr, os::Linux::page_size());
3103
3104 // According to SUSv3, mprotect() should only be used with mappings
3105 // established by mmap(), and mmap() always maps whole pages. Unaligned
3106 // 'addr' likely indicates problem in the VM (e.g. trying to change
3107 // protection of malloc'ed or statically allocated memory). Check the
3108 // caller if you hit this assert.
3109 assert(addr == bottom, "sanity check");
3110
3111 size = align_size_up(pointer_delta(addr, bottom, 1) + size, os::Linux::page_size());
3112 return ::mprotect(bottom, size, prot) == 0;
3527 bool exec) {
3528 assert(UseLargePages && UseHugeTLBFS, "only for Huge TLBFS large pages");
3529 assert(is_size_aligned(bytes, os::large_page_size()), "Unaligned size");
3530 assert(is_ptr_aligned(req_addr, os::large_page_size()), "Unaligned address");
3531
3532 int prot = exec ? PROT_READ|PROT_WRITE|PROT_EXEC : PROT_READ|PROT_WRITE;
3533 char* addr = (char*)::mmap(req_addr, bytes, prot,
3534 MAP_PRIVATE|MAP_ANONYMOUS|MAP_HUGETLB,
3535 -1, 0);
3536
3537 if (addr == MAP_FAILED) {
3538 warn_on_large_pages_failure(req_addr, bytes, errno);
3539 return NULL;
3540 }
3541
3542 assert(is_ptr_aligned(addr, os::large_page_size()), "Must be");
3543
3544 return addr;
3545 }
3546
3547 // Helper function to create a temp file in the given directory.
3548 int os::create_file_for_heap(const char* dir, size_t size) {
3549
3550 const char name_template[] = "/jvmheap.XXXXXX";
3551
3552 char *fullname = (char*)alloca(strlen(dir) + sizeof(name_template));
3553 (void)strcpy(fullname, dir);
3554 (void)strcat(fullname, name_template);
3555
3556 sigset_t set, oldset;
3557 sigfillset(&set);
3558
3559 // block all signals while we do the file operation.
3560 (void)sigprocmask(SIG_BLOCK, &set, &oldset);
3561
3562 // set the file creation mask.
3563 mode_t new_mask = S_IRUSR | S_IWUSR;
3564 mode_t prev_umask = umask(new_mask);
3565
3566 // create a new file.
3567 int fd = mkstemp(fullname);
3568
3569 // reset the file creation mask.
3570 umask(prev_umask);
3571
3572 if (fd < 0) {
3573 warning("Failure to create file %s for heap", fullname);
3574 return -1;
3575 }
3576
3577 // delete the name from the filesystem. When 'fd' is closed, the file (and space) will be deleted.
3578 (void)unlink(fullname);
3579
3580 // reset the signal mask.
3581 (void)sigprocmask(SIG_SETMASK, &oldset, NULL);
3582
3583 return fd;
3584 }
3585
3586 // Map the given address range to the provided file descriptor.
3587 char* os::map_memory_to_file(char* base, size_t size, int fd) {
3588 assert(fd != -1, "File descriptor is not valid");
3589
3590 // allocate space for the file
3591 if ((errno = posix_fallocate(fd, 0, (off_t)size)) != 0) {
3592 vm_exit_during_initialization(err_msg("Error in mapping Java heap at the given filesystem directory"));
3593 return NULL;
3594 }
3595
3596 int prot = PROT_READ | PROT_WRITE;
3597 int flags = MAP_SHARED;
3598 if( base != NULL) {
3599 flags |= MAP_FIXED;
3600 }
3601 char* addr = (char*)mmap(base, size, prot, flags, fd, 0);
3602
3603 if (addr == MAP_FAILED || (base != NULL && addr != base)) {
3604 if (addr != MAP_FAILED) {
3605 if (!pd_release_memory(addr, size)) {
3606 warning("Could not release memory on unsuccessful file mapping");
3607 }
3608 }
3609 return NULL;
3610 }
3611
3612 return addr;
3613 }
3614
3615 // Reserve memory using mmap(MAP_HUGETLB).
3616 // - bytes shall be a multiple of alignment.
3617 // - req_addr can be NULL. If not NULL, it must be a multiple of alignment.
3618 // - alignment sets the alignment at which memory shall be allocated.
3619 // It must be a multiple of allocation granularity.
3620 // Returns address of memory or NULL. If req_addr was not NULL, will only return
3621 // req_addr or NULL.
3622 char* os::Linux::reserve_memory_special_huge_tlbfs_mixed(size_t bytes,
3623 size_t alignment,
3624 char* req_addr,
3625 bool exec) {
3626 size_t large_page_size = os::large_page_size();
3627 assert(bytes >= large_page_size, "Shouldn't allocate large pages for small sizes");
3628
3629 assert(is_ptr_aligned(req_addr, alignment), "Must be");
3630 assert(is_size_aligned(bytes, alignment), "Must be");
3631
3632 // First reserve - but not commit - the address range in small pages.
3633 char* const start = anon_mmap_aligned(bytes, alignment, req_addr);
3634
3785 }
3786
3787 // With SysV SHM the entire memory region must be allocated as shared
3788 // memory.
3789 // HugeTLBFS allows application to commit large page memory on demand.
3790 // However, when committing memory with HugeTLBFS fails, the region
3791 // that was supposed to be committed will lose the old reservation
3792 // and allow other threads to steal that memory region. Because of this
3793 // behavior we can't commit HugeTLBFS memory.
3794 bool os::can_commit_large_page_memory() {
3795 return UseTransparentHugePages;
3796 }
3797
3798 bool os::can_execute_large_page_memory() {
3799 return UseTransparentHugePages || UseHugeTLBFS;
3800 }
3801
3802 // Reserve memory at an arbitrary address, only if that area is
3803 // available (and not reserved for something else).
3804
3805 char* os::pd_attempt_reserve_memory_at(size_t bytes, char* requested_addr, int file_desc) {
3806 const int max_tries = 10;
3807 char* base[max_tries];
3808 size_t size[max_tries];
3809 const size_t gap = 0x000000;
3810
3811 // Assert only that the size is a multiple of the page size, since
3812 // that's all that mmap requires, and since that's all we really know
3813 // about at this low abstraction level. If we need higher alignment,
3814 // we can either pass an alignment to this method or verify alignment
3815 // in one of the methods further up the call chain. See bug 5044738.
3816 assert(bytes % os::vm_page_size() == 0, "reserving unexpected size block");
3817
3818 // Repeatedly allocate blocks until the block is allocated at the
3819 // right spot.
3820
3821 // Linux mmap allows caller to pass an address as hint; give it a try first,
3822 // if kernel honors the hint then we can return immediately.
3823 char * addr = anon_mmap(requested_addr, bytes, false);
3824 if (addr == requested_addr) {
3825 if (file_desc == -1 || map_memory_to_file(requested_addr, bytes, file_desc) != NULL) {
3826 return requested_addr;
3827 }
3828 }
3829
3830 if (addr != NULL) {
3831 // mmap() is successful but it fails to reserve at the requested address
3832 anon_munmap(addr, bytes);
3833 }
3834
3835 int i;
3836 for (i = 0; i < max_tries; ++i) {
3837 base[i] = reserve_memory(bytes);
3838
3839 if (base[i] != NULL) {
3840 // Is this the block we wanted?
3841 if (base[i] == requested_addr) {
3842 size[i] = bytes;
3843 break;
3844 }
3845
3846 // Does this overlap the block we wanted? Give back the overlapped
3847 // parts and try again.
3848
3855 ptrdiff_t bottom_overlap = base[i] + bytes - requested_addr;
3856 if (bottom_overlap >= 0 && (size_t)bottom_overlap < bytes) {
3857 unmap_memory(requested_addr, bottom_overlap);
3858 size[i] = bytes - bottom_overlap;
3859 } else {
3860 size[i] = bytes;
3861 }
3862 }
3863 }
3864 }
3865
3866 // Give back the unused reserved pieces.
3867
3868 for (int j = 0; j < i; ++j) {
3869 if (base[j] != NULL) {
3870 unmap_memory(base[j], size[j]);
3871 }
3872 }
3873
3874 if (i < max_tries) {
3875 if (file_desc != -1 && map_memory_to_file(requested_addr, bytes, file_desc) == NULL) {
3876 vm_exit_during_initialization(err_msg("Error in mapping Java heap at the given filesystem directory"));
3877 }
3878 return requested_addr;
3879 } else {
3880 return NULL;
3881 }
3882 }
3883
3884 size_t os::read(int fd, void *buf, unsigned int nBytes) {
3885 return ::read(fd, buf, nBytes);
3886 }
3887
3888 size_t os::read_at(int fd, void *buf, unsigned int nBytes, jlong offset) {
3889 return ::pread(fd, buf, nBytes, offset);
3890 }
3891
3892 // Short sleep, direct OS call.
3893 //
3894 // Note: certain versions of Linux CFS scheduler (since 2.6.23) do not guarantee
3895 // sched_yield(2) will actually give up the CPU:
3896 //
3897 // * Alone on this pariticular CPU, keeps running.
|