3094 char* const start_aligned = (char*) align_ptr_up(start, alignment);
3095 char* const end_aligned = start_aligned + bytes;
3096 char* const end = start + extra_size;
3097 if (start_aligned > start) {
3098 ::munmap(start, start_aligned - start);
3099 }
3100 if (end_aligned < end) {
3101 ::munmap(end_aligned, end - end_aligned);
3102 }
3103 start = start_aligned;
3104 }
3105 }
3106 return start;
3107 }
3108
3109 static int anon_munmap(char * addr, size_t size) {
3110 return ::munmap(addr, size) == 0;
3111 }
3112
3113 char* os::pd_reserve_memory(size_t bytes, char* requested_addr,
3114 size_t alignment_hint) {
3115 return anon_mmap(requested_addr, bytes, (requested_addr != NULL));
3116 }
3117
3118 bool os::pd_release_memory(char* addr, size_t size) {
3119 return anon_munmap(addr, size);
3120 }
3121
3122 static bool linux_mprotect(char* addr, size_t size, int prot) {
3123 // Linux wants the mprotect address argument to be page aligned.
3124 char* bottom = (char*)align_size_down((intptr_t)addr, os::Linux::page_size());
3125
3126 // According to SUSv3, mprotect() should only be used with mappings
3127 // established by mmap(), and mmap() always maps whole pages. Unaligned
3128 // 'addr' likely indicates problem in the VM (e.g. trying to change
3129 // protection of malloc'ed or statically allocated memory). Check the
3130 // caller if you hit this assert.
3131 assert(addr == bottom, "sanity check");
3132
3133 size = align_size_up(pointer_delta(addr, bottom, 1) + size, os::Linux::page_size());
3134 return ::mprotect(bottom, size, prot) == 0;
3542 bool exec) {
3543 assert(UseLargePages && UseHugeTLBFS, "only for Huge TLBFS large pages");
3544 assert(is_size_aligned(bytes, os::large_page_size()), "Unaligned size");
3545 assert(is_ptr_aligned(req_addr, os::large_page_size()), "Unaligned address");
3546
3547 int prot = exec ? PROT_READ|PROT_WRITE|PROT_EXEC : PROT_READ|PROT_WRITE;
3548 char* addr = (char*)::mmap(req_addr, bytes, prot,
3549 MAP_PRIVATE|MAP_ANONYMOUS|MAP_HUGETLB,
3550 -1, 0);
3551
3552 if (addr == MAP_FAILED) {
3553 warn_on_large_pages_failure(req_addr, bytes, errno);
3554 return NULL;
3555 }
3556
3557 assert(is_ptr_aligned(addr, os::large_page_size()), "Must be");
3558
3559 return addr;
3560 }
3561
3562 // Reserve memory using mmap(MAP_HUGETLB).
3563 // - bytes shall be a multiple of alignment.
3564 // - req_addr can be NULL. If not NULL, it must be a multiple of alignment.
3565 // - alignment sets the alignment at which memory shall be allocated.
3566 // It must be a multiple of allocation granularity.
3567 // Returns address of memory or NULL. If req_addr was not NULL, will only return
3568 // req_addr or NULL.
3569 char* os::Linux::reserve_memory_special_huge_tlbfs_mixed(size_t bytes,
3570 size_t alignment,
3571 char* req_addr,
3572 bool exec) {
3573 size_t large_page_size = os::large_page_size();
3574 assert(bytes >= large_page_size, "Shouldn't allocate large pages for small sizes");
3575
3576 assert(is_ptr_aligned(req_addr, alignment), "Must be");
3577 assert(is_size_aligned(bytes, alignment), "Must be");
3578
3579 // First reserve - but not commit - the address range in small pages.
3580 char* const start = anon_mmap_aligned(bytes, alignment, req_addr);
3581
3732 }
3733
3734 // With SysV SHM the entire memory region must be allocated as shared
3735 // memory.
3736 // HugeTLBFS allows application to commit large page memory on demand.
3737 // However, when committing memory with HugeTLBFS fails, the region
3738 // that was supposed to be committed will lose the old reservation
3739 // and allow other threads to steal that memory region. Because of this
3740 // behavior we can't commit HugeTLBFS memory.
3741 bool os::can_commit_large_page_memory() {
3742 return UseTransparentHugePages;
3743 }
3744
3745 bool os::can_execute_large_page_memory() {
3746 return UseTransparentHugePages || UseHugeTLBFS;
3747 }
3748
3749 // Reserve memory at an arbitrary address, only if that area is
3750 // available (and not reserved for something else).
3751
3752 char* os::pd_attempt_reserve_memory_at(size_t bytes, char* requested_addr) {
3753 const int max_tries = 10;
3754 char* base[max_tries];
3755 size_t size[max_tries];
3756 const size_t gap = 0x000000;
3757
3758 // Assert only that the size is a multiple of the page size, since
3759 // that's all that mmap requires, and since that's all we really know
3760 // about at this low abstraction level. If we need higher alignment,
3761 // we can either pass an alignment to this method or verify alignment
3762 // in one of the methods further up the call chain. See bug 5044738.
3763 assert(bytes % os::vm_page_size() == 0, "reserving unexpected size block");
3764
3765 // Repeatedly allocate blocks until the block is allocated at the
3766 // right spot.
3767
3768 // Linux mmap allows caller to pass an address as hint; give it a try first,
3769 // if kernel honors the hint then we can return immediately.
3770 char * addr = anon_mmap(requested_addr, bytes, false);
3771 if (addr == requested_addr) {
3772 return requested_addr;
3773 }
3774
3775 if (addr != NULL) {
3776 // mmap() is successful but it fails to reserve at the requested address
3777 anon_munmap(addr, bytes);
3778 }
3779
3780 int i;
3781 for (i = 0; i < max_tries; ++i) {
3782 base[i] = reserve_memory(bytes);
3783
3784 if (base[i] != NULL) {
3785 // Is this the block we wanted?
3786 if (base[i] == requested_addr) {
3787 size[i] = bytes;
3788 break;
3789 }
3790
3791 // Does this overlap the block we wanted? Give back the overlapped
3792 // parts and try again.
3793
3800 ptrdiff_t bottom_overlap = base[i] + bytes - requested_addr;
3801 if (bottom_overlap >= 0 && (size_t)bottom_overlap < bytes) {
3802 unmap_memory(requested_addr, bottom_overlap);
3803 size[i] = bytes - bottom_overlap;
3804 } else {
3805 size[i] = bytes;
3806 }
3807 }
3808 }
3809 }
3810
3811 // Give back the unused reserved pieces.
3812
3813 for (int j = 0; j < i; ++j) {
3814 if (base[j] != NULL) {
3815 unmap_memory(base[j], size[j]);
3816 }
3817 }
3818
3819 if (i < max_tries) {
3820 return requested_addr;
3821 } else {
3822 return NULL;
3823 }
3824 }
3825
3826 size_t os::read(int fd, void *buf, unsigned int nBytes) {
3827 return ::read(fd, buf, nBytes);
3828 }
3829
3830 size_t os::read_at(int fd, void *buf, unsigned int nBytes, jlong offset) {
3831 return ::pread(fd, buf, nBytes, offset);
3832 }
3833
3834 // Short sleep, direct OS call.
3835 //
3836 // Note: certain versions of Linux CFS scheduler (since 2.6.23) do not guarantee
3837 // sched_yield(2) will actually give up the CPU:
3838 //
3839 // * Alone on this pariticular CPU, keeps running.
|
3094 char* const start_aligned = (char*) align_ptr_up(start, alignment);
3095 char* const end_aligned = start_aligned + bytes;
3096 char* const end = start + extra_size;
3097 if (start_aligned > start) {
3098 ::munmap(start, start_aligned - start);
3099 }
3100 if (end_aligned < end) {
3101 ::munmap(end_aligned, end - end_aligned);
3102 }
3103 start = start_aligned;
3104 }
3105 }
3106 return start;
3107 }
3108
3109 static int anon_munmap(char * addr, size_t size) {
3110 return ::munmap(addr, size) == 0;
3111 }
3112
3113 char* os::pd_reserve_memory(size_t bytes, char* requested_addr,
3114 size_t alignment_hint, int file_desc) {
3115 if (file_desc != -1) {
3116 return map_memory_to_file(requested_addr, bytes, file_desc);
3117 }
3118 return anon_mmap(requested_addr, bytes, (requested_addr != NULL));
3119 }
3120
3121 bool os::pd_release_memory(char* addr, size_t size) {
3122 return anon_munmap(addr, size);
3123 }
3124
3125 static bool linux_mprotect(char* addr, size_t size, int prot) {
3126 // Linux wants the mprotect address argument to be page aligned.
3127 char* bottom = (char*)align_size_down((intptr_t)addr, os::Linux::page_size());
3128
3129 // According to SUSv3, mprotect() should only be used with mappings
3130 // established by mmap(), and mmap() always maps whole pages. Unaligned
3131 // 'addr' likely indicates problem in the VM (e.g. trying to change
3132 // protection of malloc'ed or statically allocated memory). Check the
3133 // caller if you hit this assert.
3134 assert(addr == bottom, "sanity check");
3135
3136 size = align_size_up(pointer_delta(addr, bottom, 1) + size, os::Linux::page_size());
3137 return ::mprotect(bottom, size, prot) == 0;
3545 bool exec) {
3546 assert(UseLargePages && UseHugeTLBFS, "only for Huge TLBFS large pages");
3547 assert(is_size_aligned(bytes, os::large_page_size()), "Unaligned size");
3548 assert(is_ptr_aligned(req_addr, os::large_page_size()), "Unaligned address");
3549
3550 int prot = exec ? PROT_READ|PROT_WRITE|PROT_EXEC : PROT_READ|PROT_WRITE;
3551 char* addr = (char*)::mmap(req_addr, bytes, prot,
3552 MAP_PRIVATE|MAP_ANONYMOUS|MAP_HUGETLB,
3553 -1, 0);
3554
3555 if (addr == MAP_FAILED) {
3556 warn_on_large_pages_failure(req_addr, bytes, errno);
3557 return NULL;
3558 }
3559
3560 assert(is_ptr_aligned(addr, os::large_page_size()), "Must be");
3561
3562 return addr;
3563 }
3564
3565 // Helper function to create a temp file in the given directory
3566 int os::create_file_for_heap(const char* dir, size_t size) {
3567
3568 char name_template[] = "/jvmheap.XXXXXX";
3569
3570 char *fullname = (char*)alloca(strlen(dir) + sizeof(name_template));
3571 (void)strcpy(fullname, dir);
3572 (void)strcat(fullname, name_template);
3573
3574 sigset_t set, oldset;
3575 sigfillset(&set);
3576
3577 // block all signals while we do the file operation
3578 (void)sigprocmask(SIG_BLOCK, &set, &oldset);
3579
3580 // set the file creation mask
3581 mode_t new_mask = S_IRUSR | S_IWUSR;
3582 mode_t prev_umask = umask(new_mask);
3583
3584 // create a new file
3585 int fd = mkstemp(fullname);
3586
3587 // reset the file creation mask
3588 umask(prev_umask);
3589
3590 if (fd < 0) {
3591 warning("Could not create file for heap");
3592 return -1;
3593 }
3594
3595 // delete the name from the filesystem. When 'fd' is closed, the file (and space) will be deleted
3596 (void)unlink(fullname);
3597
3598 // reset the signal mask
3599 (void)sigprocmask(SIG_SETMASK, &oldset, NULL);
3600
3601 return fd;
3602 }
3603
3604 // Map the given address range to the provided file descriptor
3605 char* os::map_memory_to_file(char* base, size_t size, int fd) {
3606 assert(fd != -1, "File descriptor is not valid");
3607
3608 // allocate space for the file
3609 if ((errno = posix_fallocate(fd, 0, (off_t)size)) != 0) {
3610 vm_exit_during_initialization(err_msg("Error in mapping Java heap at the given filesystem directory"));
3611 return NULL;
3612 }
3613
3614 int prot = PROT_READ | PROT_WRITE;
3615 int flags = MAP_SHARED;
3616 if( base != NULL) {
3617 flags |= MAP_FIXED;
3618 }
3619 char* addr = (char*)mmap(base, size, prot, flags, fd, 0);
3620
3621 if (addr == MAP_FAILED || (base != NULL && addr != base)) {
3622 if (addr != MAP_FAILED) {
3623 if (!pd_release_memory(addr, size)) {
3624 warning("Could not release memory on unsuccessful file mapping");
3625 }
3626 }
3627 return NULL;
3628 }
3629
3630 return addr;
3631 }
3632
3633 // Reserve memory using mmap(MAP_HUGETLB).
3634 // - bytes shall be a multiple of alignment.
3635 // - req_addr can be NULL. If not NULL, it must be a multiple of alignment.
3636 // - alignment sets the alignment at which memory shall be allocated.
3637 // It must be a multiple of allocation granularity.
3638 // Returns address of memory or NULL. If req_addr was not NULL, will only return
3639 // req_addr or NULL.
3640 char* os::Linux::reserve_memory_special_huge_tlbfs_mixed(size_t bytes,
3641 size_t alignment,
3642 char* req_addr,
3643 bool exec) {
3644 size_t large_page_size = os::large_page_size();
3645 assert(bytes >= large_page_size, "Shouldn't allocate large pages for small sizes");
3646
3647 assert(is_ptr_aligned(req_addr, alignment), "Must be");
3648 assert(is_size_aligned(bytes, alignment), "Must be");
3649
3650 // First reserve - but not commit - the address range in small pages.
3651 char* const start = anon_mmap_aligned(bytes, alignment, req_addr);
3652
3803 }
3804
3805 // With SysV SHM the entire memory region must be allocated as shared
3806 // memory.
3807 // HugeTLBFS allows application to commit large page memory on demand.
3808 // However, when committing memory with HugeTLBFS fails, the region
3809 // that was supposed to be committed will lose the old reservation
3810 // and allow other threads to steal that memory region. Because of this
3811 // behavior we can't commit HugeTLBFS memory.
3812 bool os::can_commit_large_page_memory() {
3813 return UseTransparentHugePages;
3814 }
3815
3816 bool os::can_execute_large_page_memory() {
3817 return UseTransparentHugePages || UseHugeTLBFS;
3818 }
3819
3820 // Reserve memory at an arbitrary address, only if that area is
3821 // available (and not reserved for something else).
3822
3823 char* os::pd_attempt_reserve_memory_at(size_t bytes, char* requested_addr, int file_desc) {
3824 const int max_tries = 10;
3825 char* base[max_tries];
3826 size_t size[max_tries];
3827 const size_t gap = 0x000000;
3828
3829 // Assert only that the size is a multiple of the page size, since
3830 // that's all that mmap requires, and since that's all we really know
3831 // about at this low abstraction level. If we need higher alignment,
3832 // we can either pass an alignment to this method or verify alignment
3833 // in one of the methods further up the call chain. See bug 5044738.
3834 assert(bytes % os::vm_page_size() == 0, "reserving unexpected size block");
3835
3836 // Repeatedly allocate blocks until the block is allocated at the
3837 // right spot.
3838
3839 // Linux mmap allows caller to pass an address as hint; give it a try first,
3840 // if kernel honors the hint then we can return immediately.
3841 char * addr = anon_mmap(requested_addr, bytes, false);
3842 if (addr == requested_addr) {
3843 if (file_desc == -1 || map_memory_to_file(requested_addr, bytes, file_desc) != NULL) {
3844 return requested_addr;
3845 }
3846 }
3847
3848 if (addr != NULL) {
3849 // mmap() is successful but it fails to reserve at the requested address
3850 anon_munmap(addr, bytes);
3851 }
3852
3853 int i;
3854 for (i = 0; i < max_tries; ++i) {
3855 base[i] = reserve_memory(bytes);
3856
3857 if (base[i] != NULL) {
3858 // Is this the block we wanted?
3859 if (base[i] == requested_addr) {
3860 size[i] = bytes;
3861 break;
3862 }
3863
3864 // Does this overlap the block we wanted? Give back the overlapped
3865 // parts and try again.
3866
3873 ptrdiff_t bottom_overlap = base[i] + bytes - requested_addr;
3874 if (bottom_overlap >= 0 && (size_t)bottom_overlap < bytes) {
3875 unmap_memory(requested_addr, bottom_overlap);
3876 size[i] = bytes - bottom_overlap;
3877 } else {
3878 size[i] = bytes;
3879 }
3880 }
3881 }
3882 }
3883
3884 // Give back the unused reserved pieces.
3885
3886 for (int j = 0; j < i; ++j) {
3887 if (base[j] != NULL) {
3888 unmap_memory(base[j], size[j]);
3889 }
3890 }
3891
3892 if (i < max_tries) {
3893 if (file_desc != -1 && map_memory_to_file(requested_addr, bytes, file_desc) == NULL) {
3894 vm_exit_during_initialization(err_msg("Error in mapping Java heap at the given filesystem directory"));
3895 }
3896 return requested_addr;
3897 } else {
3898 return NULL;
3899 }
3900 }
3901
3902 size_t os::read(int fd, void *buf, unsigned int nBytes) {
3903 return ::read(fd, buf, nBytes);
3904 }
3905
3906 size_t os::read_at(int fd, void *buf, unsigned int nBytes, jlong offset) {
3907 return ::pread(fd, buf, nBytes, offset);
3908 }
3909
3910 // Short sleep, direct OS call.
3911 //
3912 // Note: certain versions of Linux CFS scheduler (since 2.6.23) do not guarantee
3913 // sched_yield(2) will actually give up the CPU:
3914 //
3915 // * Alone on this pariticular CPU, keeps running.
|