< prev index next >

src/os/linux/vm/os_linux.cpp

Print this page




 561 // Now alternate signal stack is gone, it's harder to use 2. For instance,
 562 // if current sp is already near the lower end of page 101, and we need to
 563 // call mmap() to map page 100, it is possible that part of the mmap() frame
 564 // will be placed in page 100. When page 100 is mapped, it is zero-filled.
 565 // That will destroy the mmap() frame and cause VM to crash.
 566 //
 567 // The following code works by adjusting sp first, then accessing the "bottom"
 568 // page to force a page fault. Linux kernel will then automatically expand the
 569 // stack mapping.
 570 //
 571 // _expand_stack_to() assumes its frame size is less than page size, which
 572 // should always be true if the function is not inlined.
 573 
 574 static void NOINLINE _expand_stack_to(address bottom) {
 575   address sp;
 576   size_t size;
 577   volatile char *p;
 578 
 579   // Adjust bottom to point to the largest address within the same page, it
 580   // gives us a one-page buffer if alloca() allocates slightly more memory.
 581   bottom = (address)align_size_down((uintptr_t)bottom, os::Linux::page_size());
 582   bottom += os::Linux::page_size() - 1;
 583 
 584   // sp might be slightly above current stack pointer; if that's the case, we
 585   // will alloca() a little more space than necessary, which is OK. Don't use
 586   // os::current_stack_pointer(), as its result can be slightly below current
 587   // stack pointer, causing us to not alloca enough to reach "bottom".
 588   sp = (address)&sp;
 589 
 590   if (sp > bottom) {
 591     size = sp - bottom;
 592     p = (volatile char *)alloca(size);
 593     assert(p != NULL && p <= (volatile char *)bottom, "alloca problem?");
 594     p[0] = '\0';
 595   }
 596 }
 597 
 598 bool os::Linux::manually_expand_stack(JavaThread * t, address addr) {
 599   assert(t!=NULL, "just checking");
 600   assert(t->osthread()->expanding_stack(), "expand should be set");
 601   assert(t->stack_base() != NULL, "stack_base was not initialized");


 698   thread->set_osthread(osthread);
 699 
 700   // init thread attributes
 701   pthread_attr_t attr;
 702   pthread_attr_init(&attr);
 703   pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED);
 704 
 705   // Calculate stack size if it's not specified by caller.
 706   size_t stack_size = os::Posix::get_initial_stack_size(thr_type, req_stack_size);
 707   // In the Linux NPTL pthread implementation the guard size mechanism
 708   // is not implemented properly. The posix standard requires adding
 709   // the size of the guard pages to the stack size, instead Linux
 710   // takes the space out of 'stacksize'. Thus we adapt the requested
 711   // stack_size by the size of the guard pages to mimick proper
 712   // behaviour. However, be careful not to end up with a size
 713   // of zero due to overflow. Don't add the guard page in that case.
 714   size_t guard_size = os::Linux::default_guard_size(thr_type);
 715   if (stack_size <= SIZE_MAX - guard_size) {
 716     stack_size += guard_size;
 717   }
 718   assert(is_size_aligned(stack_size, os::vm_page_size()), "stack_size not aligned");
 719 
 720   int status = pthread_attr_setstacksize(&attr, stack_size);
 721   assert_status(status == 0, status, "pthread_attr_setstacksize");
 722 
 723   // Configure glibc guard page.
 724   pthread_attr_setguardsize(&attr, os::Linux::default_guard_size(thr_type));
 725 
 726   ThreadState state;
 727 
 728   {
 729     pthread_t tid;
 730     int ret = pthread_create(&tid, &attr, (void* (*)(void*)) thread_native_entry, thread);
 731 
 732     char buf[64];
 733     if (ret == 0) {
 734       log_info(os, thread)("Thread started (pthread id: " UINTX_FORMAT ", attributes: %s). ",
 735         (uintx) tid, os::Posix::describe_pthread_attr(buf, sizeof(buf), &attr));
 736     } else {
 737       log_warning(os, thread)("Failed to start thread - pthread_create failed (%s) for attributes: %s.",
 738         os::errno_name(ret), os::Posix::describe_pthread_attr(buf, sizeof(buf), &attr));


1084   // and its upper limit is the real stack top. (again, this would fail if
1085   // running inside chroot, because /proc may not exist.)
1086 
1087   uintptr_t stack_top;
1088   address low, high;
1089   if (find_vma((address)stack_start, &low, &high)) {
1090     // success, "high" is the true stack top. (ignore "low", because initial
1091     // thread stack grows on demand, its real bottom is high - RLIMIT_STACK.)
1092     stack_top = (uintptr_t)high;
1093   } else {
1094     // failed, likely because /proc/self/maps does not exist
1095     warning("Can't detect initial thread stack location - find_vma failed");
1096     // best effort: stack_start is normally within a few pages below the real
1097     // stack top, use it as stack top, and reduce stack size so we won't put
1098     // guard page outside stack.
1099     stack_top = stack_start;
1100     stack_size -= 16 * page_size();
1101   }
1102 
1103   // stack_top could be partially down the page so align it
1104   stack_top = align_size_up(stack_top, page_size());
1105 
1106   // Allowed stack value is minimum of max_size and what we derived from rlimit
1107   if (max_size > 0) {
1108     _initial_thread_stack_size = MIN2(max_size, stack_size);
1109   } else {
1110     // Accept the rlimit max, but if stack is unlimited then it will be huge, so
1111     // clamp it at 8MB as we do on Solaris
1112     _initial_thread_stack_size = MIN2(stack_size, 8*M);
1113   }
1114   _initial_thread_stack_size = align_size_down(_initial_thread_stack_size, page_size());
1115   _initial_thread_stack_bottom = (address)stack_top - _initial_thread_stack_size;
1116 
1117   assert(_initial_thread_stack_bottom < (address)stack_top, "overflow!");
1118 
1119   if (log_is_enabled(Info, os, thread)) {
1120     // See if we seem to be on primordial process thread
1121     bool primordial = uintptr_t(&rlim) > uintptr_t(_initial_thread_stack_bottom) &&
1122                       uintptr_t(&rlim) < stack_top;
1123 
1124     log_info(os, thread)("Capturing initial stack in %s thread: req. size: " SIZE_FORMAT "K, actual size: "
1125                          SIZE_FORMAT "K, top=" INTPTR_FORMAT ", bottom=" INTPTR_FORMAT,
1126                          primordial ? "primordial" : "user", max_size / K,  _initial_thread_stack_size / K,
1127                          stack_top, intptr_t(_initial_thread_stack_bottom));
1128   }
1129 }
1130 
1131 ////////////////////////////////////////////////////////////////////////////////
1132 // time support
1133 
1134 // Time since start-up in seconds to a fine granularity.


3153 //  req_addr or NULL.
3154 static char* anon_mmap_aligned(size_t bytes, size_t alignment, char* req_addr) {
3155 
3156   size_t extra_size = bytes;
3157   if (req_addr == NULL && alignment > 0) {
3158     extra_size += alignment;
3159   }
3160 
3161   char* start = (char*) ::mmap(req_addr, extra_size, PROT_NONE,
3162     MAP_PRIVATE|MAP_ANONYMOUS|MAP_NORESERVE,
3163     -1, 0);
3164   if (start == MAP_FAILED) {
3165     start = NULL;
3166   } else {
3167     if (req_addr != NULL) {
3168       if (start != req_addr) {
3169         ::munmap(start, extra_size);
3170         start = NULL;
3171       }
3172     } else {
3173       char* const start_aligned = align_ptr_up(start, alignment);
3174       char* const end_aligned = start_aligned + bytes;
3175       char* const end = start + extra_size;
3176       if (start_aligned > start) {
3177         ::munmap(start, start_aligned - start);
3178       }
3179       if (end_aligned < end) {
3180         ::munmap(end_aligned, end - end_aligned);
3181       }
3182       start = start_aligned;
3183     }
3184   }
3185   return start;
3186 }
3187 
3188 static int anon_munmap(char * addr, size_t size) {
3189   return ::munmap(addr, size) == 0;
3190 }
3191 
3192 char* os::pd_reserve_memory(size_t bytes, char* requested_addr,
3193                             size_t alignment_hint) {
3194   return anon_mmap(requested_addr, bytes, (requested_addr != NULL));
3195 }
3196 
3197 bool os::pd_release_memory(char* addr, size_t size) {
3198   return anon_munmap(addr, size);
3199 }
3200 
3201 static bool linux_mprotect(char* addr, size_t size, int prot) {
3202   // Linux wants the mprotect address argument to be page aligned.
3203   char* bottom = (char*)align_size_down((intptr_t)addr, os::Linux::page_size());
3204 
3205   // According to SUSv3, mprotect() should only be used with mappings
3206   // established by mmap(), and mmap() always maps whole pages. Unaligned
3207   // 'addr' likely indicates problem in the VM (e.g. trying to change
3208   // protection of malloc'ed or statically allocated memory). Check the
3209   // caller if you hit this assert.
3210   assert(addr == bottom, "sanity check");
3211 
3212   size = align_size_up(pointer_delta(addr, bottom, 1) + size, os::Linux::page_size());
3213   return ::mprotect(bottom, size, prot) == 0;
3214 }
3215 
3216 // Set protections specified
3217 bool os::protect_memory(char* addr, size_t bytes, ProtType prot,
3218                         bool is_committed) {
3219   unsigned int p = 0;
3220   switch (prot) {
3221   case MEM_PROT_NONE: p = PROT_NONE; break;
3222   case MEM_PROT_READ: p = PROT_READ; break;
3223   case MEM_PROT_RW:   p = PROT_READ|PROT_WRITE; break;
3224   case MEM_PROT_RWX:  p = PROT_READ|PROT_WRITE|PROT_EXEC; break;
3225   default:
3226     ShouldNotReachHere();
3227   }
3228   // is_committed is unused.
3229   return linux_mprotect(addr, bytes, p);
3230 }
3231 
3232 bool os::guard_memory(char* addr, size_t size) {
3233   return linux_mprotect(addr, size, PROT_NONE);
3234 }
3235 
3236 bool os::unguard_memory(char* addr, size_t size) {
3237   return linux_mprotect(addr, size, PROT_READ|PROT_WRITE);
3238 }
3239 
3240 bool os::Linux::transparent_huge_pages_sanity_check(bool warn,
3241                                                     size_t page_size) {
3242   bool result = false;
3243   void *p = mmap(NULL, page_size * 2, PROT_READ|PROT_WRITE,
3244                  MAP_ANONYMOUS|MAP_PRIVATE,
3245                  -1, 0);
3246   if (p != MAP_FAILED) {
3247     void *aligned_p = align_ptr_up(p, page_size);
3248 
3249     result = madvise(aligned_p, page_size, MADV_HUGEPAGE) == 0;
3250 
3251     munmap(p, page_size * 2);
3252   }
3253 
3254   if (warn && !result) {
3255     warning("TransparentHugePages is not supported by the operating system.");
3256   }
3257 
3258   return result;
3259 }
3260 
3261 bool os::Linux::hugetlbfs_sanity_check(bool warn, size_t page_size) {
3262   bool result = false;
3263   void *p = mmap(NULL, page_size, PROT_READ|PROT_WRITE,
3264                  MAP_ANONYMOUS|MAP_PRIVATE|MAP_HUGETLB,
3265                  -1, 0);
3266 
3267   if (p != MAP_FAILED) {


3470 
3471 #define shm_warning_format(format, ...)              \
3472   do {                                               \
3473     if (UseLargePages &&                             \
3474         (!FLAG_IS_DEFAULT(UseLargePages) ||          \
3475          !FLAG_IS_DEFAULT(UseSHM) ||                 \
3476          !FLAG_IS_DEFAULT(LargePageSizeInBytes))) {  \
3477       warning(format, __VA_ARGS__);                  \
3478     }                                                \
3479   } while (0)
3480 
3481 #define shm_warning(str) shm_warning_format("%s", str)
3482 
3483 #define shm_warning_with_errno(str)                \
3484   do {                                             \
3485     int err = errno;                               \
3486     shm_warning_format(str " (error = %d)", err);  \
3487   } while (0)
3488 
3489 static char* shmat_with_alignment(int shmid, size_t bytes, size_t alignment) {
3490   assert(is_size_aligned(bytes, alignment), "Must be divisible by the alignment");
3491 
3492   if (!is_size_aligned(alignment, SHMLBA)) {
3493     assert(false, "Code below assumes that alignment is at least SHMLBA aligned");
3494     return NULL;
3495   }
3496 
3497   // To ensure that we get 'alignment' aligned memory from shmat,
3498   // we pre-reserve aligned virtual memory and then attach to that.
3499 
3500   char* pre_reserved_addr = anon_mmap_aligned(bytes, alignment, NULL);
3501   if (pre_reserved_addr == NULL) {
3502     // Couldn't pre-reserve aligned memory.
3503     shm_warning("Failed to pre-reserve aligned memory for shmat.");
3504     return NULL;
3505   }
3506 
3507   // SHM_REMAP is needed to allow shmat to map over an existing mapping.
3508   char* addr = (char*)shmat(shmid, pre_reserved_addr, SHM_REMAP);
3509 
3510   if ((intptr_t)addr == -1) {
3511     int err = errno;
3512     shm_warning_with_errno("Failed to attach shared memory.");
3513 
3514     assert(err != EACCES, "Unexpected error");
3515     assert(err != EIDRM,  "Unexpected error");
3516     assert(err != EINVAL, "Unexpected error");
3517 
3518     // Since we don't know if the kernel unmapped the pre-reserved memory area
3519     // we can't unmap it, since that would potentially unmap memory that was
3520     // mapped from other threads.
3521     return NULL;
3522   }
3523 
3524   return addr;
3525 }
3526 
3527 static char* shmat_at_address(int shmid, char* req_addr) {
3528   if (!is_ptr_aligned(req_addr, SHMLBA)) {
3529     assert(false, "Requested address needs to be SHMLBA aligned");
3530     return NULL;
3531   }
3532 
3533   char* addr = (char*)shmat(shmid, req_addr, 0);
3534 
3535   if ((intptr_t)addr == -1) {
3536     shm_warning_with_errno("Failed to attach shared memory.");
3537     return NULL;
3538   }
3539 
3540   return addr;
3541 }
3542 
3543 static char* shmat_large_pages(int shmid, size_t bytes, size_t alignment, char* req_addr) {
3544   // If a req_addr has been provided, we assume that the caller has already aligned the address.
3545   if (req_addr != NULL) {
3546     assert(is_ptr_aligned(req_addr, os::large_page_size()), "Must be divisible by the large page size");
3547     assert(is_ptr_aligned(req_addr, alignment), "Must be divisible by given alignment");
3548     return shmat_at_address(shmid, req_addr);
3549   }
3550 
3551   // Since shmid has been setup with SHM_HUGETLB, shmat will automatically
3552   // return large page size aligned memory addresses when req_addr == NULL.
3553   // However, if the alignment is larger than the large page size, we have
3554   // to manually ensure that the memory returned is 'alignment' aligned.
3555   if (alignment > os::large_page_size()) {
3556     assert(is_size_aligned(alignment, os::large_page_size()), "Must be divisible by the large page size");
3557     return shmat_with_alignment(shmid, bytes, alignment);
3558   } else {
3559     return shmat_at_address(shmid, NULL);
3560   }
3561 }
3562 
3563 char* os::Linux::reserve_memory_special_shm(size_t bytes, size_t alignment,
3564                                             char* req_addr, bool exec) {
3565   // "exec" is passed in but not used.  Creating the shared image for
3566   // the code cache doesn't have an SHM_X executable permission to check.
3567   assert(UseLargePages && UseSHM, "only for SHM large pages");
3568   assert(is_ptr_aligned(req_addr, os::large_page_size()), "Unaligned address");
3569   assert(is_ptr_aligned(req_addr, alignment), "Unaligned address");
3570 
3571   if (!is_size_aligned(bytes, os::large_page_size())) {
3572     return NULL; // Fallback to small pages.
3573   }
3574 
3575   // Create a large shared memory region to attach to based on size.
3576   // Currently, size is the total size of the heap.
3577   int shmid = shmget(IPC_PRIVATE, bytes, SHM_HUGETLB|IPC_CREAT|SHM_R|SHM_W);
3578   if (shmid == -1) {
3579     // Possible reasons for shmget failure:
3580     // 1. shmmax is too small for Java heap.
3581     //    > check shmmax value: cat /proc/sys/kernel/shmmax
3582     //    > increase shmmax value: echo "0xffffffff" > /proc/sys/kernel/shmmax
3583     // 2. not enough large page memory.
3584     //    > check available large pages: cat /proc/meminfo
3585     //    > increase amount of large pages:
3586     //          echo new_value > /proc/sys/vm/nr_hugepages
3587     //      Note 1: different Linux may use different name for this property,
3588     //            e.g. on Redhat AS-3 it is "hugetlb_pool".
3589     //      Note 2: it's possible there's enough physical memory available but
3590     //            they are so fragmented after a long run that they can't
3591     //            coalesce into large pages. Try to reserve large pages when


3610                                         int error) {
3611   assert(error == ENOMEM, "Only expect to fail if no memory is available");
3612 
3613   bool warn_on_failure = UseLargePages &&
3614       (!FLAG_IS_DEFAULT(UseLargePages) ||
3615        !FLAG_IS_DEFAULT(UseHugeTLBFS) ||
3616        !FLAG_IS_DEFAULT(LargePageSizeInBytes));
3617 
3618   if (warn_on_failure) {
3619     char msg[128];
3620     jio_snprintf(msg, sizeof(msg), "Failed to reserve large pages memory req_addr: "
3621                  PTR_FORMAT " bytes: " SIZE_FORMAT " (errno = %d).", req_addr, bytes, error);
3622     warning("%s", msg);
3623   }
3624 }
3625 
3626 char* os::Linux::reserve_memory_special_huge_tlbfs_only(size_t bytes,
3627                                                         char* req_addr,
3628                                                         bool exec) {
3629   assert(UseLargePages && UseHugeTLBFS, "only for Huge TLBFS large pages");
3630   assert(is_size_aligned(bytes, os::large_page_size()), "Unaligned size");
3631   assert(is_ptr_aligned(req_addr, os::large_page_size()), "Unaligned address");
3632 
3633   int prot = exec ? PROT_READ|PROT_WRITE|PROT_EXEC : PROT_READ|PROT_WRITE;
3634   char* addr = (char*)::mmap(req_addr, bytes, prot,
3635                              MAP_PRIVATE|MAP_ANONYMOUS|MAP_HUGETLB,
3636                              -1, 0);
3637 
3638   if (addr == MAP_FAILED) {
3639     warn_on_large_pages_failure(req_addr, bytes, errno);
3640     return NULL;
3641   }
3642 
3643   assert(is_ptr_aligned(addr, os::large_page_size()), "Must be");
3644 
3645   return addr;
3646 }
3647 
3648 // Reserve memory using mmap(MAP_HUGETLB).
3649 //  - bytes shall be a multiple of alignment.
3650 //  - req_addr can be NULL. If not NULL, it must be a multiple of alignment.
3651 //  - alignment sets the alignment at which memory shall be allocated.
3652 //     It must be a multiple of allocation granularity.
3653 // Returns address of memory or NULL. If req_addr was not NULL, will only return
3654 //  req_addr or NULL.
3655 char* os::Linux::reserve_memory_special_huge_tlbfs_mixed(size_t bytes,
3656                                                          size_t alignment,
3657                                                          char* req_addr,
3658                                                          bool exec) {
3659   size_t large_page_size = os::large_page_size();
3660   assert(bytes >= large_page_size, "Shouldn't allocate large pages for small sizes");
3661 
3662   assert(is_ptr_aligned(req_addr, alignment), "Must be");
3663   assert(is_size_aligned(bytes, alignment), "Must be");
3664 
3665   // First reserve - but not commit - the address range in small pages.
3666   char* const start = anon_mmap_aligned(bytes, alignment, req_addr);
3667 
3668   if (start == NULL) {
3669     return NULL;
3670   }
3671 
3672   assert(is_ptr_aligned(start, alignment), "Must be");
3673 
3674   char* end = start + bytes;
3675 
3676   // Find the regions of the allocated chunk that can be promoted to large pages.
3677   char* lp_start = align_ptr_up(start, large_page_size);
3678   char* lp_end   = align_ptr_down(end, large_page_size);
3679 
3680   size_t lp_bytes = lp_end - lp_start;
3681 
3682   assert(is_size_aligned(lp_bytes, large_page_size), "Must be");
3683 
3684   if (lp_bytes == 0) {
3685     // The mapped region doesn't even span the start and the end of a large page.
3686     // Fall back to allocate a non-special area.
3687     ::munmap(start, end - start);
3688     return NULL;
3689   }
3690 
3691   int prot = exec ? PROT_READ|PROT_WRITE|PROT_EXEC : PROT_READ|PROT_WRITE;
3692 
3693   void* result;
3694 
3695   // Commit small-paged leading area.
3696   if (start != lp_start) {
3697     result = ::mmap(start, lp_start - start, prot,
3698                     MAP_PRIVATE|MAP_ANONYMOUS|MAP_FIXED,
3699                     -1, 0);
3700     if (result == MAP_FAILED) {
3701       ::munmap(lp_start, end - lp_start);
3702       return NULL;


3723 
3724   // Commit small-paged trailing area.
3725   if (lp_end != end) {
3726     result = ::mmap(lp_end, end - lp_end, prot,
3727                     MAP_PRIVATE|MAP_ANONYMOUS|MAP_FIXED,
3728                     -1, 0);
3729     if (result == MAP_FAILED) {
3730       ::munmap(start, lp_end - start);
3731       return NULL;
3732     }
3733   }
3734 
3735   return start;
3736 }
3737 
3738 char* os::Linux::reserve_memory_special_huge_tlbfs(size_t bytes,
3739                                                    size_t alignment,
3740                                                    char* req_addr,
3741                                                    bool exec) {
3742   assert(UseLargePages && UseHugeTLBFS, "only for Huge TLBFS large pages");
3743   assert(is_ptr_aligned(req_addr, alignment), "Must be");
3744   assert(is_size_aligned(alignment, os::vm_allocation_granularity()), "Must be");
3745   assert(is_power_of_2(os::large_page_size()), "Must be");
3746   assert(bytes >= os::large_page_size(), "Shouldn't allocate large pages for small sizes");
3747 
3748   if (is_size_aligned(bytes, os::large_page_size()) && alignment <= os::large_page_size()) {
3749     return reserve_memory_special_huge_tlbfs_only(bytes, req_addr, exec);
3750   } else {
3751     return reserve_memory_special_huge_tlbfs_mixed(bytes, alignment, req_addr, exec);
3752   }
3753 }
3754 
3755 char* os::reserve_memory_special(size_t bytes, size_t alignment,
3756                                  char* req_addr, bool exec) {
3757   assert(UseLargePages, "only for large pages");
3758 
3759   char* addr;
3760   if (UseSHM) {
3761     addr = os::Linux::reserve_memory_special_shm(bytes, alignment, req_addr, exec);
3762   } else {
3763     assert(UseHugeTLBFS, "must be");
3764     addr = os::Linux::reserve_memory_special_huge_tlbfs(bytes, alignment, req_addr, exec);
3765   }
3766 
3767   if (addr != NULL) {
3768     if (UseNUMAInterleaving) {


5950       PROT_NONE, MAP_PRIVATE|MAP_ANONYMOUS|MAP_NORESERVE,
5951       -1, 0);
5952     assert(mapping1 != MAP_FAILED, "should work");
5953 
5954     char* const mapping2 = (char*) ::mmap(NULL, mapping_size,
5955       PROT_NONE, MAP_PRIVATE|MAP_ANONYMOUS|MAP_NORESERVE,
5956       -1, 0);
5957     assert(mapping2 != MAP_FAILED, "should work");
5958 
5959     // Unmap the first mapping, but leave the second mapping intact: the first
5960     // mapping will serve as a value for a "good" req_addr (case 2). The second
5961     // mapping, still intact, as "bad" req_addr (case 3).
5962     ::munmap(mapping1, mapping_size);
5963 
5964     // Case 1
5965     test_log("%s, req_addr NULL:", __FUNCTION__);
5966     test_log("size            align           result");
5967 
5968     for (int i = 0; i < num_sizes; i++) {
5969       const size_t size = sizes[i];
5970       for (size_t alignment = ag; is_size_aligned(size, alignment); alignment *= 2) {
5971         char* p = os::Linux::reserve_memory_special_huge_tlbfs_mixed(size, alignment, NULL, false);
5972         test_log(SIZE_FORMAT_HEX " " SIZE_FORMAT_HEX " ->  " PTR_FORMAT " %s",
5973                  size, alignment, p2i(p), (p != NULL ? "" : "(failed)"));
5974         if (p != NULL) {
5975           assert(is_ptr_aligned(p, alignment), "must be");
5976           small_page_write(p, size);
5977           os::Linux::release_memory_special_huge_tlbfs(p, size);
5978         }
5979       }
5980     }
5981 
5982     // Case 2
5983     test_log("%s, req_addr non-NULL:", __FUNCTION__);
5984     test_log("size            align           req_addr         result");
5985 
5986     for (int i = 0; i < num_sizes; i++) {
5987       const size_t size = sizes[i];
5988       for (size_t alignment = ag; is_size_aligned(size, alignment); alignment *= 2) {
5989         char* const req_addr = align_ptr_up(mapping1, alignment);
5990         char* p = os::Linux::reserve_memory_special_huge_tlbfs_mixed(size, alignment, req_addr, false);
5991         test_log(SIZE_FORMAT_HEX " " SIZE_FORMAT_HEX " " PTR_FORMAT " ->  " PTR_FORMAT " %s",
5992                  size, alignment, p2i(req_addr), p2i(p),
5993                  ((p != NULL ? (p == req_addr ? "(exact match)" : "") : "(failed)")));
5994         if (p != NULL) {
5995           assert(p == req_addr, "must be");
5996           small_page_write(p, size);
5997           os::Linux::release_memory_special_huge_tlbfs(p, size);
5998         }
5999       }
6000     }
6001 
6002     // Case 3
6003     test_log("%s, req_addr non-NULL with preexisting mapping:", __FUNCTION__);
6004     test_log("size            align           req_addr         result");
6005 
6006     for (int i = 0; i < num_sizes; i++) {
6007       const size_t size = sizes[i];
6008       for (size_t alignment = ag; is_size_aligned(size, alignment); alignment *= 2) {
6009         char* const req_addr = align_ptr_up(mapping2, alignment);
6010         char* p = os::Linux::reserve_memory_special_huge_tlbfs_mixed(size, alignment, req_addr, false);
6011         test_log(SIZE_FORMAT_HEX " " SIZE_FORMAT_HEX " " PTR_FORMAT " ->  " PTR_FORMAT " %s",
6012                  size, alignment, p2i(req_addr), p2i(p), ((p != NULL ? "" : "(failed)")));
6013         // as the area around req_addr contains already existing mappings, the API should always
6014         // return NULL (as per contract, it cannot return another address)
6015         assert(p == NULL, "must be");
6016       }
6017     }
6018 
6019     ::munmap(mapping2, mapping_size);
6020 
6021   }
6022 
6023   static void test_reserve_memory_special_huge_tlbfs() {
6024     if (!UseHugeTLBFS) {
6025       return;
6026     }
6027 
6028     test_reserve_memory_special_huge_tlbfs_only();
6029     test_reserve_memory_special_huge_tlbfs_mixed();
6030   }
6031 
6032   static void test_reserve_memory_special_shm(size_t size, size_t alignment) {
6033     if (!UseSHM) {
6034       return;
6035     }
6036 
6037     test_log("test_reserve_memory_special_shm(" SIZE_FORMAT ", " SIZE_FORMAT ")", size, alignment);
6038 
6039     char* addr = os::Linux::reserve_memory_special_shm(size, alignment, NULL, false);
6040 
6041     if (addr != NULL) {
6042       assert(is_ptr_aligned(addr, alignment), "Check");
6043       assert(is_ptr_aligned(addr, os::large_page_size()), "Check");
6044 
6045       small_page_write(addr, size);
6046 
6047       os::Linux::release_memory_special_shm(addr, size);
6048     }
6049   }
6050 
6051   static void test_reserve_memory_special_shm() {
6052     size_t lp = os::large_page_size();
6053     size_t ag = os::vm_allocation_granularity();
6054 
6055     for (size_t size = ag; size < lp * 3; size += ag) {
6056       for (size_t alignment = ag; is_size_aligned(size, alignment); alignment *= 2) {
6057         test_reserve_memory_special_shm(size, alignment);
6058       }
6059     }
6060   }
6061 
6062   static void test() {
6063     test_reserve_memory_special_huge_tlbfs();
6064     test_reserve_memory_special_shm();
6065   }
6066 };
6067 
6068 void TestReserveMemorySpecial_test() {
6069   TestReserveMemorySpecial::test();
6070 }
6071 
6072 #endif


 561 // Now alternate signal stack is gone, it's harder to use 2. For instance,
 562 // if current sp is already near the lower end of page 101, and we need to
 563 // call mmap() to map page 100, it is possible that part of the mmap() frame
 564 // will be placed in page 100. When page 100 is mapped, it is zero-filled.
 565 // That will destroy the mmap() frame and cause VM to crash.
 566 //
 567 // The following code works by adjusting sp first, then accessing the "bottom"
 568 // page to force a page fault. Linux kernel will then automatically expand the
 569 // stack mapping.
 570 //
 571 // _expand_stack_to() assumes its frame size is less than page size, which
 572 // should always be true if the function is not inlined.
 573 
 574 static void NOINLINE _expand_stack_to(address bottom) {
 575   address sp;
 576   size_t size;
 577   volatile char *p;
 578 
 579   // Adjust bottom to point to the largest address within the same page, it
 580   // gives us a one-page buffer if alloca() allocates slightly more memory.
 581   bottom = (address)align_down((uintptr_t)bottom, os::Linux::page_size());
 582   bottom += os::Linux::page_size() - 1;
 583 
 584   // sp might be slightly above current stack pointer; if that's the case, we
 585   // will alloca() a little more space than necessary, which is OK. Don't use
 586   // os::current_stack_pointer(), as its result can be slightly below current
 587   // stack pointer, causing us to not alloca enough to reach "bottom".
 588   sp = (address)&sp;
 589 
 590   if (sp > bottom) {
 591     size = sp - bottom;
 592     p = (volatile char *)alloca(size);
 593     assert(p != NULL && p <= (volatile char *)bottom, "alloca problem?");
 594     p[0] = '\0';
 595   }
 596 }
 597 
 598 bool os::Linux::manually_expand_stack(JavaThread * t, address addr) {
 599   assert(t!=NULL, "just checking");
 600   assert(t->osthread()->expanding_stack(), "expand should be set");
 601   assert(t->stack_base() != NULL, "stack_base was not initialized");


 698   thread->set_osthread(osthread);
 699 
 700   // init thread attributes
 701   pthread_attr_t attr;
 702   pthread_attr_init(&attr);
 703   pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED);
 704 
 705   // Calculate stack size if it's not specified by caller.
 706   size_t stack_size = os::Posix::get_initial_stack_size(thr_type, req_stack_size);
 707   // In the Linux NPTL pthread implementation the guard size mechanism
 708   // is not implemented properly. The posix standard requires adding
 709   // the size of the guard pages to the stack size, instead Linux
 710   // takes the space out of 'stacksize'. Thus we adapt the requested
 711   // stack_size by the size of the guard pages to mimick proper
 712   // behaviour. However, be careful not to end up with a size
 713   // of zero due to overflow. Don't add the guard page in that case.
 714   size_t guard_size = os::Linux::default_guard_size(thr_type);
 715   if (stack_size <= SIZE_MAX - guard_size) {
 716     stack_size += guard_size;
 717   }
 718   assert(is_aligned(stack_size, os::vm_page_size()), "stack_size not aligned");
 719 
 720   int status = pthread_attr_setstacksize(&attr, stack_size);
 721   assert_status(status == 0, status, "pthread_attr_setstacksize");
 722 
 723   // Configure glibc guard page.
 724   pthread_attr_setguardsize(&attr, os::Linux::default_guard_size(thr_type));
 725 
 726   ThreadState state;
 727 
 728   {
 729     pthread_t tid;
 730     int ret = pthread_create(&tid, &attr, (void* (*)(void*)) thread_native_entry, thread);
 731 
 732     char buf[64];
 733     if (ret == 0) {
 734       log_info(os, thread)("Thread started (pthread id: " UINTX_FORMAT ", attributes: %s). ",
 735         (uintx) tid, os::Posix::describe_pthread_attr(buf, sizeof(buf), &attr));
 736     } else {
 737       log_warning(os, thread)("Failed to start thread - pthread_create failed (%s) for attributes: %s.",
 738         os::errno_name(ret), os::Posix::describe_pthread_attr(buf, sizeof(buf), &attr));


1084   // and its upper limit is the real stack top. (again, this would fail if
1085   // running inside chroot, because /proc may not exist.)
1086 
1087   uintptr_t stack_top;
1088   address low, high;
1089   if (find_vma((address)stack_start, &low, &high)) {
1090     // success, "high" is the true stack top. (ignore "low", because initial
1091     // thread stack grows on demand, its real bottom is high - RLIMIT_STACK.)
1092     stack_top = (uintptr_t)high;
1093   } else {
1094     // failed, likely because /proc/self/maps does not exist
1095     warning("Can't detect initial thread stack location - find_vma failed");
1096     // best effort: stack_start is normally within a few pages below the real
1097     // stack top, use it as stack top, and reduce stack size so we won't put
1098     // guard page outside stack.
1099     stack_top = stack_start;
1100     stack_size -= 16 * page_size();
1101   }
1102 
1103   // stack_top could be partially down the page so align it
1104   stack_top = align_up(stack_top, page_size());
1105 
1106   // Allowed stack value is minimum of max_size and what we derived from rlimit
1107   if (max_size > 0) {
1108     _initial_thread_stack_size = MIN2(max_size, stack_size);
1109   } else {
1110     // Accept the rlimit max, but if stack is unlimited then it will be huge, so
1111     // clamp it at 8MB as we do on Solaris
1112     _initial_thread_stack_size = MIN2(stack_size, 8*M);
1113   }
1114   _initial_thread_stack_size = align_down(_initial_thread_stack_size, page_size());
1115   _initial_thread_stack_bottom = (address)stack_top - _initial_thread_stack_size;
1116 
1117   assert(_initial_thread_stack_bottom < (address)stack_top, "overflow!");
1118 
1119   if (log_is_enabled(Info, os, thread)) {
1120     // See if we seem to be on primordial process thread
1121     bool primordial = uintptr_t(&rlim) > uintptr_t(_initial_thread_stack_bottom) &&
1122                       uintptr_t(&rlim) < stack_top;
1123 
1124     log_info(os, thread)("Capturing initial stack in %s thread: req. size: " SIZE_FORMAT "K, actual size: "
1125                          SIZE_FORMAT "K, top=" INTPTR_FORMAT ", bottom=" INTPTR_FORMAT,
1126                          primordial ? "primordial" : "user", max_size / K,  _initial_thread_stack_size / K,
1127                          stack_top, intptr_t(_initial_thread_stack_bottom));
1128   }
1129 }
1130 
1131 ////////////////////////////////////////////////////////////////////////////////
1132 // time support
1133 
1134 // Time since start-up in seconds to a fine granularity.


3153 //  req_addr or NULL.
3154 static char* anon_mmap_aligned(size_t bytes, size_t alignment, char* req_addr) {
3155 
3156   size_t extra_size = bytes;
3157   if (req_addr == NULL && alignment > 0) {
3158     extra_size += alignment;
3159   }
3160 
3161   char* start = (char*) ::mmap(req_addr, extra_size, PROT_NONE,
3162     MAP_PRIVATE|MAP_ANONYMOUS|MAP_NORESERVE,
3163     -1, 0);
3164   if (start == MAP_FAILED) {
3165     start = NULL;
3166   } else {
3167     if (req_addr != NULL) {
3168       if (start != req_addr) {
3169         ::munmap(start, extra_size);
3170         start = NULL;
3171       }
3172     } else {
3173       char* const start_aligned = align_up(start, alignment);
3174       char* const end_aligned = start_aligned + bytes;
3175       char* const end = start + extra_size;
3176       if (start_aligned > start) {
3177         ::munmap(start, start_aligned - start);
3178       }
3179       if (end_aligned < end) {
3180         ::munmap(end_aligned, end - end_aligned);
3181       }
3182       start = start_aligned;
3183     }
3184   }
3185   return start;
3186 }
3187 
3188 static int anon_munmap(char * addr, size_t size) {
3189   return ::munmap(addr, size) == 0;
3190 }
3191 
3192 char* os::pd_reserve_memory(size_t bytes, char* requested_addr,
3193                             size_t alignment_hint) {
3194   return anon_mmap(requested_addr, bytes, (requested_addr != NULL));
3195 }
3196 
3197 bool os::pd_release_memory(char* addr, size_t size) {
3198   return anon_munmap(addr, size);
3199 }
3200 
3201 static bool linux_mprotect(char* addr, size_t size, int prot) {
3202   // Linux wants the mprotect address argument to be page aligned.
3203   char* bottom = (char*)align_down((intptr_t)addr, os::Linux::page_size());
3204 
3205   // According to SUSv3, mprotect() should only be used with mappings
3206   // established by mmap(), and mmap() always maps whole pages. Unaligned
3207   // 'addr' likely indicates problem in the VM (e.g. trying to change
3208   // protection of malloc'ed or statically allocated memory). Check the
3209   // caller if you hit this assert.
3210   assert(addr == bottom, "sanity check");
3211 
3212   size = align_up(pointer_delta(addr, bottom, 1) + size, os::Linux::page_size());
3213   return ::mprotect(bottom, size, prot) == 0;
3214 }
3215 
3216 // Set protections specified
3217 bool os::protect_memory(char* addr, size_t bytes, ProtType prot,
3218                         bool is_committed) {
3219   unsigned int p = 0;
3220   switch (prot) {
3221   case MEM_PROT_NONE: p = PROT_NONE; break;
3222   case MEM_PROT_READ: p = PROT_READ; break;
3223   case MEM_PROT_RW:   p = PROT_READ|PROT_WRITE; break;
3224   case MEM_PROT_RWX:  p = PROT_READ|PROT_WRITE|PROT_EXEC; break;
3225   default:
3226     ShouldNotReachHere();
3227   }
3228   // is_committed is unused.
3229   return linux_mprotect(addr, bytes, p);
3230 }
3231 
3232 bool os::guard_memory(char* addr, size_t size) {
3233   return linux_mprotect(addr, size, PROT_NONE);
3234 }
3235 
3236 bool os::unguard_memory(char* addr, size_t size) {
3237   return linux_mprotect(addr, size, PROT_READ|PROT_WRITE);
3238 }
3239 
3240 bool os::Linux::transparent_huge_pages_sanity_check(bool warn,
3241                                                     size_t page_size) {
3242   bool result = false;
3243   void *p = mmap(NULL, page_size * 2, PROT_READ|PROT_WRITE,
3244                  MAP_ANONYMOUS|MAP_PRIVATE,
3245                  -1, 0);
3246   if (p != MAP_FAILED) {
3247     void *aligned_p = align_up(p, page_size);
3248 
3249     result = madvise(aligned_p, page_size, MADV_HUGEPAGE) == 0;
3250 
3251     munmap(p, page_size * 2);
3252   }
3253 
3254   if (warn && !result) {
3255     warning("TransparentHugePages is not supported by the operating system.");
3256   }
3257 
3258   return result;
3259 }
3260 
3261 bool os::Linux::hugetlbfs_sanity_check(bool warn, size_t page_size) {
3262   bool result = false;
3263   void *p = mmap(NULL, page_size, PROT_READ|PROT_WRITE,
3264                  MAP_ANONYMOUS|MAP_PRIVATE|MAP_HUGETLB,
3265                  -1, 0);
3266 
3267   if (p != MAP_FAILED) {


3470 
3471 #define shm_warning_format(format, ...)              \
3472   do {                                               \
3473     if (UseLargePages &&                             \
3474         (!FLAG_IS_DEFAULT(UseLargePages) ||          \
3475          !FLAG_IS_DEFAULT(UseSHM) ||                 \
3476          !FLAG_IS_DEFAULT(LargePageSizeInBytes))) {  \
3477       warning(format, __VA_ARGS__);                  \
3478     }                                                \
3479   } while (0)
3480 
3481 #define shm_warning(str) shm_warning_format("%s", str)
3482 
3483 #define shm_warning_with_errno(str)                \
3484   do {                                             \
3485     int err = errno;                               \
3486     shm_warning_format(str " (error = %d)", err);  \
3487   } while (0)
3488 
3489 static char* shmat_with_alignment(int shmid, size_t bytes, size_t alignment) {
3490   assert(is_aligned(bytes, alignment), "Must be divisible by the alignment");
3491 
3492   if (!is_aligned(alignment, SHMLBA)) {
3493     assert(false, "Code below assumes that alignment is at least SHMLBA aligned");
3494     return NULL;
3495   }
3496 
3497   // To ensure that we get 'alignment' aligned memory from shmat,
3498   // we pre-reserve aligned virtual memory and then attach to that.
3499 
3500   char* pre_reserved_addr = anon_mmap_aligned(bytes, alignment, NULL);
3501   if (pre_reserved_addr == NULL) {
3502     // Couldn't pre-reserve aligned memory.
3503     shm_warning("Failed to pre-reserve aligned memory for shmat.");
3504     return NULL;
3505   }
3506 
3507   // SHM_REMAP is needed to allow shmat to map over an existing mapping.
3508   char* addr = (char*)shmat(shmid, pre_reserved_addr, SHM_REMAP);
3509 
3510   if ((intptr_t)addr == -1) {
3511     int err = errno;
3512     shm_warning_with_errno("Failed to attach shared memory.");
3513 
3514     assert(err != EACCES, "Unexpected error");
3515     assert(err != EIDRM,  "Unexpected error");
3516     assert(err != EINVAL, "Unexpected error");
3517 
3518     // Since we don't know if the kernel unmapped the pre-reserved memory area
3519     // we can't unmap it, since that would potentially unmap memory that was
3520     // mapped from other threads.
3521     return NULL;
3522   }
3523 
3524   return addr;
3525 }
3526 
3527 static char* shmat_at_address(int shmid, char* req_addr) {
3528   if (!is_aligned(req_addr, SHMLBA)) {
3529     assert(false, "Requested address needs to be SHMLBA aligned");
3530     return NULL;
3531   }
3532 
3533   char* addr = (char*)shmat(shmid, req_addr, 0);
3534 
3535   if ((intptr_t)addr == -1) {
3536     shm_warning_with_errno("Failed to attach shared memory.");
3537     return NULL;
3538   }
3539 
3540   return addr;
3541 }
3542 
3543 static char* shmat_large_pages(int shmid, size_t bytes, size_t alignment, char* req_addr) {
3544   // If a req_addr has been provided, we assume that the caller has already aligned the address.
3545   if (req_addr != NULL) {
3546     assert(is_aligned(req_addr, os::large_page_size()), "Must be divisible by the large page size");
3547     assert(is_aligned(req_addr, alignment), "Must be divisible by given alignment");
3548     return shmat_at_address(shmid, req_addr);
3549   }
3550 
3551   // Since shmid has been setup with SHM_HUGETLB, shmat will automatically
3552   // return large page size aligned memory addresses when req_addr == NULL.
3553   // However, if the alignment is larger than the large page size, we have
3554   // to manually ensure that the memory returned is 'alignment' aligned.
3555   if (alignment > os::large_page_size()) {
3556     assert(is_aligned(alignment, os::large_page_size()), "Must be divisible by the large page size");
3557     return shmat_with_alignment(shmid, bytes, alignment);
3558   } else {
3559     return shmat_at_address(shmid, NULL);
3560   }
3561 }
3562 
3563 char* os::Linux::reserve_memory_special_shm(size_t bytes, size_t alignment,
3564                                             char* req_addr, bool exec) {
3565   // "exec" is passed in but not used.  Creating the shared image for
3566   // the code cache doesn't have an SHM_X executable permission to check.
3567   assert(UseLargePages && UseSHM, "only for SHM large pages");
3568   assert(is_aligned(req_addr, os::large_page_size()), "Unaligned address");
3569   assert(is_aligned(req_addr, alignment), "Unaligned address");
3570 
3571   if (!is_aligned(bytes, os::large_page_size())) {
3572     return NULL; // Fallback to small pages.
3573   }
3574 
3575   // Create a large shared memory region to attach to based on size.
3576   // Currently, size is the total size of the heap.
3577   int shmid = shmget(IPC_PRIVATE, bytes, SHM_HUGETLB|IPC_CREAT|SHM_R|SHM_W);
3578   if (shmid == -1) {
3579     // Possible reasons for shmget failure:
3580     // 1. shmmax is too small for Java heap.
3581     //    > check shmmax value: cat /proc/sys/kernel/shmmax
3582     //    > increase shmmax value: echo "0xffffffff" > /proc/sys/kernel/shmmax
3583     // 2. not enough large page memory.
3584     //    > check available large pages: cat /proc/meminfo
3585     //    > increase amount of large pages:
3586     //          echo new_value > /proc/sys/vm/nr_hugepages
3587     //      Note 1: different Linux may use different name for this property,
3588     //            e.g. on Redhat AS-3 it is "hugetlb_pool".
3589     //      Note 2: it's possible there's enough physical memory available but
3590     //            they are so fragmented after a long run that they can't
3591     //            coalesce into large pages. Try to reserve large pages when


3610                                         int error) {
3611   assert(error == ENOMEM, "Only expect to fail if no memory is available");
3612 
3613   bool warn_on_failure = UseLargePages &&
3614       (!FLAG_IS_DEFAULT(UseLargePages) ||
3615        !FLAG_IS_DEFAULT(UseHugeTLBFS) ||
3616        !FLAG_IS_DEFAULT(LargePageSizeInBytes));
3617 
3618   if (warn_on_failure) {
3619     char msg[128];
3620     jio_snprintf(msg, sizeof(msg), "Failed to reserve large pages memory req_addr: "
3621                  PTR_FORMAT " bytes: " SIZE_FORMAT " (errno = %d).", req_addr, bytes, error);
3622     warning("%s", msg);
3623   }
3624 }
3625 
3626 char* os::Linux::reserve_memory_special_huge_tlbfs_only(size_t bytes,
3627                                                         char* req_addr,
3628                                                         bool exec) {
3629   assert(UseLargePages && UseHugeTLBFS, "only for Huge TLBFS large pages");
3630   assert(is_aligned(bytes, os::large_page_size()), "Unaligned size");
3631   assert(is_aligned(req_addr, os::large_page_size()), "Unaligned address");
3632 
3633   int prot = exec ? PROT_READ|PROT_WRITE|PROT_EXEC : PROT_READ|PROT_WRITE;
3634   char* addr = (char*)::mmap(req_addr, bytes, prot,
3635                              MAP_PRIVATE|MAP_ANONYMOUS|MAP_HUGETLB,
3636                              -1, 0);
3637 
3638   if (addr == MAP_FAILED) {
3639     warn_on_large_pages_failure(req_addr, bytes, errno);
3640     return NULL;
3641   }
3642 
3643   assert(is_aligned(addr, os::large_page_size()), "Must be");
3644 
3645   return addr;
3646 }
3647 
3648 // Reserve memory using mmap(MAP_HUGETLB).
3649 //  - bytes shall be a multiple of alignment.
3650 //  - req_addr can be NULL. If not NULL, it must be a multiple of alignment.
3651 //  - alignment sets the alignment at which memory shall be allocated.
3652 //     It must be a multiple of allocation granularity.
3653 // Returns address of memory or NULL. If req_addr was not NULL, will only return
3654 //  req_addr or NULL.
3655 char* os::Linux::reserve_memory_special_huge_tlbfs_mixed(size_t bytes,
3656                                                          size_t alignment,
3657                                                          char* req_addr,
3658                                                          bool exec) {
3659   size_t large_page_size = os::large_page_size();
3660   assert(bytes >= large_page_size, "Shouldn't allocate large pages for small sizes");
3661 
3662   assert(is_aligned(req_addr, alignment), "Must be");
3663   assert(is_aligned(bytes, alignment), "Must be");
3664 
3665   // First reserve - but not commit - the address range in small pages.
3666   char* const start = anon_mmap_aligned(bytes, alignment, req_addr);
3667 
3668   if (start == NULL) {
3669     return NULL;
3670   }
3671 
3672   assert(is_aligned(start, alignment), "Must be");
3673 
3674   char* end = start + bytes;
3675 
3676   // Find the regions of the allocated chunk that can be promoted to large pages.
3677   char* lp_start = align_up(start, large_page_size);
3678   char* lp_end   = align_down(end, large_page_size);
3679 
3680   size_t lp_bytes = lp_end - lp_start;
3681 
3682   assert(is_aligned(lp_bytes, large_page_size), "Must be");
3683 
3684   if (lp_bytes == 0) {
3685     // The mapped region doesn't even span the start and the end of a large page.
3686     // Fall back to allocate a non-special area.
3687     ::munmap(start, end - start);
3688     return NULL;
3689   }
3690 
3691   int prot = exec ? PROT_READ|PROT_WRITE|PROT_EXEC : PROT_READ|PROT_WRITE;
3692 
3693   void* result;
3694 
3695   // Commit small-paged leading area.
3696   if (start != lp_start) {
3697     result = ::mmap(start, lp_start - start, prot,
3698                     MAP_PRIVATE|MAP_ANONYMOUS|MAP_FIXED,
3699                     -1, 0);
3700     if (result == MAP_FAILED) {
3701       ::munmap(lp_start, end - lp_start);
3702       return NULL;


3723 
3724   // Commit small-paged trailing area.
3725   if (lp_end != end) {
3726     result = ::mmap(lp_end, end - lp_end, prot,
3727                     MAP_PRIVATE|MAP_ANONYMOUS|MAP_FIXED,
3728                     -1, 0);
3729     if (result == MAP_FAILED) {
3730       ::munmap(start, lp_end - start);
3731       return NULL;
3732     }
3733   }
3734 
3735   return start;
3736 }
3737 
3738 char* os::Linux::reserve_memory_special_huge_tlbfs(size_t bytes,
3739                                                    size_t alignment,
3740                                                    char* req_addr,
3741                                                    bool exec) {
3742   assert(UseLargePages && UseHugeTLBFS, "only for Huge TLBFS large pages");
3743   assert(is_aligned(req_addr, alignment), "Must be");
3744   assert(is_aligned(alignment, os::vm_allocation_granularity()), "Must be");
3745   assert(is_power_of_2(os::large_page_size()), "Must be");
3746   assert(bytes >= os::large_page_size(), "Shouldn't allocate large pages for small sizes");
3747 
3748   if (is_aligned(bytes, os::large_page_size()) && alignment <= os::large_page_size()) {
3749     return reserve_memory_special_huge_tlbfs_only(bytes, req_addr, exec);
3750   } else {
3751     return reserve_memory_special_huge_tlbfs_mixed(bytes, alignment, req_addr, exec);
3752   }
3753 }
3754 
3755 char* os::reserve_memory_special(size_t bytes, size_t alignment,
3756                                  char* req_addr, bool exec) {
3757   assert(UseLargePages, "only for large pages");
3758 
3759   char* addr;
3760   if (UseSHM) {
3761     addr = os::Linux::reserve_memory_special_shm(bytes, alignment, req_addr, exec);
3762   } else {
3763     assert(UseHugeTLBFS, "must be");
3764     addr = os::Linux::reserve_memory_special_huge_tlbfs(bytes, alignment, req_addr, exec);
3765   }
3766 
3767   if (addr != NULL) {
3768     if (UseNUMAInterleaving) {


5950       PROT_NONE, MAP_PRIVATE|MAP_ANONYMOUS|MAP_NORESERVE,
5951       -1, 0);
5952     assert(mapping1 != MAP_FAILED, "should work");
5953 
5954     char* const mapping2 = (char*) ::mmap(NULL, mapping_size,
5955       PROT_NONE, MAP_PRIVATE|MAP_ANONYMOUS|MAP_NORESERVE,
5956       -1, 0);
5957     assert(mapping2 != MAP_FAILED, "should work");
5958 
5959     // Unmap the first mapping, but leave the second mapping intact: the first
5960     // mapping will serve as a value for a "good" req_addr (case 2). The second
5961     // mapping, still intact, as "bad" req_addr (case 3).
5962     ::munmap(mapping1, mapping_size);
5963 
5964     // Case 1
5965     test_log("%s, req_addr NULL:", __FUNCTION__);
5966     test_log("size            align           result");
5967 
5968     for (int i = 0; i < num_sizes; i++) {
5969       const size_t size = sizes[i];
5970       for (size_t alignment = ag; is_aligned(size, alignment); alignment *= 2) {
5971         char* p = os::Linux::reserve_memory_special_huge_tlbfs_mixed(size, alignment, NULL, false);
5972         test_log(SIZE_FORMAT_HEX " " SIZE_FORMAT_HEX " ->  " PTR_FORMAT " %s",
5973                  size, alignment, p2i(p), (p != NULL ? "" : "(failed)"));
5974         if (p != NULL) {
5975           assert(is_aligned(p, alignment), "must be");
5976           small_page_write(p, size);
5977           os::Linux::release_memory_special_huge_tlbfs(p, size);
5978         }
5979       }
5980     }
5981 
5982     // Case 2
5983     test_log("%s, req_addr non-NULL:", __FUNCTION__);
5984     test_log("size            align           req_addr         result");
5985 
5986     for (int i = 0; i < num_sizes; i++) {
5987       const size_t size = sizes[i];
5988       for (size_t alignment = ag; is_aligned(size, alignment); alignment *= 2) {
5989         char* const req_addr = align_up(mapping1, alignment);
5990         char* p = os::Linux::reserve_memory_special_huge_tlbfs_mixed(size, alignment, req_addr, false);
5991         test_log(SIZE_FORMAT_HEX " " SIZE_FORMAT_HEX " " PTR_FORMAT " ->  " PTR_FORMAT " %s",
5992                  size, alignment, p2i(req_addr), p2i(p),
5993                  ((p != NULL ? (p == req_addr ? "(exact match)" : "") : "(failed)")));
5994         if (p != NULL) {
5995           assert(p == req_addr, "must be");
5996           small_page_write(p, size);
5997           os::Linux::release_memory_special_huge_tlbfs(p, size);
5998         }
5999       }
6000     }
6001 
6002     // Case 3
6003     test_log("%s, req_addr non-NULL with preexisting mapping:", __FUNCTION__);
6004     test_log("size            align           req_addr         result");
6005 
6006     for (int i = 0; i < num_sizes; i++) {
6007       const size_t size = sizes[i];
6008       for (size_t alignment = ag; is_aligned(size, alignment); alignment *= 2) {
6009         char* const req_addr = align_up(mapping2, alignment);
6010         char* p = os::Linux::reserve_memory_special_huge_tlbfs_mixed(size, alignment, req_addr, false);
6011         test_log(SIZE_FORMAT_HEX " " SIZE_FORMAT_HEX " " PTR_FORMAT " ->  " PTR_FORMAT " %s",
6012                  size, alignment, p2i(req_addr), p2i(p), ((p != NULL ? "" : "(failed)")));
6013         // as the area around req_addr contains already existing mappings, the API should always
6014         // return NULL (as per contract, it cannot return another address)
6015         assert(p == NULL, "must be");
6016       }
6017     }
6018 
6019     ::munmap(mapping2, mapping_size);
6020 
6021   }
6022 
6023   static void test_reserve_memory_special_huge_tlbfs() {
6024     if (!UseHugeTLBFS) {
6025       return;
6026     }
6027 
6028     test_reserve_memory_special_huge_tlbfs_only();
6029     test_reserve_memory_special_huge_tlbfs_mixed();
6030   }
6031 
6032   static void test_reserve_memory_special_shm(size_t size, size_t alignment) {
6033     if (!UseSHM) {
6034       return;
6035     }
6036 
6037     test_log("test_reserve_memory_special_shm(" SIZE_FORMAT ", " SIZE_FORMAT ")", size, alignment);
6038 
6039     char* addr = os::Linux::reserve_memory_special_shm(size, alignment, NULL, false);
6040 
6041     if (addr != NULL) {
6042       assert(is_aligned(addr, alignment), "Check");
6043       assert(is_aligned(addr, os::large_page_size()), "Check");
6044 
6045       small_page_write(addr, size);
6046 
6047       os::Linux::release_memory_special_shm(addr, size);
6048     }
6049   }
6050 
6051   static void test_reserve_memory_special_shm() {
6052     size_t lp = os::large_page_size();
6053     size_t ag = os::vm_allocation_granularity();
6054 
6055     for (size_t size = ag; size < lp * 3; size += ag) {
6056       for (size_t alignment = ag; is_aligned(size, alignment); alignment *= 2) {
6057         test_reserve_memory_special_shm(size, alignment);
6058       }
6059     }
6060   }
6061 
6062   static void test() {
6063     test_reserve_memory_special_huge_tlbfs();
6064     test_reserve_memory_special_shm();
6065   }
6066 };
6067 
6068 void TestReserveMemorySpecial_test() {
6069   TestReserveMemorySpecial::test();
6070 }
6071 
6072 #endif
< prev index next >