3052 os::Linux::numa_node_to_cpus_func_t os::Linux::_numa_node_to_cpus;
3053 os::Linux::numa_max_node_func_t os::Linux::_numa_max_node;
3054 os::Linux::numa_num_configured_nodes_func_t os::Linux::_numa_num_configured_nodes;
3055 os::Linux::numa_available_func_t os::Linux::_numa_available;
3056 os::Linux::numa_tonode_memory_func_t os::Linux::_numa_tonode_memory;
3057 os::Linux::numa_interleave_memory_func_t os::Linux::_numa_interleave_memory;
3058 os::Linux::numa_interleave_memory_v2_func_t os::Linux::_numa_interleave_memory_v2;
3059 os::Linux::numa_set_bind_policy_func_t os::Linux::_numa_set_bind_policy;
3060 os::Linux::numa_bitmask_isbitset_func_t os::Linux::_numa_bitmask_isbitset;
3061 os::Linux::numa_distance_func_t os::Linux::_numa_distance;
3062 unsigned long* os::Linux::_numa_all_nodes;
3063 struct bitmask* os::Linux::_numa_all_nodes_ptr;
3064 struct bitmask* os::Linux::_numa_nodes_ptr;
3065
3066 bool os::pd_uncommit_memory(char* addr, size_t size) {
3067 uintptr_t res = (uintptr_t) ::mmap(addr, size, PROT_NONE,
3068 MAP_PRIVATE|MAP_FIXED|MAP_NORESERVE|MAP_ANONYMOUS, -1, 0);
3069 return res != (uintptr_t) MAP_FAILED;
3070 }
3071
3072 static address get_stack_commited_bottom(address bottom, size_t size) {
3073 address nbot = bottom;
3074 address ntop = bottom + size;
3075
3076 size_t page_sz = os::vm_page_size();
3077 unsigned pages = size / page_sz;
3078
3079 unsigned char vec[1];
3080 unsigned imin = 1, imax = pages + 1, imid;
3081 int mincore_return_value = 0;
3082
3083 assert(imin <= imax, "Unexpected page size");
3084
3085 while (imin < imax) {
3086 imid = (imax + imin) / 2;
3087 nbot = ntop - (imid * page_sz);
3088
3089 // Use a trick with mincore to check whether the page is mapped or not.
3090 // mincore sets vec to 1 if page resides in memory and to 0 if page
3091 // is swapped output but if page we are asking for is unmapped
3092 // it returns -1,ENOMEM
3093 mincore_return_value = mincore(nbot, page_sz, vec);
3094
3095 if (mincore_return_value == -1) {
3096 // Page is not mapped go up
3097 // to find first mapped page
3098 if (errno != EAGAIN) {
3099 assert(errno == ENOMEM, "Unexpected mincore errno");
3100 imax = imid;
3101 }
3102 } else {
3103 // Page is mapped go down
3104 // to find first not mapped page
3105 imin = imid + 1;
3106 }
3107 }
3108
3109 nbot = nbot + page_sz;
3110
3111 // Adjust stack bottom one page up if last checked page is not mapped
3112 if (mincore_return_value == -1) {
3113 nbot = nbot + page_sz;
3114 }
3115
3116 return nbot;
3117 }
3118
3119
3120 // Linux uses a growable mapping for the stack, and if the mapping for
3121 // the stack guard pages is not removed when we detach a thread the
3122 // stack cannot grow beyond the pages where the stack guard was
3123 // mapped. If at some point later in the process the stack expands to
3124 // that point, the Linux kernel cannot expand the stack any further
3125 // because the guard pages are in the way, and a segfault occurs.
3126 //
3127 // However, it's essential not to split the stack region by unmapping
3128 // a region (leaving a hole) that's already part of the stack mapping,
3129 // so if the stack mapping has already grown beyond the guard pages at
3130 // the time we create them, we have to truncate the stack mapping.
3131 // So, we need to know the extent of the stack mapping when
3132 // create_stack_guard_pages() is called.
3133
3134 // We only need this for stacks that are growable: at the time of
3135 // writing thread stacks don't use growable mappings (i.e. those
3136 // creeated with MAP_GROWSDOWN), and aren't marked "[stack]", so this
3137 // only applies to the main thread.
3138
3139 // If the (growable) stack mapping already extends beyond the point
3140 // where we're going to put our guard pages, truncate the mapping at
3141 // that point by munmap()ping it. This ensures that when we later
3142 // munmap() the guard pages we don't leave a hole in the stack
3143 // mapping. This only affects the main/primordial thread
3144
3145 bool os::pd_create_stack_guard_pages(char* addr, size_t size) {
3146 if (os::is_primordial_thread()) {
3147 // As we manually grow stack up to bottom inside create_attached_thread(),
3148 // it's likely that os::Linux::initial_thread_stack_bottom is mapped and
3149 // we don't need to do anything special.
3150 // Check it first, before calling heavy function.
3151 uintptr_t stack_extent = (uintptr_t) os::Linux::initial_thread_stack_bottom();
3152 unsigned char vec[1];
3153
3154 if (mincore((address)stack_extent, os::vm_page_size(), vec) == -1) {
3155 // Fallback to slow path on all errors, including EAGAIN
3156 stack_extent = (uintptr_t) get_stack_commited_bottom(
3157 os::Linux::initial_thread_stack_bottom(),
3158 (size_t)addr - stack_extent);
3159 }
3160
3161 if (stack_extent < (uintptr_t)addr) {
3162 ::munmap((void*)stack_extent, (uintptr_t)(addr - stack_extent));
3163 }
3164 }
3165
3166 return os::commit_memory(addr, size, !ExecMem);
3167 }
3168
3169 // If this is a growable mapping, remove the guard pages entirely by
3170 // munmap()ping them. If not, just call uncommit_memory(). This only
3171 // affects the main/primordial thread, but guard against future OS changes.
3172 // It's safe to always unmap guard pages for primordial thread because we
3173 // always place it right after end of the mapped region.
3174
3175 bool os::remove_stack_guard_pages(char* addr, size_t size) {
3176 uintptr_t stack_extent, stack_base;
3177
3178 if (os::is_primordial_thread()) {
3179 return ::munmap(addr, size) == 0;
3180 }
3181
3182 return os::uncommit_memory(addr, size);
3183 }
3184
3185 // If 'fixed' is true, anon_mmap() will attempt to reserve anonymous memory
3186 // at 'requested_addr'. If there are existing memory mappings at the same
3187 // location, however, they will be overwritten. If 'fixed' is false,
3188 // 'requested_addr' is only treated as a hint, the return value may or
3189 // may not start from the requested address. Unlike Linux mmap(), this
3190 // function returns NULL to indicate failure.
3191 static char* anon_mmap(char* requested_addr, size_t bytes, bool fixed) {
3192 char * addr;
3193 int flags;
3194
3195 flags = MAP_PRIVATE | MAP_NORESERVE | MAP_ANONYMOUS;
3196 if (fixed) {
3197 assert((uintptr_t)requested_addr % os::Linux::page_size() == 0, "unaligned address");
3198 flags |= MAP_FIXED;
3199 }
3200
3201 // Map reserved/uncommitted pages PROT_NONE so we fail early if we
3202 // touch an uncommitted page. Otherwise, the read/write might
|
3052 os::Linux::numa_node_to_cpus_func_t os::Linux::_numa_node_to_cpus;
3053 os::Linux::numa_max_node_func_t os::Linux::_numa_max_node;
3054 os::Linux::numa_num_configured_nodes_func_t os::Linux::_numa_num_configured_nodes;
3055 os::Linux::numa_available_func_t os::Linux::_numa_available;
3056 os::Linux::numa_tonode_memory_func_t os::Linux::_numa_tonode_memory;
3057 os::Linux::numa_interleave_memory_func_t os::Linux::_numa_interleave_memory;
3058 os::Linux::numa_interleave_memory_v2_func_t os::Linux::_numa_interleave_memory_v2;
3059 os::Linux::numa_set_bind_policy_func_t os::Linux::_numa_set_bind_policy;
3060 os::Linux::numa_bitmask_isbitset_func_t os::Linux::_numa_bitmask_isbitset;
3061 os::Linux::numa_distance_func_t os::Linux::_numa_distance;
3062 unsigned long* os::Linux::_numa_all_nodes;
3063 struct bitmask* os::Linux::_numa_all_nodes_ptr;
3064 struct bitmask* os::Linux::_numa_nodes_ptr;
3065
3066 bool os::pd_uncommit_memory(char* addr, size_t size) {
3067 uintptr_t res = (uintptr_t) ::mmap(addr, size, PROT_NONE,
3068 MAP_PRIVATE|MAP_FIXED|MAP_NORESERVE|MAP_ANONYMOUS, -1, 0);
3069 return res != (uintptr_t) MAP_FAILED;
3070 }
3071
3072 // If there is no page mapped/committed, top (bottom + size) is returned
3073 static address get_stack_mapped_bottom(address bottom,
3074 size_t size,
3075 bool committed_only /* must have backing pages */) {
3076 // address used to test if the page is mapped/committed
3077 address test_addr = bottom + size;
3078 size_t page_sz = os::vm_page_size();
3079 unsigned pages = size / page_sz;
3080
3081 unsigned char vec[1];
3082 unsigned imin = 1, imax = pages + 1, imid;
3083 int mincore_return_value = 0;
3084
3085 assert(imin <= imax, "Unexpected page size");
3086
3087 while (imin < imax) {
3088 imid = (imax + imin) / 2;
3089 test_addr = bottom + (imid * page_sz);
3090
3091 // Use a trick with mincore to check whether the page is mapped or not.
3092 // mincore sets vec to 1 if page resides in memory and to 0 if page
3093 // is swapped output but if page we are asking for is unmapped
3094 // it returns -1,ENOMEM
3095 mincore_return_value = mincore(test_addr, page_sz, vec);
3096
3097 if (mincore_return_value == -1 || (committed_only && (vec[0] & 0x01) == 0)) {
3098 // Page is not mapped/committed go up
3099 // to find first mapped/committed page
3100 if ((mincore_return_value == -1 && errno != EAGAIN)
3101 || (committed_only && (vec[0] & 0x01) == 0)) {
3102 assert(mincore_return_value != -1 || errno == ENOMEM, "Unexpected mincore errno");
3103
3104 imin = imid + 1;
3105 }
3106 } else {
3107 // mapped/committed, go down
3108 imax= imid;
3109 }
3110 }
3111
3112 // Adjust stack bottom one page up if last checked page is not mapped/committed
3113 if (mincore_return_value == -1 || (committed_only && (vec[0] & 0x01) == 0)) {
3114 assert(mincore_return_value != -1 || (errno != EAGAIN && errno != ENOMEM),
3115 "Should not get to here");
3116
3117 test_addr = test_addr + page_sz;
3118 }
3119
3120 return test_addr;
3121 }
3122
3123 // Linux uses a growable mapping for the stack, and if the mapping for
3124 // the stack guard pages is not removed when we detach a thread the
3125 // stack cannot grow beyond the pages where the stack guard was
3126 // mapped. If at some point later in the process the stack expands to
3127 // that point, the Linux kernel cannot expand the stack any further
3128 // because the guard pages are in the way, and a segfault occurs.
3129 //
3130 // However, it's essential not to split the stack region by unmapping
3131 // a region (leaving a hole) that's already part of the stack mapping,
3132 // so if the stack mapping has already grown beyond the guard pages at
3133 // the time we create them, we have to truncate the stack mapping.
3134 // So, we need to know the extent of the stack mapping when
3135 // create_stack_guard_pages() is called.
3136
3137 // We only need this for stacks that are growable: at the time of
3138 // writing thread stacks don't use growable mappings (i.e. those
3139 // creeated with MAP_GROWSDOWN), and aren't marked "[stack]", so this
3140 // only applies to the main thread.
3141
3142 // If the (growable) stack mapping already extends beyond the point
3143 // where we're going to put our guard pages, truncate the mapping at
3144 // that point by munmap()ping it. This ensures that when we later
3145 // munmap() the guard pages we don't leave a hole in the stack
3146 // mapping. This only affects the main/primordial thread
3147
3148 bool os::pd_create_stack_guard_pages(char* addr, size_t size) {
3149 if (os::is_primordial_thread()) {
3150 // As we manually grow stack up to bottom inside create_attached_thread(),
3151 // it's likely that os::Linux::initial_thread_stack_bottom is mapped and
3152 // we don't need to do anything special.
3153 // Check it first, before calling heavy function.
3154 uintptr_t stack_extent = (uintptr_t) os::Linux::initial_thread_stack_bottom();
3155 unsigned char vec[1];
3156
3157 if (mincore((address)stack_extent, os::vm_page_size(), vec) == -1) {
3158 // Fallback to slow path on all errors, including EAGAIN
3159 stack_extent = (uintptr_t) get_stack_mapped_bottom(os::Linux::initial_thread_stack_bottom(),
3160 (size_t)addr - stack_extent,
3161 false /* committed_only */);
3162 }
3163
3164 if (stack_extent < (uintptr_t)addr) {
3165 ::munmap((void*)stack_extent, (uintptr_t)(addr - stack_extent));
3166 }
3167 }
3168
3169 return os::commit_memory(addr, size, !ExecMem);
3170 }
3171
3172 // If this is a growable mapping, remove the guard pages entirely by
3173 // munmap()ping them. If not, just call uncommit_memory(). This only
3174 // affects the main/primordial thread, but guard against future OS changes.
3175 // It's safe to always unmap guard pages for primordial thread because we
3176 // always place it right after end of the mapped region.
3177
3178 bool os::remove_stack_guard_pages(char* addr, size_t size) {
3179 uintptr_t stack_extent, stack_base;
3180
3181 if (os::is_primordial_thread()) {
3182 return ::munmap(addr, size) == 0;
3183 }
3184
3185 return os::uncommit_memory(addr, size);
3186 }
3187
3188 size_t os::committed_stack_size(address bottom, size_t size) {
3189 address bot = get_stack_mapped_bottom(bottom, size, true /* committed_only */);
3190 return size_t(bottom + size - bot);
3191 }
3192
3193 // If 'fixed' is true, anon_mmap() will attempt to reserve anonymous memory
3194 // at 'requested_addr'. If there are existing memory mappings at the same
3195 // location, however, they will be overwritten. If 'fixed' is false,
3196 // 'requested_addr' is only treated as a hint, the return value may or
3197 // may not start from the requested address. Unlike Linux mmap(), this
3198 // function returns NULL to indicate failure.
3199 static char* anon_mmap(char* requested_addr, size_t bytes, bool fixed) {
3200 char * addr;
3201 int flags;
3202
3203 flags = MAP_PRIVATE | MAP_NORESERVE | MAP_ANONYMOUS;
3204 if (fixed) {
3205 assert((uintptr_t)requested_addr % os::Linux::page_size() == 0, "unaligned address");
3206 flags |= MAP_FIXED;
3207 }
3208
3209 // Map reserved/uncommitted pages PROT_NONE so we fail early if we
3210 // touch an uncommitted page. Otherwise, the read/write might
|