1 /*
2 * Copyright (c) 1997, 2018, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "logging/log.hpp"
27 #include "memory/resourceArea.hpp"
28 #include "memory/virtualspace.hpp"
29 #include "oops/markOop.hpp"
30 #include "oops/oop.inline.hpp"
31 #include "runtime/os.inline.hpp"
32 #include "services/memTracker.hpp"
33 #include "utilities/align.hpp"
34
35 // ReservedSpace
36
37 // Dummy constructor
38 ReservedSpace::ReservedSpace() : _base(NULL), _size(0), _noaccess_prefix(0),
39 _nvdimm_base_nv(NULL), _nvdimm_size(0), _dram_size(0),
40 _alignment(0), _special(false), _executable(false), _fd_for_heap(-1), _fd_for_nvdimm(-1) {
41 }
42
43 ReservedSpace::ReservedSpace(size_t size, size_t preferred_page_size) : _fd_for_heap(-1), _fd_for_nvdimm(-1),
44 _nvdimm_base_nv(NULL), _nvdimm_size(0), _dram_size(0) {
45 bool has_preferred_page_size = preferred_page_size != 0;
46 // Want to use large pages where possible and pad with small pages.
47 size_t page_size = has_preferred_page_size ? preferred_page_size : os::page_size_for_region_unaligned(size, 1);
48 bool large_pages = page_size != (size_t)os::vm_page_size();
49 size_t alignment;
50 if (large_pages && has_preferred_page_size) {
51 alignment = MAX2(page_size, (size_t)os::vm_allocation_granularity());
52 // ReservedSpace initialization requires size to be aligned to the given
53 // alignment. Align the size up.
54 size = align_up(size, alignment);
55 } else {
56 // Don't force the alignment to be large page aligned,
57 // since that will waste memory.
58 alignment = os::vm_allocation_granularity();
59 }
60 initialize(size, alignment, large_pages, NULL, false);
61 }
62
63 ReservedSpace::ReservedSpace(size_t size, size_t alignment,
64 bool large,
65 char* requested_address) : _fd_for_heap(-1), _fd_for_nvdimm(-1),
66 _nvdimm_base_nv(NULL), _nvdimm_size(0), _dram_size(0) {
67 initialize(size, alignment, large, requested_address, false);
68 }
69
70 ReservedSpace::ReservedSpace(size_t size, size_t alignment,
71 bool large,
72 bool executable) : _fd_for_heap(-1), _fd_for_nvdimm(-1),
73 _nvdimm_base_nv(NULL), _nvdimm_size(0), _dram_size(0) {
74 initialize(size, alignment, large, NULL, executable);
75 }
76
77 // Helper method
78 static void unmap_or_release_memory(char* base, size_t size, bool is_file_mapped) {
79 if (is_file_mapped) {
80 if (!os::unmap_memory(base, size)) {
81 fatal("os::unmap_memory failed");
82 }
83 } else if (!os::release_memory(base, size)) {
84 fatal("os::release_memory failed");
85 }
86 }
87
88 // Helper method.
89 static bool failed_to_reserve_as_requested(char* base, char* requested_address,
90 const size_t size, bool special, bool is_file_mapped = false)
91 {
92 if (base == requested_address || requested_address == NULL)
93 return false; // did not fail
94
95 if (base != NULL) {
96 // Different reserve address may be acceptable in other cases
97 // but for compressed oops heap should be at requested address.
98 assert(UseCompressedOops, "currently requested address used only for compressed oops");
99 log_debug(gc, heap, coops)("Reserved memory not at requested address: " PTR_FORMAT " vs " PTR_FORMAT, p2i(base), p2i(requested_address));
100 // OS ignored requested address. Try different address.
101 if (special) {
102 if (!os::release_memory_special(base, size)) {
103 fatal("os::release_memory_special failed");
104 }
105 } else {
106 unmap_or_release_memory(base, size, is_file_mapped);
107 }
108 }
109 return true;
110 }
111
112 void ReservedSpace::initialize(size_t size, size_t alignment, bool large,
113 char* requested_address,
114 bool executable) {
115 const size_t granularity = os::vm_allocation_granularity();
116 assert((size & (granularity - 1)) == 0,
117 "size not aligned to os::vm_allocation_granularity()");
118 assert((alignment & (granularity - 1)) == 0,
119 "alignment not aligned to os::vm_allocation_granularity()");
120 assert(alignment == 0 || is_power_of_2((intptr_t)alignment),
121 "not a power of 2");
122
123 alignment = MAX2(alignment, (size_t)os::vm_page_size());
124
125 _base = NULL;
126 _size = 0;
127 _special = false;
128 _executable = executable;
129 _alignment = 0;
130 _noaccess_prefix = 0;
131 if (size == 0) {
132 return;
133 }
134
135 // If OS doesn't support demand paging for large page memory, we need
136 // to use reserve_memory_special() to reserve and pin the entire region.
137 // If there is a backing file directory for this space then whether
138 // large pages are allocated is up to the filesystem of the backing file.
139 // So we ignore the UseLargePages flag in this case.
140 bool special = large && !os::can_commit_large_page_memory();
141 if (special && _fd_for_heap != -1) {
142 special = false;
143 if (UseLargePages && (!FLAG_IS_DEFAULT(UseLargePages) ||
144 !FLAG_IS_DEFAULT(LargePageSizeInBytes))) {
145 log_debug(gc, heap)("Ignoring UseLargePages since large page support is up to the file system of the backing file for Java heap");
146 }
147 }
148
149 char* base = NULL;
150 char* nvdimm_base = NULL;
151
152 if (special) {
153
154 base = os::reserve_memory_special(size, alignment, requested_address, executable);
155
156 if (base != NULL) {
157 if (failed_to_reserve_as_requested(base, requested_address, size, true)) {
158 // OS ignored requested address. Try different address.
159 return;
160 }
161 // Check alignment constraints.
162 assert((uintptr_t) base % alignment == 0,
163 "Large pages returned a non-aligned address, base: "
164 PTR_FORMAT " alignment: " SIZE_FORMAT_HEX,
165 p2i(base), alignment);
166 _special = true;
167 } else {
168 // failed; try to reserve regular memory below
169 if (UseLargePages && (!FLAG_IS_DEFAULT(UseLargePages) ||
170 !FLAG_IS_DEFAULT(LargePageSizeInBytes))) {
171 log_debug(gc, heap, coops)("Reserve regular memory without large pages");
172 }
173 }
174 }
175
176 if (base == NULL) {
177 // Optimistically assume that the OSes returns an aligned base pointer.
178 // When reserving a large address range, most OSes seem to align to at
179 // least 64K.
180
181 // If the memory was requested at a particular address, use
182 // os::attempt_reserve_memory_at() to avoid over mapping something
183 // important. If available space is not detected, return NULL.
184
185 if (requested_address != 0) {
186 base = os::attempt_reserve_memory_at(size, requested_address, _fd_for_heap);
187 if (failed_to_reserve_as_requested(base, requested_address, size, false, _fd_for_heap != -1)) {
188 // OS ignored requested address. Try different address.
189 base = NULL;
190 }
191 } else {
192 if (_nvdimm_base_nv != NULL && _fd_for_nvdimm != -1) {
193 base = os::reserve_memory(_dram_size, _nvdimm_base_nv, alignment, _fd_for_heap);
194 } else {
195 base = os::reserve_memory(size, NULL, alignment, _fd_for_heap);
196 }
197 }
198
199 if (base == NULL) return;
200
201 // Check alignment constraints
202 if ((((size_t)base) & (alignment - 1)) != 0) {
203 // Base not aligned, retry
204 unmap_or_release_memory(base, size, _fd_for_heap != -1 /*is_file_mapped*/);
205
206 // Make sure that size is aligned
207 size = align_up(size, alignment);
208 base = os::reserve_memory_aligned(size, alignment, _fd_for_heap);
209
210 if (requested_address != 0 &&
211 failed_to_reserve_as_requested(base, requested_address, size, false, _fd_for_heap != -1)) {
212 // As a result of the alignment constraints, the allocated base differs
213 // from the requested address. Return back to the caller who can
214 // take remedial action (like try again without a requested address).
215 assert(_base == NULL, "should be");
216 return;
217 }
218 }
219 }
220 // Done
221 _base = base;
222 _nvdimm_base = _base-_nvdimm_size;
223 _nvdimm_base_nv = NULL;
224 _dram_size = (size_t)size;
225 _size = size;
226 _alignment = alignment;
227 // If heap is reserved with a backing file, the entire space has been committed. So set the _special flag to true
228 if (_fd_for_heap != -1) {
229 _special = true;
230 }
231 }
232
233
234 ReservedSpace::ReservedSpace(char* base, size_t size, size_t alignment,
235 bool special, bool executable) {
236 assert((size % os::vm_allocation_granularity()) == 0,
237 "size not allocation aligned");
238 _base = base;
239 _size = size;
240 _nvdimm_base = NULL;
241 _nvdimm_base_nv = NULL;
242 _dram_size = (size_t)size;
243 _alignment = alignment;
244 _noaccess_prefix = 0;
245 _special = special;
246 _executable = executable;
247 }
248
249 ReservedSpace ReservedSpace::first_part(size_t partition_size, size_t alignment,
250 bool split, bool realloc) {
251 assert(partition_size <= size(), "partition failed");
252 if (split) {
253 os::split_reserved_memory(base(), size(), partition_size, realloc);
254 }
255 ReservedSpace result(base(), partition_size, alignment, special(),
256 executable());
257 return result;
258 }
259
260
261 ReservedSpace
262 ReservedSpace::last_part(size_t partition_size, size_t alignment) {
263 assert(partition_size <= size(), "partition failed");
264 ReservedSpace result(base() + partition_size, size() - partition_size,
265 alignment, special(), executable());
266 return result;
267 }
268
269
270 size_t ReservedSpace::page_align_size_up(size_t size) {
271 return align_up(size, os::vm_page_size());
272 }
273
274
275 size_t ReservedSpace::page_align_size_down(size_t size) {
276 return align_down(size, os::vm_page_size());
277 }
278
279
280 size_t ReservedSpace::allocation_align_size_up(size_t size) {
281 return align_up(size, os::vm_allocation_granularity());
282 }
283
284
285 size_t ReservedSpace::allocation_align_size_down(size_t size) {
286 return align_down(size, os::vm_allocation_granularity());
287 }
288
289
290 void ReservedSpace::release() {
291 if (is_reserved()) {
292 char *real_base = _base - _noaccess_prefix;
293 const size_t real_size = _size + _noaccess_prefix;
294 // unmap nvdimm
295 if (_fd_for_nvdimm != -1) {
296 os::unmap_memory(real_base+real_size, _nvdimm_size);
297 }
298 if (special()) {
299 if (_fd_for_heap != -1) {
300 os::unmap_memory(real_base, real_size);
301 } else {
302 os::release_memory_special(real_base, real_size);
303 }
304 } else{
305 os::release_memory(real_base, real_size);
306 }
307 _base = NULL;
308 _nvdimm_base = NULL;
309 _nvdimm_base_nv = NULL;
310 _dram_size = 0;
311 _nvdimm_size = 0;
312 _size = 0;
313 _noaccess_prefix = 0;
314 _alignment = 0;
315 _special = false;
316 _executable = false;
317 }
318 }
319
320 static size_t noaccess_prefix_size(size_t alignment) {
321 return lcm(os::vm_page_size(), alignment);
322 }
323
324 void ReservedHeapSpace::establish_noaccess_prefix() {
325 assert(_alignment >= (size_t)os::vm_page_size(), "must be at least page size big");
326 _noaccess_prefix = noaccess_prefix_size(_alignment);
327
328 if (base() && base() + _size > (char *)OopEncodingHeapMax) {
329 if (true
330 WIN64_ONLY(&& !UseLargePages)
331 AIX_ONLY(&& os::vm_page_size() != 64*K)) {
332 // Protect memory at the base of the allocated region.
333 // If special, the page was committed (only matters on windows)
334 if (!os::protect_memory(_base, _noaccess_prefix, os::MEM_PROT_NONE, _special)) {
335 fatal("cannot protect protection page");
336 }
337 log_debug(gc, heap, coops)("Protected page at the reserved heap base: "
338 PTR_FORMAT " / " INTX_FORMAT " bytes",
339 p2i(_base),
340 _noaccess_prefix);
341 assert(Universe::narrow_oop_use_implicit_null_checks() == true, "not initialized?");
342 } else {
343 Universe::set_narrow_oop_use_implicit_null_checks(false);
344 }
345 }
346
347 _base += _noaccess_prefix;
348 _size -= _noaccess_prefix;
349 assert(((uintptr_t)_base % _alignment == 0), "must be exactly of required alignment");
350 }
351
352 // Tries to allocate memory of size 'size' at address requested_address with alignment 'alignment'.
353 // Does not check whether the reserved memory actually is at requested_address, as the memory returned
354 // might still fulfill the wishes of the caller.
355 // Assures the memory is aligned to 'alignment'.
356 // NOTE: If ReservedHeapSpace already points to some reserved memory this is freed, first.
357 void ReservedHeapSpace::try_reserve_heap(size_t size,
358 size_t alignment,
359 bool large,
360 char* requested_address) {
361 if (_base != NULL) {
362 // We tried before, but we didn't like the address delivered.
363 release();
364 }
365
366 if (_fd_for_nvdimm != -1 && UseG1GC) {
367 char* base_nv = os::reserve_memory(size, requested_address, alignment);
368 initialize_g1gc_nvdimm_dram_sizes(size, alignment);
369 _nvdimm_base_nv = base_nv+_nvdimm_size; // hint for allocation address of DRAM COMPRESSED HEAP.
370 }
371
372 // If OS doesn't support demand paging for large page memory, we need
373 // to use reserve_memory_special() to reserve and pin the entire region.
374 // If there is a backing file directory for this space then whether
375 // large pages are allocated is up to the filesystem of the backing file.
376 // So we ignore the UseLargePages flag in this case.
377 bool special = large && !os::can_commit_large_page_memory();
378 if (special && _fd_for_heap != -1) {
379 special = false;
380 if (UseLargePages && (!FLAG_IS_DEFAULT(UseLargePages) ||
381 !FLAG_IS_DEFAULT(LargePageSizeInBytes))) {
382 log_debug(gc, heap)("Cannot allocate large pages for Java Heap when AllocateHeapAt option is set.");
383 }
384 }
385 char* base = NULL;
386 char* nvdimm_base = NULL;
387
388 log_trace(gc, heap, coops)("Trying to allocate at address " PTR_FORMAT
389 " heap of size " SIZE_FORMAT_HEX,
390 p2i(requested_address),
391 size);
392
393 if (special) {
394 base = os::reserve_memory_special(size, alignment, requested_address, false);
395
396 if (base != NULL) {
397 // Check alignment constraints.
398 assert((uintptr_t) base % alignment == 0,
399 "Large pages returned a non-aligned address, base: "
400 PTR_FORMAT " alignment: " SIZE_FORMAT_HEX,
401 p2i(base), alignment);
402 _special = true;
403 }
404 }
405
406 if (base == NULL) {
407 // Failed; try to reserve regular memory below
408 if (UseLargePages && (!FLAG_IS_DEFAULT(UseLargePages) ||
409 !FLAG_IS_DEFAULT(LargePageSizeInBytes))) {
410 log_debug(gc, heap, coops)("Reserve regular memory without large pages");
411 }
412
413 // Optimistically assume that the OSes returns an aligned base pointer.
414 // When reserving a large address range, most OSes seem to align to at
415 // least 64K.
416
417 // If the memory was requested at a particular address, use
418 // os::attempt_reserve_memory_at() to avoid over mapping something
419 // important. If available space is not detected, return NULL.
420
421 if (requested_address != 0) {
422 if (_nvdimm_base_nv != NULL && _fd_for_nvdimm != -1) {
423 // first unmap so that OS does not keep trying.
424 os::unmap_memory(_nvdimm_base_nv, _dram_size);
425 base = os::attempt_reserve_memory_at(_dram_size, _nvdimm_base_nv);
426 } else {
427 base = os::attempt_reserve_memory_at(size, requested_address, _fd_for_heap);
428 }
429 } else {
430 if (_nvdimm_base_nv != NULL && _fd_for_nvdimm != -1) {
431 // first unmap so that OS does not keep trying.
432 os::unmap_memory(_nvdimm_base_nv, _dram_size);
433 base = os::reserve_memory(_dram_size, _nvdimm_base_nv, alignment);
434 } else {
435 base = os::reserve_memory(size, NULL, alignment, _fd_for_heap);
436 }
437 }
438 }
439 if (base == NULL) { return; }
440
441 // Done
442 _base = base;
443 _nvdimm_base = _base-_nvdimm_size;
444 if (_nvdimm_base_nv != NULL && _fd_for_nvdimm != -1) {
445 _size = _dram_size;
446 } else {
447 _size = size;
448 }
449 _alignment = alignment;
450
451 // If heap is reserved with a backing file, the entire space has been committed. So set the _special flag to true
452 if (_fd_for_heap != -1) {
453 _special = true;
454 }
455
456 // Check alignment constraints
457 if ((((size_t)base) & (alignment - 1)) != 0) {
458 // Base not aligned, retry.
459 release();
460 }
461 }
462
463 void ReservedHeapSpace::try_reserve_range(char *highest_start,
464 char *lowest_start,
465 size_t attach_point_alignment,
466 char *aligned_heap_base_min_address,
467 char *upper_bound,
468 size_t size,
469 size_t alignment,
470 bool large) {
471 const size_t attach_range = highest_start - lowest_start;
472 // Cap num_attempts at possible number.
473 // At least one is possible even for 0 sized attach range.
474 const uint64_t num_attempts_possible = (attach_range / attach_point_alignment) + 1;
475 const uint64_t num_attempts_to_try = MIN2((uint64_t)HeapSearchSteps, num_attempts_possible);
476
477 const size_t stepsize = (attach_range == 0) ? // Only one try.
478 (size_t) highest_start : align_up(attach_range / num_attempts_to_try, attach_point_alignment);
479
480 // Try attach points from top to bottom.
481 char* attach_point = highest_start;
482 while (attach_point >= lowest_start &&
483 attach_point <= highest_start && // Avoid wrap around.
484 ((_base == NULL) ||
485 (_base < aligned_heap_base_min_address || _base + size > upper_bound))) {
486 try_reserve_heap(size, alignment, large, attach_point);
487 attach_point -= stepsize;
488 }
489 }
490
491 #define SIZE_64K ((uint64_t) UCONST64( 0x10000))
492 #define SIZE_256M ((uint64_t) UCONST64( 0x10000000))
493 #define SIZE_32G ((uint64_t) UCONST64( 0x800000000))
494
495 // Helper for heap allocation. Returns an array with addresses
496 // (OS-specific) which are suited for disjoint base mode. Array is
497 // NULL terminated.
498 static char** get_attach_addresses_for_disjoint_mode() {
499 static uint64_t addresses[] = {
500 2 * SIZE_32G,
501 3 * SIZE_32G,
502 4 * SIZE_32G,
503 8 * SIZE_32G,
504 10 * SIZE_32G,
505 1 * SIZE_64K * SIZE_32G,
506 2 * SIZE_64K * SIZE_32G,
507 3 * SIZE_64K * SIZE_32G,
508 4 * SIZE_64K * SIZE_32G,
509 16 * SIZE_64K * SIZE_32G,
510 32 * SIZE_64K * SIZE_32G,
511 34 * SIZE_64K * SIZE_32G,
512 0
513 };
514
515 // Sort out addresses smaller than HeapBaseMinAddress. This assumes
516 // the array is sorted.
517 uint i = 0;
518 while (addresses[i] != 0 &&
519 (addresses[i] < OopEncodingHeapMax || addresses[i] < HeapBaseMinAddress)) {
520 i++;
521 }
522 uint start = i;
523
524 // Avoid more steps than requested.
525 i = 0;
526 while (addresses[start+i] != 0) {
527 if (i == HeapSearchSteps) {
528 addresses[start+i] = 0;
529 break;
530 }
531 i++;
532 }
533
534 return (char**) &addresses[start];
535 }
536
537 void ReservedHeapSpace::initialize_compressed_heap(const size_t size, size_t alignment, bool large) {
538 guarantee(size + noaccess_prefix_size(alignment) <= OopEncodingHeapMax,
539 "can not allocate compressed oop heap for this size");
540 guarantee(alignment == MAX2(alignment, (size_t)os::vm_page_size()), "alignment too small");
541 assert(HeapBaseMinAddress > 0, "sanity");
542
543 const size_t granularity = os::vm_allocation_granularity();
544 assert((size & (granularity - 1)) == 0,
545 "size not aligned to os::vm_allocation_granularity()");
546 assert((alignment & (granularity - 1)) == 0,
547 "alignment not aligned to os::vm_allocation_granularity()");
548 assert(alignment == 0 || is_power_of_2((intptr_t)alignment),
549 "not a power of 2");
550
551 // The necessary attach point alignment for generated wish addresses.
552 // This is needed to increase the chance of attaching for mmap and shmat.
553 const size_t os_attach_point_alignment =
554 AIX_ONLY(SIZE_256M) // Known shm boundary alignment.
555 NOT_AIX(os::vm_allocation_granularity());
556 const size_t attach_point_alignment = lcm(alignment, os_attach_point_alignment);
557
558 char *aligned_heap_base_min_address = (char *)align_up((void *)HeapBaseMinAddress, alignment);
559 size_t noaccess_prefix = ((aligned_heap_base_min_address + size) > (char*)OopEncodingHeapMax) ?
560 noaccess_prefix_size(alignment) : 0;
561
562 // Attempt to alloc at user-given address.
563 if (!FLAG_IS_DEFAULT(HeapBaseMinAddress)) {
564 try_reserve_heap(size + noaccess_prefix, alignment, large, aligned_heap_base_min_address);
565 if (_base != aligned_heap_base_min_address) { // Enforce this exact address.
566 release();
567 }
568 }
569
570 // Keep heap at HeapBaseMinAddress.
571 if (_base == NULL) {
572
573 // Try to allocate the heap at addresses that allow efficient oop compression.
574 // Different schemes are tried, in order of decreasing optimization potential.
575 //
576 // For this, try_reserve_heap() is called with the desired heap base addresses.
577 // A call into the os layer to allocate at a given address can return memory
578 // at a different address than requested. Still, this might be memory at a useful
579 // address. try_reserve_heap() always returns this allocated memory, as only here
580 // the criteria for a good heap are checked.
581
582 // Attempt to allocate so that we can run without base and scale (32-Bit unscaled compressed oops).
583 // Give it several tries from top of range to bottom.
584 if (aligned_heap_base_min_address + size <= (char *)UnscaledOopHeapMax) {
585
586 // Calc address range within we try to attach (range of possible start addresses).
587 char* const highest_start = align_down((char *)UnscaledOopHeapMax - size, attach_point_alignment);
588 char* const lowest_start = align_up(aligned_heap_base_min_address, attach_point_alignment);
589 try_reserve_range(highest_start, lowest_start, attach_point_alignment,
590 aligned_heap_base_min_address, (char *)UnscaledOopHeapMax, size, alignment, large);
591 }
592
593 // zerobased: Attempt to allocate in the lower 32G.
594 // But leave room for the compressed class pointers, which is allocated above
595 // the heap.
596 char *zerobased_max = (char *)OopEncodingHeapMax;
597 const size_t class_space = align_up(CompressedClassSpaceSize, alignment);
598 // For small heaps, save some space for compressed class pointer
599 // space so it can be decoded with no base.
600 if (UseCompressedClassPointers && !UseSharedSpaces &&
601 OopEncodingHeapMax <= KlassEncodingMetaspaceMax &&
602 (uint64_t)(aligned_heap_base_min_address + size + class_space) <= KlassEncodingMetaspaceMax) {
603 zerobased_max = (char *)OopEncodingHeapMax - class_space;
604 }
605
606 // Give it several tries from top of range to bottom.
607 if (aligned_heap_base_min_address + size <= zerobased_max && // Zerobased theoretical possible.
608 ((_base == NULL) || // No previous try succeeded.
609 (_base + size > zerobased_max))) { // Unscaled delivered an arbitrary address.
610
611 // Calc address range within we try to attach (range of possible start addresses).
612 char *const highest_start = align_down(zerobased_max - size, attach_point_alignment);
613 // Need to be careful about size being guaranteed to be less
614 // than UnscaledOopHeapMax due to type constraints.
615 char *lowest_start = aligned_heap_base_min_address;
616 uint64_t unscaled_end = UnscaledOopHeapMax - size;
617 if (unscaled_end < UnscaledOopHeapMax) { // unscaled_end wrapped if size is large
618 lowest_start = MAX2(lowest_start, (char*)unscaled_end);
619 }
620 lowest_start = align_up(lowest_start, attach_point_alignment);
621 try_reserve_range(highest_start, lowest_start, attach_point_alignment,
622 aligned_heap_base_min_address, zerobased_max, size, alignment, large);
623 }
624
625 // Now we go for heaps with base != 0. We need a noaccess prefix to efficiently
626 // implement null checks.
627 noaccess_prefix = noaccess_prefix_size(alignment);
628
629 // Try to attach at addresses that are aligned to OopEncodingHeapMax. Disjointbase mode.
630 char** addresses = get_attach_addresses_for_disjoint_mode();
631 int i = 0;
632 while (addresses[i] && // End of array not yet reached.
633 ((_base == NULL) || // No previous try succeeded.
634 (_base + size > (char *)OopEncodingHeapMax && // Not zerobased or unscaled address.
635 !Universe::is_disjoint_heap_base_address((address)_base)))) { // Not disjoint address.
636 char* const attach_point = addresses[i];
637 assert(attach_point >= aligned_heap_base_min_address, "Flag support broken");
638 try_reserve_heap(size + noaccess_prefix, alignment, large, attach_point);
639 i++;
640 }
641
642 // Last, desperate try without any placement.
643 if (_base == NULL) {
644 log_trace(gc, heap, coops)("Trying to allocate at address NULL heap of size " SIZE_FORMAT_HEX, size + noaccess_prefix);
645 initialize(size + noaccess_prefix, alignment, large, NULL, false);
646 }
647 }
648 }
649
650 void ReservedHeapSpace::initialize_g1gc_nvdimm_dram_sizes(size_t size, size_t alignment) {
651 _dram_size = (size_t)((size * G1MaxNewSizePercent)/100);
652 size_t page_sz = os::vm_page_size() -1 ;
653 _dram_size = (_dram_size + page_sz) & (~page_sz);
654 // align sizes.
655 _dram_size = align_down(_dram_size, alignment);
656 _nvdimm_size = size - _dram_size;
657 _nvdimm_size = (_nvdimm_size + page_sz) & (~page_sz);
658 _nvdimm_size = align_down(_nvdimm_size, alignment);
659 }
660
661 ReservedHeapSpace::ReservedHeapSpace(size_t size, size_t alignment, bool large, const char* heap_allocation_directory) : ReservedSpace() {
662
663 if (size == 0) {
664 return;
665 }
666
667 // if AllocateOldGen is used
668 if (AllocateOldGenAt != NULL) {
669 _fd_for_nvdimm = os::create_file_for_heap(AllocateOldGenAt);
670 if (_fd_for_nvdimm == -1) {
671 vm_exit_during_initialization(
672 err_msg("Could not create file for Heap at location %s", AllocateOldGenAt));
673 }
674 if (UseParallelOldGC) {
675 // For ParallelOldGC, adaptive sizing picks _old_gen virtual space sizes as needed.
676 // allocate Xmx on NVDIMM as adaptive sizing may put lot of pressure on NVDIMM.
677 os::allocate_file(_fd_for_nvdimm, MaxHeapSize);
678 os::set_nvdimm_fd(_fd_for_nvdimm);
679 os::set_nvdimm_present(true);
680 }
681 } else {
682 _fd_for_nvdimm = -1;
683 }
684
685 if (heap_allocation_directory != NULL) {
686 _fd_for_heap = os::create_file_for_heap(heap_allocation_directory);
687 if (_fd_for_heap == -1) {
688 vm_exit_during_initialization(
689 err_msg("Could not create file for Heap at location %s", heap_allocation_directory));
690 }
691 }
692
693 // Heap size should be aligned to alignment, too.
694 guarantee(is_aligned(size, alignment), "set by caller");
695
696 char* base_nv = NULL;
697 _nvdimm_base_nv = NULL;
698
699 if (_fd_for_nvdimm != -1 && UseG1GC) {
700 if (!UseCompressedOops) {
701 // if compressed oops use requested address.
702 initialize_g1gc_nvdimm_dram_sizes(size, alignment);
703 base_nv = os::reserve_memory(size, NULL, alignment);
704 _nvdimm_base_nv = base_nv+_nvdimm_size; // hint for allocation address of DRAM heap
705 }
706 }
707
708 if (UseCompressedOops) {
709 initialize_compressed_heap(size, alignment, large);
710 if (_size > size) {
711 // We allocated heap with noaccess prefix.
712 // It can happen we get a zerobased/unscaled heap with noaccess prefix,
713 // if we had to try at arbitrary address.
714 establish_noaccess_prefix();
715 }
716 } else {
717 if (_fd_for_nvdimm != -1 && UseG1GC) {
718 initialize(_dram_size, alignment, large, NULL, false);
719 } else {
720 initialize(size, alignment, large, NULL, false);
721 }
722 }
723
724 assert(markOopDesc::encode_pointer_as_mark(_base)->decode_pointer() == _base,
725 "area must be distinguishable from marks for mark-sweep");
726 assert(markOopDesc::encode_pointer_as_mark(&_base[size])->decode_pointer() == &_base[size],
727 "area must be distinguishable from marks for mark-sweep");
728
729 if (base() != NULL) {
730 MemTracker::record_virtual_memory_type((address)base(), mtJavaHeap);
731 if (_fd_for_nvdimm != -1 && UseG1GC) {
732 os::set_nvdimm_present(true);
733 os::set_dram_heapbase((address)_base);
734 os::set_nvdimm_heapbase((address)_nvdimm_base);
735 os::set_nvdimm_fd(_fd_for_nvdimm);
736 _size += _nvdimm_size;
737 _base = _nvdimm_base;
738 log_info(gc, heap)("Java DRAM Heap at [%p - %p] & NVDIMM Old Gen at [%p - %p] %ld \n", _nvdimm_base+_nvdimm_size, (char*)(_nvdimm_base+_nvdimm_size+_dram_size), _nvdimm_base, (char*)(_nvdimm_base+_nvdimm_size), size);
739 }
740 }
741
742 if (_fd_for_heap != -1) {
743 os::close(_fd_for_heap);
744 }
745 }
746
747 // Reserve space for code segment. Same as Java heap only we mark this as
748 // executable.
749 ReservedCodeSpace::ReservedCodeSpace(size_t r_size,
750 size_t rs_align,
751 bool large) :
752 ReservedSpace(r_size, rs_align, large, /*executable*/ true) {
753 MemTracker::record_virtual_memory_type((address)base(), mtCode);
754 }
755
756 // VirtualSpace
757
758 VirtualSpace::VirtualSpace() {
759 _low_boundary = NULL;
760 _high_boundary = NULL;
761 _low = NULL;
762 _high = NULL;
763 _lower_high = NULL;
764 _middle_high = NULL;
765 _upper_high = NULL;
766 _lower_high_boundary = NULL;
767 _middle_high_boundary = NULL;
768 _upper_high_boundary = NULL;
769 _lower_alignment = 0;
770 _middle_alignment = 0;
771 _upper_alignment = 0;
772 _special = false;
773 _executable = false;
774 }
775
776
777 bool VirtualSpace::initialize(ReservedSpace rs, size_t committed_size) {
778 const size_t max_commit_granularity = os::page_size_for_region_unaligned(rs.size(), 1);
779 return initialize_with_granularity(rs, committed_size, max_commit_granularity);
780 }
781
782 bool VirtualSpace::initialize_with_granularity(ReservedSpace rs, size_t committed_size, size_t max_commit_granularity) {
783 if(!rs.is_reserved()) return false; // allocation failed.
784 assert(_low_boundary == NULL, "VirtualSpace already initialized");
785 assert(max_commit_granularity > 0, "Granularity must be non-zero.");
786
787 _low_boundary = rs.base();
788 _high_boundary = low_boundary() + rs.size();
789
790 _low = low_boundary();
791 _high = low();
792
793 _special = rs.special();
794 _executable = rs.executable();
795
796 // When a VirtualSpace begins life at a large size, make all future expansion
797 // and shrinking occur aligned to a granularity of large pages. This avoids
798 // fragmentation of physical addresses that inhibits the use of large pages
799 // by the OS virtual memory system. Empirically, we see that with a 4MB
800 // page size, the only spaces that get handled this way are codecache and
801 // the heap itself, both of which provide a substantial performance
802 // boost in many benchmarks when covered by large pages.
803 //
804 // No attempt is made to force large page alignment at the very top and
805 // bottom of the space if they are not aligned so already.
806 _lower_alignment = os::vm_page_size();
807 _middle_alignment = max_commit_granularity;
808 _upper_alignment = os::vm_page_size();
809
810 // End of each region
811 _lower_high_boundary = align_up(low_boundary(), middle_alignment());
812 _middle_high_boundary = align_down(high_boundary(), middle_alignment());
813 _upper_high_boundary = high_boundary();
814
815 // High address of each region
816 _lower_high = low_boundary();
817 _middle_high = lower_high_boundary();
818 _upper_high = middle_high_boundary();
819
820 // commit to initial size
821 if (committed_size > 0) {
822 if (!expand_by(committed_size)) {
823 return false;
824 }
825 }
826 return true;
827 }
828
829
830 VirtualSpace::~VirtualSpace() {
831 release();
832 }
833
834
835 void VirtualSpace::release() {
836 // This does not release memory it reserved.
837 // Caller must release via rs.release();
838 _low_boundary = NULL;
839 _high_boundary = NULL;
840 _low = NULL;
841 _high = NULL;
842 _lower_high = NULL;
843 _middle_high = NULL;
844 _upper_high = NULL;
845 _lower_high_boundary = NULL;
846 _middle_high_boundary = NULL;
847 _upper_high_boundary = NULL;
848 _lower_alignment = 0;
849 _middle_alignment = 0;
850 _upper_alignment = 0;
851 _special = false;
852 _executable = false;
853 }
854
855
856 size_t VirtualSpace::committed_size() const {
857 return pointer_delta(high(), low(), sizeof(char));
858 }
859
860
861 size_t VirtualSpace::reserved_size() const {
862 return pointer_delta(high_boundary(), low_boundary(), sizeof(char));
863 }
864
865
866 size_t VirtualSpace::uncommitted_size() const {
867 return reserved_size() - committed_size();
868 }
869
870 size_t VirtualSpace::actual_committed_size() const {
871 // Special VirtualSpaces commit all reserved space up front.
872 if (special()) {
873 return reserved_size();
874 }
875
876 size_t committed_low = pointer_delta(_lower_high, _low_boundary, sizeof(char));
877 size_t committed_middle = pointer_delta(_middle_high, _lower_high_boundary, sizeof(char));
878 size_t committed_high = pointer_delta(_upper_high, _middle_high_boundary, sizeof(char));
879
880 #ifdef ASSERT
881 size_t lower = pointer_delta(_lower_high_boundary, _low_boundary, sizeof(char));
882 size_t middle = pointer_delta(_middle_high_boundary, _lower_high_boundary, sizeof(char));
883 size_t upper = pointer_delta(_upper_high_boundary, _middle_high_boundary, sizeof(char));
884
885 if (committed_high > 0) {
886 assert(committed_low == lower, "Must be");
887 assert(committed_middle == middle, "Must be");
888 }
889
890 if (committed_middle > 0) {
891 assert(committed_low == lower, "Must be");
892 }
893 if (committed_middle < middle) {
894 assert(committed_high == 0, "Must be");
895 }
896
897 if (committed_low < lower) {
898 assert(committed_high == 0, "Must be");
899 assert(committed_middle == 0, "Must be");
900 }
901 #endif
902
903 return committed_low + committed_middle + committed_high;
904 }
905
906
907 bool VirtualSpace::contains(const void* p) const {
908 return low() <= (const char*) p && (const char*) p < high();
909 }
910
911 static void pretouch_expanded_memory(void* start, void* end) {
912 assert(is_aligned(start, os::vm_page_size()), "Unexpected alignment");
913 assert(is_aligned(end, os::vm_page_size()), "Unexpected alignment");
914
915 os::pretouch_memory(start, end);
916 }
917
918 static bool commit_expanded(char* start, size_t size, size_t alignment, bool pre_touch, bool executable) {
919 if (os::commit_memory(start, size, alignment, executable)) {
920 if (pre_touch || AlwaysPreTouch) {
921 pretouch_expanded_memory(start, start + size);
922 }
923 return true;
924 }
925
926 debug_only(warning(
927 "INFO: os::commit_memory(" PTR_FORMAT ", " PTR_FORMAT
928 " size=" SIZE_FORMAT ", executable=%d) failed",
929 p2i(start), p2i(start + size), size, executable);)
930
931 return false;
932 }
933
934 /*
935 First we need to determine if a particular virtual space is using large
936 pages. This is done at the initialize function and only virtual spaces
937 that are larger than LargePageSizeInBytes use large pages. Once we
938 have determined this, all expand_by and shrink_by calls must grow and
939 shrink by large page size chunks. If a particular request
940 is within the current large page, the call to commit and uncommit memory
941 can be ignored. In the case that the low and high boundaries of this
942 space is not large page aligned, the pages leading to the first large
943 page address and the pages after the last large page address must be
944 allocated with default pages.
945 */
946 bool VirtualSpace::expand_by(size_t bytes, bool pre_touch) {
947 if (uncommitted_size() < bytes) {
948 return false;
949 }
950
951 if (special()) {
952 // don't commit memory if the entire space is pinned in memory
953 _high += bytes;
954 return true;
955 }
956
957 char* previous_high = high();
958 char* unaligned_new_high = high() + bytes;
959 assert(unaligned_new_high <= high_boundary(), "cannot expand by more than upper boundary");
960
961 // Calculate where the new high for each of the regions should be. If
962 // the low_boundary() and high_boundary() are LargePageSizeInBytes aligned
963 // then the unaligned lower and upper new highs would be the
964 // lower_high() and upper_high() respectively.
965 char* unaligned_lower_new_high = MIN2(unaligned_new_high, lower_high_boundary());
966 char* unaligned_middle_new_high = MIN2(unaligned_new_high, middle_high_boundary());
967 char* unaligned_upper_new_high = MIN2(unaligned_new_high, upper_high_boundary());
968
969 // Align the new highs based on the regions alignment. lower and upper
970 // alignment will always be default page size. middle alignment will be
971 // LargePageSizeInBytes if the actual size of the virtual space is in
972 // fact larger than LargePageSizeInBytes.
973 char* aligned_lower_new_high = align_up(unaligned_lower_new_high, lower_alignment());
974 char* aligned_middle_new_high = align_up(unaligned_middle_new_high, middle_alignment());
975 char* aligned_upper_new_high = align_up(unaligned_upper_new_high, upper_alignment());
976
977 // Determine which regions need to grow in this expand_by call.
978 // If you are growing in the lower region, high() must be in that
979 // region so calculate the size based on high(). For the middle and
980 // upper regions, determine the starting point of growth based on the
981 // location of high(). By getting the MAX of the region's low address
982 // (or the previous region's high address) and high(), we can tell if it
983 // is an intra or inter region growth.
984 size_t lower_needs = 0;
985 if (aligned_lower_new_high > lower_high()) {
986 lower_needs = pointer_delta(aligned_lower_new_high, lower_high(), sizeof(char));
987 }
988 size_t middle_needs = 0;
989 if (aligned_middle_new_high > middle_high()) {
990 middle_needs = pointer_delta(aligned_middle_new_high, middle_high(), sizeof(char));
991 }
992 size_t upper_needs = 0;
993 if (aligned_upper_new_high > upper_high()) {
994 upper_needs = pointer_delta(aligned_upper_new_high, upper_high(), sizeof(char));
995 }
996
997 // Check contiguity.
998 assert(low_boundary() <= lower_high() && lower_high() <= lower_high_boundary(),
999 "high address must be contained within the region");
1000 assert(lower_high_boundary() <= middle_high() && middle_high() <= middle_high_boundary(),
1001 "high address must be contained within the region");
1002 assert(middle_high_boundary() <= upper_high() && upper_high() <= upper_high_boundary(),
1003 "high address must be contained within the region");
1004
1005 // Commit regions
1006 if (lower_needs > 0) {
1007 assert(lower_high() + lower_needs <= lower_high_boundary(), "must not expand beyond region");
1008 if (!commit_expanded(lower_high(), lower_needs, _lower_alignment, pre_touch, _executable)) {
1009 return false;
1010 }
1011 _lower_high += lower_needs;
1012 }
1013
1014 if (middle_needs > 0) {
1015 assert(middle_high() + middle_needs <= middle_high_boundary(), "must not expand beyond region");
1016 if (!commit_expanded(middle_high(), middle_needs, _middle_alignment, pre_touch, _executable)) {
1017 return false;
1018 }
1019 _middle_high += middle_needs;
1020 }
1021
1022 if (upper_needs > 0) {
1023 assert(upper_high() + upper_needs <= upper_high_boundary(), "must not expand beyond region");
1024 if (!commit_expanded(upper_high(), upper_needs, _upper_alignment, pre_touch, _executable)) {
1025 return false;
1026 }
1027 _upper_high += upper_needs;
1028 }
1029
1030 _high += bytes;
1031 return true;
1032 }
1033
1034 // A page is uncommitted if the contents of the entire page is deemed unusable.
1035 // Continue to decrement the high() pointer until it reaches a page boundary
1036 // in which case that particular page can now be uncommitted.
1037 void VirtualSpace::shrink_by(size_t size) {
1038 if (committed_size() < size)
1039 fatal("Cannot shrink virtual space to negative size");
1040
1041 if (special()) {
1042 // don't uncommit if the entire space is pinned in memory
1043 _high -= size;
1044 return;
1045 }
1046
1047 char* unaligned_new_high = high() - size;
1048 assert(unaligned_new_high >= low_boundary(), "cannot shrink past lower boundary");
1049
1050 // Calculate new unaligned address
1051 char* unaligned_upper_new_high =
1052 MAX2(unaligned_new_high, middle_high_boundary());
1053 char* unaligned_middle_new_high =
1054 MAX2(unaligned_new_high, lower_high_boundary());
1055 char* unaligned_lower_new_high =
1056 MAX2(unaligned_new_high, low_boundary());
1057
1058 // Align address to region's alignment
1059 char* aligned_upper_new_high = align_up(unaligned_upper_new_high, upper_alignment());
1060 char* aligned_middle_new_high = align_up(unaligned_middle_new_high, middle_alignment());
1061 char* aligned_lower_new_high = align_up(unaligned_lower_new_high, lower_alignment());
1062
1063 // Determine which regions need to shrink
1064 size_t upper_needs = 0;
1065 if (aligned_upper_new_high < upper_high()) {
1066 upper_needs =
1067 pointer_delta(upper_high(), aligned_upper_new_high, sizeof(char));
1068 }
1069 size_t middle_needs = 0;
1070 if (aligned_middle_new_high < middle_high()) {
1071 middle_needs =
1072 pointer_delta(middle_high(), aligned_middle_new_high, sizeof(char));
1073 }
1074 size_t lower_needs = 0;
1075 if (aligned_lower_new_high < lower_high()) {
1076 lower_needs =
1077 pointer_delta(lower_high(), aligned_lower_new_high, sizeof(char));
1078 }
1079
1080 // Check contiguity.
1081 assert(middle_high_boundary() <= upper_high() &&
1082 upper_high() <= upper_high_boundary(),
1083 "high address must be contained within the region");
1084 assert(lower_high_boundary() <= middle_high() &&
1085 middle_high() <= middle_high_boundary(),
1086 "high address must be contained within the region");
1087 assert(low_boundary() <= lower_high() &&
1088 lower_high() <= lower_high_boundary(),
1089 "high address must be contained within the region");
1090
1091 // Uncommit
1092 if (upper_needs > 0) {
1093 assert(middle_high_boundary() <= aligned_upper_new_high &&
1094 aligned_upper_new_high + upper_needs <= upper_high_boundary(),
1095 "must not shrink beyond region");
1096 if (!os::uncommit_memory(aligned_upper_new_high, upper_needs)) {
1097 debug_only(warning("os::uncommit_memory failed"));
1098 return;
1099 } else {
1100 _upper_high -= upper_needs;
1101 }
1102 }
1103 if (middle_needs > 0) {
1104 assert(lower_high_boundary() <= aligned_middle_new_high &&
1105 aligned_middle_new_high + middle_needs <= middle_high_boundary(),
1106 "must not shrink beyond region");
1107 if (!os::uncommit_memory(aligned_middle_new_high, middle_needs)) {
1108 debug_only(warning("os::uncommit_memory failed"));
1109 return;
1110 } else {
1111 _middle_high -= middle_needs;
1112 }
1113 }
1114 if (lower_needs > 0) {
1115 assert(low_boundary() <= aligned_lower_new_high &&
1116 aligned_lower_new_high + lower_needs <= lower_high_boundary(),
1117 "must not shrink beyond region");
1118 if (!os::uncommit_memory(aligned_lower_new_high, lower_needs)) {
1119 debug_only(warning("os::uncommit_memory failed"));
1120 return;
1121 } else {
1122 _lower_high -= lower_needs;
1123 }
1124 }
1125
1126 _high -= size;
1127 }
1128
1129 #ifndef PRODUCT
1130 void VirtualSpace::check_for_contiguity() {
1131 // Check contiguity.
1132 assert(low_boundary() <= lower_high() &&
1133 lower_high() <= lower_high_boundary(),
1134 "high address must be contained within the region");
1135 assert(lower_high_boundary() <= middle_high() &&
1136 middle_high() <= middle_high_boundary(),
1137 "high address must be contained within the region");
1138 assert(middle_high_boundary() <= upper_high() &&
1139 upper_high() <= upper_high_boundary(),
1140 "high address must be contained within the region");
1141 assert(low() >= low_boundary(), "low");
1142 assert(low_boundary() <= lower_high_boundary(), "lower high boundary");
1143 assert(upper_high_boundary() <= high_boundary(), "upper high boundary");
1144 assert(high() <= upper_high(), "upper high");
1145 }
1146
1147 void VirtualSpace::print_on(outputStream* out) {
1148 out->print ("Virtual space:");
1149 if (special()) out->print(" (pinned in memory)");
1150 out->cr();
1151 out->print_cr(" - committed: " SIZE_FORMAT, committed_size());
1152 out->print_cr(" - reserved: " SIZE_FORMAT, reserved_size());
1153 out->print_cr(" - [low, high]: [" INTPTR_FORMAT ", " INTPTR_FORMAT "]", p2i(low()), p2i(high()));
1154 out->print_cr(" - [low_b, high_b]: [" INTPTR_FORMAT ", " INTPTR_FORMAT "]", p2i(low_boundary()), p2i(high_boundary()));
1155 }
1156
1157 void VirtualSpace::print() {
1158 print_on(tty);
1159 }
1160
1161 /////////////// Unit tests ///////////////
1162
1163 #ifndef PRODUCT
1164
1165 #define test_log(...) \
1166 do {\
1167 if (VerboseInternalVMTests) { \
1168 tty->print_cr(__VA_ARGS__); \
1169 tty->flush(); \
1170 }\
1171 } while (false)
1172
1173 class TestReservedSpace : AllStatic {
1174 public:
1175 static void small_page_write(void* addr, size_t size) {
1176 size_t page_size = os::vm_page_size();
1177
1178 char* end = (char*)addr + size;
1179 for (char* p = (char*)addr; p < end; p += page_size) {
1180 *p = 1;
1181 }
1182 }
1183
1184 static void release_memory_for_test(ReservedSpace rs) {
1185 if (rs.special()) {
1186 guarantee(os::release_memory_special(rs.base(), rs.size()), "Shouldn't fail");
1187 } else {
1188 guarantee(os::release_memory(rs.base(), rs.size()), "Shouldn't fail");
1189 }
1190 }
1191
1192 static void test_reserved_space1(size_t size, size_t alignment) {
1193 test_log("test_reserved_space1(%p)", (void*) (uintptr_t) size);
1194
1195 assert(is_aligned(size, alignment), "Incorrect input parameters");
1196
1197 ReservedSpace rs(size, // size
1198 alignment, // alignment
1199 UseLargePages, // large
1200 (char *)NULL); // requested_address
1201
1202 test_log(" rs.special() == %d", rs.special());
1203
1204 assert(rs.base() != NULL, "Must be");
1205 assert(rs.size() == size, "Must be");
1206
1207 assert(is_aligned(rs.base(), alignment), "aligned sizes should always give aligned addresses");
1208 assert(is_aligned(rs.size(), alignment), "aligned sizes should always give aligned addresses");
1209
1210 if (rs.special()) {
1211 small_page_write(rs.base(), size);
1212 }
1213
1214 release_memory_for_test(rs);
1215 }
1216
1217 static void test_reserved_space2(size_t size) {
1218 test_log("test_reserved_space2(%p)", (void*)(uintptr_t)size);
1219
1220 assert(is_aligned(size, os::vm_allocation_granularity()), "Must be at least AG aligned");
1221
1222 ReservedSpace rs(size);
1223
1224 test_log(" rs.special() == %d", rs.special());
1225
1226 assert(rs.base() != NULL, "Must be");
1227 assert(rs.size() == size, "Must be");
1228
1229 if (rs.special()) {
1230 small_page_write(rs.base(), size);
1231 }
1232
1233 release_memory_for_test(rs);
1234 }
1235
1236 static void test_reserved_space3(size_t size, size_t alignment, bool maybe_large) {
1237 test_log("test_reserved_space3(%p, %p, %d)",
1238 (void*)(uintptr_t)size, (void*)(uintptr_t)alignment, maybe_large);
1239
1240 if (size < alignment) {
1241 // Tests might set -XX:LargePageSizeInBytes=<small pages> and cause unexpected input arguments for this test.
1242 assert((size_t)os::vm_page_size() == os::large_page_size(), "Test needs further refinement");
1243 return;
1244 }
1245
1246 assert(is_aligned(size, os::vm_allocation_granularity()), "Must be at least AG aligned");
1247 assert(is_aligned(size, alignment), "Must be at least aligned against alignment");
1248
1249 bool large = maybe_large && UseLargePages && size >= os::large_page_size();
1250
1251 ReservedSpace rs(size, alignment, large, false);
1252
1253 test_log(" rs.special() == %d", rs.special());
1254
1255 assert(rs.base() != NULL, "Must be");
1256 assert(rs.size() == size, "Must be");
1257
1258 if (rs.special()) {
1259 small_page_write(rs.base(), size);
1260 }
1261
1262 release_memory_for_test(rs);
1263 }
1264
1265
1266 static void test_reserved_space1() {
1267 size_t size = 2 * 1024 * 1024;
1268 size_t ag = os::vm_allocation_granularity();
1269
1270 test_reserved_space1(size, ag);
1271 test_reserved_space1(size * 2, ag);
1272 test_reserved_space1(size * 10, ag);
1273 }
1274
1275 static void test_reserved_space2() {
1276 size_t size = 2 * 1024 * 1024;
1277 size_t ag = os::vm_allocation_granularity();
1278
1279 test_reserved_space2(size * 1);
1280 test_reserved_space2(size * 2);
1281 test_reserved_space2(size * 10);
1282 test_reserved_space2(ag);
1283 test_reserved_space2(size - ag);
1284 test_reserved_space2(size);
1285 test_reserved_space2(size + ag);
1286 test_reserved_space2(size * 2);
1287 test_reserved_space2(size * 2 - ag);
1288 test_reserved_space2(size * 2 + ag);
1289 test_reserved_space2(size * 3);
1290 test_reserved_space2(size * 3 - ag);
1291 test_reserved_space2(size * 3 + ag);
1292 test_reserved_space2(size * 10);
1293 test_reserved_space2(size * 10 + size / 2);
1294 }
1295
1296 static void test_reserved_space3() {
1297 size_t ag = os::vm_allocation_granularity();
1298
1299 test_reserved_space3(ag, ag , false);
1300 test_reserved_space3(ag * 2, ag , false);
1301 test_reserved_space3(ag * 3, ag , false);
1302 test_reserved_space3(ag * 2, ag * 2, false);
1303 test_reserved_space3(ag * 4, ag * 2, false);
1304 test_reserved_space3(ag * 8, ag * 2, false);
1305 test_reserved_space3(ag * 4, ag * 4, false);
1306 test_reserved_space3(ag * 8, ag * 4, false);
1307 test_reserved_space3(ag * 16, ag * 4, false);
1308
1309 if (UseLargePages) {
1310 size_t lp = os::large_page_size();
1311
1312 // Without large pages
1313 test_reserved_space3(lp, ag * 4, false);
1314 test_reserved_space3(lp * 2, ag * 4, false);
1315 test_reserved_space3(lp * 4, ag * 4, false);
1316 test_reserved_space3(lp, lp , false);
1317 test_reserved_space3(lp * 2, lp , false);
1318 test_reserved_space3(lp * 3, lp , false);
1319 test_reserved_space3(lp * 2, lp * 2, false);
1320 test_reserved_space3(lp * 4, lp * 2, false);
1321 test_reserved_space3(lp * 8, lp * 2, false);
1322
1323 // With large pages
1324 test_reserved_space3(lp, ag * 4 , true);
1325 test_reserved_space3(lp * 2, ag * 4, true);
1326 test_reserved_space3(lp * 4, ag * 4, true);
1327 test_reserved_space3(lp, lp , true);
1328 test_reserved_space3(lp * 2, lp , true);
1329 test_reserved_space3(lp * 3, lp , true);
1330 test_reserved_space3(lp * 2, lp * 2, true);
1331 test_reserved_space3(lp * 4, lp * 2, true);
1332 test_reserved_space3(lp * 8, lp * 2, true);
1333 }
1334 }
1335
1336 static void test_reserved_space() {
1337 test_reserved_space1();
1338 test_reserved_space2();
1339 test_reserved_space3();
1340 }
1341 };
1342
1343 void TestReservedSpace_test() {
1344 TestReservedSpace::test_reserved_space();
1345 }
1346
1347 #define assert_equals(actual, expected) \
1348 assert(actual == expected, \
1349 "Got " SIZE_FORMAT " expected " \
1350 SIZE_FORMAT, actual, expected);
1351
1352 #define assert_ge(value1, value2) \
1353 assert(value1 >= value2, \
1354 "'" #value1 "': " SIZE_FORMAT " '" \
1355 #value2 "': " SIZE_FORMAT, value1, value2);
1356
1357 #define assert_lt(value1, value2) \
1358 assert(value1 < value2, \
1359 "'" #value1 "': " SIZE_FORMAT " '" \
1360 #value2 "': " SIZE_FORMAT, value1, value2);
1361
1362
1363 class TestVirtualSpace : AllStatic {
1364 enum TestLargePages {
1365 Default,
1366 Disable,
1367 Reserve,
1368 Commit
1369 };
1370
1371 static ReservedSpace reserve_memory(size_t reserve_size_aligned, TestLargePages mode) {
1372 switch(mode) {
1373 default:
1374 case Default:
1375 case Reserve:
1376 return ReservedSpace(reserve_size_aligned);
1377 case Disable:
1378 case Commit:
1379 return ReservedSpace(reserve_size_aligned,
1380 os::vm_allocation_granularity(),
1381 /* large */ false, /* exec */ false);
1382 }
1383 }
1384
1385 static bool initialize_virtual_space(VirtualSpace& vs, ReservedSpace rs, TestLargePages mode) {
1386 switch(mode) {
1387 default:
1388 case Default:
1389 case Reserve:
1390 return vs.initialize(rs, 0);
1391 case Disable:
1392 return vs.initialize_with_granularity(rs, 0, os::vm_page_size());
1393 case Commit:
1394 return vs.initialize_with_granularity(rs, 0, os::page_size_for_region_unaligned(rs.size(), 1));
1395 }
1396 }
1397
1398 public:
1399 static void test_virtual_space_actual_committed_space(size_t reserve_size, size_t commit_size,
1400 TestLargePages mode = Default) {
1401 size_t granularity = os::vm_allocation_granularity();
1402 size_t reserve_size_aligned = align_up(reserve_size, granularity);
1403
1404 ReservedSpace reserved = reserve_memory(reserve_size_aligned, mode);
1405
1406 assert(reserved.is_reserved(), "Must be");
1407
1408 VirtualSpace vs;
1409 bool initialized = initialize_virtual_space(vs, reserved, mode);
1410 assert(initialized, "Failed to initialize VirtualSpace");
1411
1412 vs.expand_by(commit_size, false);
1413
1414 if (vs.special()) {
1415 assert_equals(vs.actual_committed_size(), reserve_size_aligned);
1416 } else {
1417 assert_ge(vs.actual_committed_size(), commit_size);
1418 // Approximate the commit granularity.
1419 // Make sure that we don't commit using large pages
1420 // if large pages has been disabled for this VirtualSpace.
1421 size_t commit_granularity = (mode == Disable || !UseLargePages) ?
1422 os::vm_page_size() : os::large_page_size();
1423 assert_lt(vs.actual_committed_size(), commit_size + commit_granularity);
1424 }
1425
1426 reserved.release();
1427 }
1428
1429 static void test_virtual_space_actual_committed_space_one_large_page() {
1430 if (!UseLargePages) {
1431 return;
1432 }
1433
1434 size_t large_page_size = os::large_page_size();
1435
1436 ReservedSpace reserved(large_page_size, large_page_size, true, false);
1437
1438 assert(reserved.is_reserved(), "Must be");
1439
1440 VirtualSpace vs;
1441 bool initialized = vs.initialize(reserved, 0);
1442 assert(initialized, "Failed to initialize VirtualSpace");
1443
1444 vs.expand_by(large_page_size, false);
1445
1446 assert_equals(vs.actual_committed_size(), large_page_size);
1447
1448 reserved.release();
1449 }
1450
1451 static void test_virtual_space_actual_committed_space() {
1452 test_virtual_space_actual_committed_space(4 * K, 0);
1453 test_virtual_space_actual_committed_space(4 * K, 4 * K);
1454 test_virtual_space_actual_committed_space(8 * K, 0);
1455 test_virtual_space_actual_committed_space(8 * K, 4 * K);
1456 test_virtual_space_actual_committed_space(8 * K, 8 * K);
1457 test_virtual_space_actual_committed_space(12 * K, 0);
1458 test_virtual_space_actual_committed_space(12 * K, 4 * K);
1459 test_virtual_space_actual_committed_space(12 * K, 8 * K);
1460 test_virtual_space_actual_committed_space(12 * K, 12 * K);
1461 test_virtual_space_actual_committed_space(64 * K, 0);
1462 test_virtual_space_actual_committed_space(64 * K, 32 * K);
1463 test_virtual_space_actual_committed_space(64 * K, 64 * K);
1464 test_virtual_space_actual_committed_space(2 * M, 0);
1465 test_virtual_space_actual_committed_space(2 * M, 4 * K);
1466 test_virtual_space_actual_committed_space(2 * M, 64 * K);
1467 test_virtual_space_actual_committed_space(2 * M, 1 * M);
1468 test_virtual_space_actual_committed_space(2 * M, 2 * M);
1469 test_virtual_space_actual_committed_space(10 * M, 0);
1470 test_virtual_space_actual_committed_space(10 * M, 4 * K);
1471 test_virtual_space_actual_committed_space(10 * M, 8 * K);
1472 test_virtual_space_actual_committed_space(10 * M, 1 * M);
1473 test_virtual_space_actual_committed_space(10 * M, 2 * M);
1474 test_virtual_space_actual_committed_space(10 * M, 5 * M);
1475 test_virtual_space_actual_committed_space(10 * M, 10 * M);
1476 }
1477
1478 static void test_virtual_space_disable_large_pages() {
1479 if (!UseLargePages) {
1480 return;
1481 }
1482 // These test cases verify that if we force VirtualSpace to disable large pages
1483 test_virtual_space_actual_committed_space(10 * M, 0, Disable);
1484 test_virtual_space_actual_committed_space(10 * M, 4 * K, Disable);
1485 test_virtual_space_actual_committed_space(10 * M, 8 * K, Disable);
1486 test_virtual_space_actual_committed_space(10 * M, 1 * M, Disable);
1487 test_virtual_space_actual_committed_space(10 * M, 2 * M, Disable);
1488 test_virtual_space_actual_committed_space(10 * M, 5 * M, Disable);
1489 test_virtual_space_actual_committed_space(10 * M, 10 * M, Disable);
1490
1491 test_virtual_space_actual_committed_space(10 * M, 0, Reserve);
1492 test_virtual_space_actual_committed_space(10 * M, 4 * K, Reserve);
1493 test_virtual_space_actual_committed_space(10 * M, 8 * K, Reserve);
1494 test_virtual_space_actual_committed_space(10 * M, 1 * M, Reserve);
1495 test_virtual_space_actual_committed_space(10 * M, 2 * M, Reserve);
1496 test_virtual_space_actual_committed_space(10 * M, 5 * M, Reserve);
1497 test_virtual_space_actual_committed_space(10 * M, 10 * M, Reserve);
1498
1499 test_virtual_space_actual_committed_space(10 * M, 0, Commit);
1500 test_virtual_space_actual_committed_space(10 * M, 4 * K, Commit);
1501 test_virtual_space_actual_committed_space(10 * M, 8 * K, Commit);
1502 test_virtual_space_actual_committed_space(10 * M, 1 * M, Commit);
1503 test_virtual_space_actual_committed_space(10 * M, 2 * M, Commit);
1504 test_virtual_space_actual_committed_space(10 * M, 5 * M, Commit);
1505 test_virtual_space_actual_committed_space(10 * M, 10 * M, Commit);
1506 }
1507
1508 static void test_virtual_space() {
1509 test_virtual_space_actual_committed_space();
1510 test_virtual_space_actual_committed_space_one_large_page();
1511 test_virtual_space_disable_large_pages();
1512 }
1513 };
1514
1515 void TestVirtualSpace_test() {
1516 TestVirtualSpace::test_virtual_space();
1517 }
1518
1519 #endif // PRODUCT
1520
1521 #endif