1 /*
2 * Copyright (c) 1997, 2014, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "oops/markOop.hpp"
27 #include "oops/oop.inline.hpp"
28 #include "runtime/virtualspace.hpp"
29 #include "services/memTracker.hpp"
30 #ifdef TARGET_OS_FAMILY_linux
31 # include "os_linux.inline.hpp"
32 #endif
33 #ifdef TARGET_OS_FAMILY_solaris
34 # include "os_solaris.inline.hpp"
35 #endif
36 #ifdef TARGET_OS_FAMILY_windows
37 # include "os_windows.inline.hpp"
38 #endif
39 #ifdef TARGET_OS_FAMILY_aix
40 # include "os_aix.inline.hpp"
41 #endif
42 #ifdef TARGET_OS_FAMILY_bsd
43 # include "os_bsd.inline.hpp"
44 #endif
45
46 PRAGMA_FORMAT_MUTE_WARNINGS_FOR_GCC
47
48 // ReservedSpace
49
50 // Dummy constructor
51 ReservedSpace::ReservedSpace() : _base(NULL), _size(0), _noaccess_prefix(0),
52 _alignment(0), _special(false), _executable(false) {
53 }
54
55 ReservedSpace::ReservedSpace(size_t size) {
56 // Want to use large pages where possible and pad with small pages.
57 size_t page_size = os::page_size_for_region_unaligned(size, 1);
58 bool large_pages = page_size != (size_t)os::vm_page_size();
59 // Don't force the alignment to be large page aligned,
60 // since that will waste memory.
61 size_t alignment = os::vm_allocation_granularity();
62 initialize(size, alignment, large_pages, NULL, 0, false);
63 }
64
65 ReservedSpace::ReservedSpace(size_t size, size_t alignment,
66 bool large,
67 char* requested_address,
68 const size_t noaccess_prefix) {
69 initialize(size+noaccess_prefix, alignment, large, requested_address,
70 noaccess_prefix, false);
71 }
72
73 ReservedSpace::ReservedSpace(size_t size, size_t alignment,
74 bool large,
75 bool executable) {
76 initialize(size, alignment, large, NULL, 0, executable);
77 }
78
79 // Helper method.
80 static bool failed_to_reserve_as_requested(char* base, char* requested_address,
81 const size_t size, bool special)
82 {
83 if (base == requested_address || requested_address == NULL)
84 return false; // did not fail
85
86 if (base != NULL) {
87 // Different reserve address may be acceptable in other cases
88 // but for compressed oops heap should be at requested address.
89 assert(UseCompressedOops, "currently requested address used only for compressed oops");
90 if (PrintCompressedOopsMode) {
91 tty->cr();
92 tty->print_cr("Reserved memory not at requested address: " PTR_FORMAT " vs " PTR_FORMAT, base, requested_address);
93 }
94 // OS ignored requested address. Try different address.
95 if (special) {
96 if (!os::release_memory_special(base, size)) {
97 fatal("os::release_memory_special failed");
98 }
99 } else {
100 if (!os::release_memory(base, size)) {
101 fatal("os::release_memory failed");
102 }
103 }
104 }
105 return true;
106 }
107
108 void ReservedSpace::initialize(size_t size, size_t alignment, bool large,
109 char* requested_address,
110 const size_t noaccess_prefix,
111 bool executable) {
112 const size_t granularity = os::vm_allocation_granularity();
113 assert((size & (granularity - 1)) == 0,
114 "size not aligned to os::vm_allocation_granularity()");
115 assert((alignment & (granularity - 1)) == 0,
116 "alignment not aligned to os::vm_allocation_granularity()");
117 assert(alignment == 0 || is_power_of_2((intptr_t)alignment),
118 "not a power of 2");
119
120 alignment = MAX2(alignment, (size_t)os::vm_page_size());
121
122 // Assert that if noaccess_prefix is used, it is the same as alignment.
123 assert(noaccess_prefix == 0 ||
124 noaccess_prefix == alignment, "noaccess prefix wrong");
125
126 _base = NULL;
127 _size = 0;
128 _special = false;
129 _executable = executable;
130 _alignment = 0;
131 _noaccess_prefix = 0;
132 if (size == 0) {
133 return;
134 }
135
136 // If OS doesn't support demand paging for large page memory, we need
137 // to use reserve_memory_special() to reserve and pin the entire region.
138 bool special = large && !os::can_commit_large_page_memory();
139 char* base = NULL;
140
141 if (requested_address != 0) {
142 requested_address -= noaccess_prefix; // adjust requested address
143 assert(requested_address != NULL, "huge noaccess prefix?");
144 }
145
146 if (special) {
147
148 base = os::reserve_memory_special(size, alignment, requested_address, executable);
149
150 if (base != NULL) {
151 if (failed_to_reserve_as_requested(base, requested_address, size, true)) {
152 // OS ignored requested address. Try different address.
153 return;
154 }
155 // Check alignment constraints.
156 assert((uintptr_t) base % alignment == 0,
157 err_msg("Large pages returned a non-aligned address, base: "
158 PTR_FORMAT " alignment: " PTR_FORMAT,
159 base, (void*)(uintptr_t)alignment));
160 _special = true;
161 } else {
162 // failed; try to reserve regular memory below
163 if (UseLargePages && (!FLAG_IS_DEFAULT(UseLargePages) ||
164 !FLAG_IS_DEFAULT(LargePageSizeInBytes))) {
165 if (PrintCompressedOopsMode) {
166 tty->cr();
167 tty->print_cr("Reserve regular memory without large pages.");
168 }
169 }
170 }
171 }
172
173 if (base == NULL) {
174 // Optimistically assume that the OSes returns an aligned base pointer.
175 // When reserving a large address range, most OSes seem to align to at
176 // least 64K.
177
178 // If the memory was requested at a particular address, use
179 // os::attempt_reserve_memory_at() to avoid over mapping something
180 // important. If available space is not detected, return NULL.
181
182 if (requested_address != 0) {
183 base = os::attempt_reserve_memory_at(size, requested_address);
184 if (failed_to_reserve_as_requested(base, requested_address, size, false)) {
185 // OS ignored requested address. Try different address.
186 base = NULL;
187 }
188 } else {
189 base = os::reserve_memory(size, NULL, alignment);
190 }
191
192 if (base == NULL) return;
193
194 // Check alignment constraints
195 if ((((size_t)base + noaccess_prefix) & (alignment - 1)) != 0) {
196 // Base not aligned, retry
197 if (!os::release_memory(base, size)) fatal("os::release_memory failed");
198 // Make sure that size is aligned
199 size = align_size_up(size, alignment);
200 base = os::reserve_memory_aligned(size, alignment);
201
202 if (requested_address != 0 &&
203 failed_to_reserve_as_requested(base, requested_address, size, false)) {
204 // As a result of the alignment constraints, the allocated base differs
205 // from the requested address. Return back to the caller who can
206 // take remedial action (like try again without a requested address).
207 assert(_base == NULL, "should be");
208 return;
209 }
210 }
211 }
212 // Done
213 _base = base;
214 _size = size;
215 _alignment = alignment;
216 _noaccess_prefix = noaccess_prefix;
217
218 // Assert that if noaccess_prefix is used, it is the same as alignment.
219 assert(noaccess_prefix == 0 ||
220 noaccess_prefix == _alignment, "noaccess prefix wrong");
221
222 assert(markOopDesc::encode_pointer_as_mark(_base)->decode_pointer() == _base,
223 "area must be distinguisable from marks for mark-sweep");
224 assert(markOopDesc::encode_pointer_as_mark(&_base[size])->decode_pointer() == &_base[size],
225 "area must be distinguisable from marks for mark-sweep");
226 }
227
228
229 ReservedSpace::ReservedSpace(char* base, size_t size, size_t alignment,
230 bool special, bool executable) {
231 assert((size % os::vm_allocation_granularity()) == 0,
232 "size not allocation aligned");
233 _base = base;
234 _size = size;
235 _alignment = alignment;
236 _noaccess_prefix = 0;
237 _special = special;
238 _executable = executable;
239 }
240
241
242 ReservedSpace ReservedSpace::first_part(size_t partition_size, size_t alignment,
243 bool split, bool realloc) {
244 assert(partition_size <= size(), "partition failed");
245 if (split) {
246 os::split_reserved_memory(base(), size(), partition_size, realloc);
247 }
248 ReservedSpace result(base(), partition_size, alignment, special(),
249 executable());
250 return result;
251 }
252
253
254 ReservedSpace
255 ReservedSpace::last_part(size_t partition_size, size_t alignment) {
256 assert(partition_size <= size(), "partition failed");
257 ReservedSpace result(base() + partition_size, size() - partition_size,
258 alignment, special(), executable());
259 return result;
260 }
261
262
263 size_t ReservedSpace::page_align_size_up(size_t size) {
264 return align_size_up(size, os::vm_page_size());
265 }
266
267
268 size_t ReservedSpace::page_align_size_down(size_t size) {
269 return align_size_down(size, os::vm_page_size());
270 }
271
272
273 size_t ReservedSpace::allocation_align_size_up(size_t size) {
274 return align_size_up(size, os::vm_allocation_granularity());
275 }
276
277
278 size_t ReservedSpace::allocation_align_size_down(size_t size) {
279 return align_size_down(size, os::vm_allocation_granularity());
280 }
281
282
283 void ReservedSpace::release() {
284 if (is_reserved()) {
285 char *real_base = _base - _noaccess_prefix;
286 const size_t real_size = _size + _noaccess_prefix;
287 if (special()) {
288 os::release_memory_special(real_base, real_size);
289 } else{
290 os::release_memory(real_base, real_size);
291 }
292 _base = NULL;
293 _size = 0;
294 _noaccess_prefix = 0;
295 _special = false;
296 _executable = false;
297 }
298 }
299
300 void ReservedSpace::protect_noaccess_prefix(const size_t size) {
301 assert( (_noaccess_prefix != 0) == (UseCompressedOops && _base != NULL &&
302 (Universe::narrow_oop_base() != NULL) &&
303 Universe::narrow_oop_use_implicit_null_checks()),
304 "noaccess_prefix should be used only with non zero based compressed oops");
305
306 // If there is no noaccess prefix, return.
307 if (_noaccess_prefix == 0) return;
308
309 assert(_noaccess_prefix >= (size_t)os::vm_page_size(),
310 "must be at least page size big");
311
312 // Protect memory at the base of the allocated region.
313 // If special, the page was committed (only matters on windows)
314 if (!os::protect_memory(_base, _noaccess_prefix, os::MEM_PROT_NONE,
315 _special)) {
316 fatal("cannot protect protection page");
317 }
318 if (PrintCompressedOopsMode) {
319 tty->cr();
320 tty->print_cr("Protected page at the reserved heap base: " PTR_FORMAT " / " INTX_FORMAT " bytes", _base, _noaccess_prefix);
321 }
322
323 _base += _noaccess_prefix;
324 _size -= _noaccess_prefix;
325 assert((size == _size) && ((uintptr_t)_base % _alignment == 0),
326 "must be exactly of required size and alignment");
327 }
328
329 ReservedHeapSpace::ReservedHeapSpace(size_t size, size_t alignment,
330 bool large, char* requested_address) :
331 ReservedSpace(size, alignment, large,
332 requested_address,
333 (UseCompressedOops && (Universe::narrow_oop_base() != NULL) &&
334 Universe::narrow_oop_use_implicit_null_checks()) ?
335 lcm(os::vm_page_size(), alignment) : 0) {
336 if (base() > 0) {
337 MemTracker::record_virtual_memory_type((address)base(), mtJavaHeap);
338 }
339
340 // Only reserved space for the java heap should have a noaccess_prefix
341 // if using compressed oops.
342 protect_noaccess_prefix(size);
343 }
344
345 // Reserve space for code segment. Same as Java heap only we mark this as
346 // executable.
347 ReservedCodeSpace::ReservedCodeSpace(size_t r_size,
348 size_t rs_align,
349 bool large) :
350 ReservedSpace(r_size, rs_align, large, /*executable*/ true) {
351 MemTracker::record_virtual_memory_type((address)base(), mtCode);
352 }
353
354 // VirtualSpace
355
356 VirtualSpace::VirtualSpace() {
357 _low_boundary = NULL;
358 _high_boundary = NULL;
359 _low = NULL;
360 _high = NULL;
361 _lower_high = NULL;
362 _middle_high = NULL;
363 _upper_high = NULL;
364 _lower_high_boundary = NULL;
365 _middle_high_boundary = NULL;
366 _upper_high_boundary = NULL;
367 _lower_alignment = 0;
368 _middle_alignment = 0;
369 _upper_alignment = 0;
370 _special = false;
371 _executable = false;
372 }
373
374
375 bool VirtualSpace::initialize(ReservedSpace rs, size_t committed_size) {
376 const size_t max_commit_granularity = os::page_size_for_region_unaligned(rs.size(), 1);
377 return initialize_with_granularity(rs, committed_size, max_commit_granularity);
378 }
379
380 bool VirtualSpace::initialize_with_granularity(ReservedSpace rs, size_t committed_size, size_t max_commit_granularity) {
381 if(!rs.is_reserved()) return false; // allocation failed.
382 assert(_low_boundary == NULL, "VirtualSpace already initialized");
383 assert(max_commit_granularity > 0, "Granularity must be non-zero.");
384
385 _low_boundary = rs.base();
386 _high_boundary = low_boundary() + rs.size();
387
388 _low = low_boundary();
389 _high = low();
390
391 _special = rs.special();
392 _executable = rs.executable();
393
394 // When a VirtualSpace begins life at a large size, make all future expansion
395 // and shrinking occur aligned to a granularity of large pages. This avoids
396 // fragmentation of physical addresses that inhibits the use of large pages
397 // by the OS virtual memory system. Empirically, we see that with a 4MB
398 // page size, the only spaces that get handled this way are codecache and
399 // the heap itself, both of which provide a substantial performance
400 // boost in many benchmarks when covered by large pages.
401 //
402 // No attempt is made to force large page alignment at the very top and
403 // bottom of the space if they are not aligned so already.
404 _lower_alignment = os::vm_page_size();
405 _middle_alignment = max_commit_granularity;
406 _upper_alignment = os::vm_page_size();
407
408 // End of each region
409 _lower_high_boundary = (char*) round_to((intptr_t) low_boundary(), middle_alignment());
410 _middle_high_boundary = (char*) round_down((intptr_t) high_boundary(), middle_alignment());
411 _upper_high_boundary = high_boundary();
412
413 // High address of each region
414 _lower_high = low_boundary();
415 _middle_high = lower_high_boundary();
416 _upper_high = middle_high_boundary();
417
418 // commit to initial size
419 if (committed_size > 0) {
420 if (!expand_by(committed_size)) {
421 return false;
422 }
423 }
424 return true;
425 }
426
427
428 VirtualSpace::~VirtualSpace() {
429 release();
430 }
431
432
433 void VirtualSpace::release() {
434 // This does not release memory it never reserved.
435 // Caller must release via rs.release();
436 _low_boundary = NULL;
437 _high_boundary = NULL;
438 _low = NULL;
439 _high = NULL;
440 _lower_high = NULL;
441 _middle_high = NULL;
442 _upper_high = NULL;
443 _lower_high_boundary = NULL;
444 _middle_high_boundary = NULL;
445 _upper_high_boundary = NULL;
446 _lower_alignment = 0;
447 _middle_alignment = 0;
448 _upper_alignment = 0;
449 _special = false;
450 _executable = false;
451 }
452
453
454 size_t VirtualSpace::committed_size() const {
455 return pointer_delta(high(), low(), sizeof(char));
456 }
457
458
459 size_t VirtualSpace::reserved_size() const {
460 return pointer_delta(high_boundary(), low_boundary(), sizeof(char));
461 }
462
463
464 size_t VirtualSpace::uncommitted_size() const {
465 return reserved_size() - committed_size();
466 }
467
468 size_t VirtualSpace::actual_committed_size() const {
469 // Special VirtualSpaces commit all reserved space up front.
470 if (special()) {
471 return reserved_size();
472 }
473
474 size_t committed_low = pointer_delta(_lower_high, _low_boundary, sizeof(char));
475 size_t committed_middle = pointer_delta(_middle_high, _lower_high_boundary, sizeof(char));
476 size_t committed_high = pointer_delta(_upper_high, _middle_high_boundary, sizeof(char));
477
478 #ifdef ASSERT
479 size_t lower = pointer_delta(_lower_high_boundary, _low_boundary, sizeof(char));
480 size_t middle = pointer_delta(_middle_high_boundary, _lower_high_boundary, sizeof(char));
481 size_t upper = pointer_delta(_upper_high_boundary, _middle_high_boundary, sizeof(char));
482
483 if (committed_high > 0) {
484 assert(committed_low == lower, "Must be");
485 assert(committed_middle == middle, "Must be");
486 }
487
488 if (committed_middle > 0) {
489 assert(committed_low == lower, "Must be");
490 }
491 if (committed_middle < middle) {
492 assert(committed_high == 0, "Must be");
493 }
494
495 if (committed_low < lower) {
496 assert(committed_high == 0, "Must be");
497 assert(committed_middle == 0, "Must be");
498 }
499 #endif
500
501 return committed_low + committed_middle + committed_high;
502 }
503
504
505 bool VirtualSpace::contains(const void* p) const {
506 return low() <= (const char*) p && (const char*) p < high();
507 }
508
509 /*
510 First we need to determine if a particular virtual space is using large
511 pages. This is done at the initialize function and only virtual spaces
512 that are larger than LargePageSizeInBytes use large pages. Once we
513 have determined this, all expand_by and shrink_by calls must grow and
514 shrink by large page size chunks. If a particular request
515 is within the current large page, the call to commit and uncommit memory
516 can be ignored. In the case that the low and high boundaries of this
517 space is not large page aligned, the pages leading to the first large
518 page address and the pages after the last large page address must be
519 allocated with default pages.
520 */
521 bool VirtualSpace::expand_by(size_t bytes, bool pre_touch) {
522 if (uncommitted_size() < bytes) return false;
523
524 if (special()) {
525 // don't commit memory if the entire space is pinned in memory
526 _high += bytes;
527 return true;
528 }
529
530 char* previous_high = high();
531 char* unaligned_new_high = high() + bytes;
532 assert(unaligned_new_high <= high_boundary(),
533 "cannot expand by more than upper boundary");
534
535 // Calculate where the new high for each of the regions should be. If
536 // the low_boundary() and high_boundary() are LargePageSizeInBytes aligned
537 // then the unaligned lower and upper new highs would be the
538 // lower_high() and upper_high() respectively.
539 char* unaligned_lower_new_high =
540 MIN2(unaligned_new_high, lower_high_boundary());
541 char* unaligned_middle_new_high =
542 MIN2(unaligned_new_high, middle_high_boundary());
543 char* unaligned_upper_new_high =
544 MIN2(unaligned_new_high, upper_high_boundary());
545
546 // Align the new highs based on the regions alignment. lower and upper
547 // alignment will always be default page size. middle alignment will be
548 // LargePageSizeInBytes if the actual size of the virtual space is in
549 // fact larger than LargePageSizeInBytes.
550 char* aligned_lower_new_high =
551 (char*) round_to((intptr_t) unaligned_lower_new_high, lower_alignment());
552 char* aligned_middle_new_high =
553 (char*) round_to((intptr_t) unaligned_middle_new_high, middle_alignment());
554 char* aligned_upper_new_high =
555 (char*) round_to((intptr_t) unaligned_upper_new_high, upper_alignment());
556
557 // Determine which regions need to grow in this expand_by call.
558 // If you are growing in the lower region, high() must be in that
559 // region so calcuate the size based on high(). For the middle and
560 // upper regions, determine the starting point of growth based on the
561 // location of high(). By getting the MAX of the region's low address
562 // (or the prevoius region's high address) and high(), we can tell if it
563 // is an intra or inter region growth.
564 size_t lower_needs = 0;
565 if (aligned_lower_new_high > lower_high()) {
566 lower_needs =
567 pointer_delta(aligned_lower_new_high, lower_high(), sizeof(char));
568 }
569 size_t middle_needs = 0;
570 if (aligned_middle_new_high > middle_high()) {
571 middle_needs =
572 pointer_delta(aligned_middle_new_high, middle_high(), sizeof(char));
573 }
574 size_t upper_needs = 0;
575 if (aligned_upper_new_high > upper_high()) {
576 upper_needs =
577 pointer_delta(aligned_upper_new_high, upper_high(), sizeof(char));
578 }
579
580 // Check contiguity.
581 assert(low_boundary() <= lower_high() &&
582 lower_high() <= lower_high_boundary(),
583 "high address must be contained within the region");
584 assert(lower_high_boundary() <= middle_high() &&
585 middle_high() <= middle_high_boundary(),
586 "high address must be contained within the region");
587 assert(middle_high_boundary() <= upper_high() &&
588 upper_high() <= upper_high_boundary(),
589 "high address must be contained within the region");
590
591 // Commit regions
592 if (lower_needs > 0) {
593 assert(low_boundary() <= lower_high() &&
594 lower_high() + lower_needs <= lower_high_boundary(),
595 "must not expand beyond region");
596 if (!os::commit_memory(lower_high(), lower_needs, _executable)) {
597 debug_only(warning("INFO: os::commit_memory(" PTR_FORMAT
598 ", lower_needs=" SIZE_FORMAT ", %d) failed",
599 lower_high(), lower_needs, _executable);)
600 return false;
601 } else {
602 _lower_high += lower_needs;
603 }
604 }
605 if (middle_needs > 0) {
606 assert(lower_high_boundary() <= middle_high() &&
607 middle_high() + middle_needs <= middle_high_boundary(),
608 "must not expand beyond region");
609 if (!os::commit_memory(middle_high(), middle_needs, middle_alignment(),
610 _executable)) {
611 debug_only(warning("INFO: os::commit_memory(" PTR_FORMAT
612 ", middle_needs=" SIZE_FORMAT ", " SIZE_FORMAT
613 ", %d) failed", middle_high(), middle_needs,
614 middle_alignment(), _executable);)
615 return false;
616 }
617 _middle_high += middle_needs;
618 }
619 if (upper_needs > 0) {
620 assert(middle_high_boundary() <= upper_high() &&
621 upper_high() + upper_needs <= upper_high_boundary(),
622 "must not expand beyond region");
623 if (!os::commit_memory(upper_high(), upper_needs, _executable)) {
624 debug_only(warning("INFO: os::commit_memory(" PTR_FORMAT
625 ", upper_needs=" SIZE_FORMAT ", %d) failed",
626 upper_high(), upper_needs, _executable);)
627 return false;
628 } else {
629 _upper_high += upper_needs;
630 }
631 }
632
633 if (pre_touch || AlwaysPreTouch) {
634 os::pretouch_memory(previous_high, unaligned_new_high);
635 }
636
637 _high += bytes;
638 return true;
639 }
640
641 // A page is uncommitted if the contents of the entire page is deemed unusable.
642 // Continue to decrement the high() pointer until it reaches a page boundary
643 // in which case that particular page can now be uncommitted.
644 void VirtualSpace::shrink_by(size_t size) {
645 if (committed_size() < size)
646 fatal("Cannot shrink virtual space to negative size");
647
648 if (special()) {
649 // don't uncommit if the entire space is pinned in memory
650 _high -= size;
651 return;
652 }
653
654 char* unaligned_new_high = high() - size;
655 assert(unaligned_new_high >= low_boundary(), "cannot shrink past lower boundary");
656
657 // Calculate new unaligned address
658 char* unaligned_upper_new_high =
659 MAX2(unaligned_new_high, middle_high_boundary());
660 char* unaligned_middle_new_high =
661 MAX2(unaligned_new_high, lower_high_boundary());
662 char* unaligned_lower_new_high =
663 MAX2(unaligned_new_high, low_boundary());
664
665 // Align address to region's alignment
666 char* aligned_upper_new_high =
667 (char*) round_to((intptr_t) unaligned_upper_new_high, upper_alignment());
668 char* aligned_middle_new_high =
669 (char*) round_to((intptr_t) unaligned_middle_new_high, middle_alignment());
670 char* aligned_lower_new_high =
671 (char*) round_to((intptr_t) unaligned_lower_new_high, lower_alignment());
672
673 // Determine which regions need to shrink
674 size_t upper_needs = 0;
675 if (aligned_upper_new_high < upper_high()) {
676 upper_needs =
677 pointer_delta(upper_high(), aligned_upper_new_high, sizeof(char));
678 }
679 size_t middle_needs = 0;
680 if (aligned_middle_new_high < middle_high()) {
681 middle_needs =
682 pointer_delta(middle_high(), aligned_middle_new_high, sizeof(char));
683 }
684 size_t lower_needs = 0;
685 if (aligned_lower_new_high < lower_high()) {
686 lower_needs =
687 pointer_delta(lower_high(), aligned_lower_new_high, sizeof(char));
688 }
689
690 // Check contiguity.
691 assert(middle_high_boundary() <= upper_high() &&
692 upper_high() <= upper_high_boundary(),
693 "high address must be contained within the region");
694 assert(lower_high_boundary() <= middle_high() &&
695 middle_high() <= middle_high_boundary(),
696 "high address must be contained within the region");
697 assert(low_boundary() <= lower_high() &&
698 lower_high() <= lower_high_boundary(),
699 "high address must be contained within the region");
700
701 // Uncommit
702 if (upper_needs > 0) {
703 assert(middle_high_boundary() <= aligned_upper_new_high &&
704 aligned_upper_new_high + upper_needs <= upper_high_boundary(),
705 "must not shrink beyond region");
706 if (!os::uncommit_memory(aligned_upper_new_high, upper_needs)) {
707 debug_only(warning("os::uncommit_memory failed"));
708 return;
709 } else {
710 _upper_high -= upper_needs;
711 }
712 }
713 if (middle_needs > 0) {
714 assert(lower_high_boundary() <= aligned_middle_new_high &&
715 aligned_middle_new_high + middle_needs <= middle_high_boundary(),
716 "must not shrink beyond region");
717 if (!os::uncommit_memory(aligned_middle_new_high, middle_needs)) {
718 debug_only(warning("os::uncommit_memory failed"));
719 return;
720 } else {
721 _middle_high -= middle_needs;
722 }
723 }
724 if (lower_needs > 0) {
725 assert(low_boundary() <= aligned_lower_new_high &&
726 aligned_lower_new_high + lower_needs <= lower_high_boundary(),
727 "must not shrink beyond region");
728 if (!os::uncommit_memory(aligned_lower_new_high, lower_needs)) {
729 debug_only(warning("os::uncommit_memory failed"));
730 return;
731 } else {
732 _lower_high -= lower_needs;
733 }
734 }
735
736 _high -= size;
737 }
738
739 #ifndef PRODUCT
740 void VirtualSpace::check_for_contiguity() {
741 // Check contiguity.
742 assert(low_boundary() <= lower_high() &&
743 lower_high() <= lower_high_boundary(),
744 "high address must be contained within the region");
745 assert(lower_high_boundary() <= middle_high() &&
746 middle_high() <= middle_high_boundary(),
747 "high address must be contained within the region");
748 assert(middle_high_boundary() <= upper_high() &&
749 upper_high() <= upper_high_boundary(),
750 "high address must be contained within the region");
751 assert(low() >= low_boundary(), "low");
752 assert(low_boundary() <= lower_high_boundary(), "lower high boundary");
753 assert(upper_high_boundary() <= high_boundary(), "upper high boundary");
754 assert(high() <= upper_high(), "upper high");
755 }
756
757 void VirtualSpace::print_on(outputStream* out) {
758 out->print ("Virtual space:");
759 if (special()) out->print(" (pinned in memory)");
760 out->cr();
761 out->print_cr(" - committed: " SIZE_FORMAT, committed_size());
762 out->print_cr(" - reserved: " SIZE_FORMAT, reserved_size());
763 out->print_cr(" - [low, high]: [" INTPTR_FORMAT ", " INTPTR_FORMAT "]", low(), high());
764 out->print_cr(" - [low_b, high_b]: [" INTPTR_FORMAT ", " INTPTR_FORMAT "]", low_boundary(), high_boundary());
765 }
766
767 void VirtualSpace::print() {
768 print_on(tty);
769 }
770
771 /////////////// Unit tests ///////////////
772
773 #ifndef PRODUCT
774
775 #define test_log(...) \
776 do {\
777 if (VerboseInternalVMTests) { \
778 tty->print_cr(__VA_ARGS__); \
779 tty->flush(); \
780 }\
781 } while (false)
782
783 class TestReservedSpace : AllStatic {
784 public:
785 static void small_page_write(void* addr, size_t size) {
786 size_t page_size = os::vm_page_size();
787
788 char* end = (char*)addr + size;
789 for (char* p = (char*)addr; p < end; p += page_size) {
790 *p = 1;
791 }
792 }
793
794 static void release_memory_for_test(ReservedSpace rs) {
795 if (rs.special()) {
796 guarantee(os::release_memory_special(rs.base(), rs.size()), "Shouldn't fail");
797 } else {
798 guarantee(os::release_memory(rs.base(), rs.size()), "Shouldn't fail");
799 }
800 }
801
802 static void test_reserved_space1(size_t size, size_t alignment) {
803 test_log("test_reserved_space1(%p)", (void*) (uintptr_t) size);
804
805 assert(is_size_aligned(size, alignment), "Incorrect input parameters");
806
807 ReservedSpace rs(size, // size
808 alignment, // alignment
809 UseLargePages, // large
810 NULL, // requested_address
811 0); // noacces_prefix
812
813 test_log(" rs.special() == %d", rs.special());
814
815 assert(rs.base() != NULL, "Must be");
816 assert(rs.size() == size, "Must be");
817
818 assert(is_ptr_aligned(rs.base(), alignment), "aligned sizes should always give aligned addresses");
819 assert(is_size_aligned(rs.size(), alignment), "aligned sizes should always give aligned addresses");
820
821 if (rs.special()) {
822 small_page_write(rs.base(), size);
823 }
824
825 release_memory_for_test(rs);
826 }
827
828 static void test_reserved_space2(size_t size) {
829 test_log("test_reserved_space2(%p)", (void*)(uintptr_t)size);
830
831 assert(is_size_aligned(size, os::vm_allocation_granularity()), "Must be at least AG aligned");
832
833 ReservedSpace rs(size);
834
835 test_log(" rs.special() == %d", rs.special());
836
837 assert(rs.base() != NULL, "Must be");
838 assert(rs.size() == size, "Must be");
839
840 if (rs.special()) {
841 small_page_write(rs.base(), size);
842 }
843
844 release_memory_for_test(rs);
845 }
846
847 static void test_reserved_space3(size_t size, size_t alignment, bool maybe_large) {
848 test_log("test_reserved_space3(%p, %p, %d)",
849 (void*)(uintptr_t)size, (void*)(uintptr_t)alignment, maybe_large);
850
851 assert(is_size_aligned(size, os::vm_allocation_granularity()), "Must be at least AG aligned");
852 assert(is_size_aligned(size, alignment), "Must be at least aligned against alignment");
853
854 bool large = maybe_large && UseLargePages && size >= os::large_page_size();
855
856 ReservedSpace rs(size, alignment, large, false);
857
858 test_log(" rs.special() == %d", rs.special());
859
860 assert(rs.base() != NULL, "Must be");
861 assert(rs.size() == size, "Must be");
862
863 if (rs.special()) {
864 small_page_write(rs.base(), size);
865 }
866
867 release_memory_for_test(rs);
868 }
869
870
871 static void test_reserved_space1() {
872 size_t size = 2 * 1024 * 1024;
873 size_t ag = os::vm_allocation_granularity();
874
875 test_reserved_space1(size, ag);
876 test_reserved_space1(size * 2, ag);
877 test_reserved_space1(size * 10, ag);
878 }
879
880 static void test_reserved_space2() {
881 size_t size = 2 * 1024 * 1024;
882 size_t ag = os::vm_allocation_granularity();
883
884 test_reserved_space2(size * 1);
885 test_reserved_space2(size * 2);
886 test_reserved_space2(size * 10);
887 test_reserved_space2(ag);
888 test_reserved_space2(size - ag);
889 test_reserved_space2(size);
890 test_reserved_space2(size + ag);
891 test_reserved_space2(size * 2);
892 test_reserved_space2(size * 2 - ag);
893 test_reserved_space2(size * 2 + ag);
894 test_reserved_space2(size * 3);
895 test_reserved_space2(size * 3 - ag);
896 test_reserved_space2(size * 3 + ag);
897 test_reserved_space2(size * 10);
898 test_reserved_space2(size * 10 + size / 2);
899 }
900
901 static void test_reserved_space3() {
902 size_t ag = os::vm_allocation_granularity();
903
904 test_reserved_space3(ag, ag , false);
905 test_reserved_space3(ag * 2, ag , false);
906 test_reserved_space3(ag * 3, ag , false);
907 test_reserved_space3(ag * 2, ag * 2, false);
908 test_reserved_space3(ag * 4, ag * 2, false);
909 test_reserved_space3(ag * 8, ag * 2, false);
910 test_reserved_space3(ag * 4, ag * 4, false);
911 test_reserved_space3(ag * 8, ag * 4, false);
912 test_reserved_space3(ag * 16, ag * 4, false);
913
914 if (UseLargePages) {
915 size_t lp = os::large_page_size();
916
917 // Without large pages
918 test_reserved_space3(lp, ag * 4, false);
919 test_reserved_space3(lp * 2, ag * 4, false);
920 test_reserved_space3(lp * 4, ag * 4, false);
921 test_reserved_space3(lp, lp , false);
922 test_reserved_space3(lp * 2, lp , false);
923 test_reserved_space3(lp * 3, lp , false);
924 test_reserved_space3(lp * 2, lp * 2, false);
925 test_reserved_space3(lp * 4, lp * 2, false);
926 test_reserved_space3(lp * 8, lp * 2, false);
927
928 // With large pages
929 test_reserved_space3(lp, ag * 4 , true);
930 test_reserved_space3(lp * 2, ag * 4, true);
931 test_reserved_space3(lp * 4, ag * 4, true);
932 test_reserved_space3(lp, lp , true);
933 test_reserved_space3(lp * 2, lp , true);
934 test_reserved_space3(lp * 3, lp , true);
935 test_reserved_space3(lp * 2, lp * 2, true);
936 test_reserved_space3(lp * 4, lp * 2, true);
937 test_reserved_space3(lp * 8, lp * 2, true);
938 }
939 }
940
941 static void test_reserved_space() {
942 test_reserved_space1();
943 test_reserved_space2();
944 test_reserved_space3();
945 }
946 };
947
948 void TestReservedSpace_test() {
949 TestReservedSpace::test_reserved_space();
950 }
951
952 #define assert_equals(actual, expected) \
953 assert(actual == expected, \
954 err_msg("Got " SIZE_FORMAT " expected " \
955 SIZE_FORMAT, actual, expected));
956
957 #define assert_ge(value1, value2) \
958 assert(value1 >= value2, \
959 err_msg("'" #value1 "': " SIZE_FORMAT " '" \
960 #value2 "': " SIZE_FORMAT, value1, value2));
961
962 #define assert_lt(value1, value2) \
963 assert(value1 < value2, \
964 err_msg("'" #value1 "': " SIZE_FORMAT " '" \
965 #value2 "': " SIZE_FORMAT, value1, value2));
966
967
968 class TestVirtualSpace : AllStatic {
969 enum TestLargePages {
970 Default,
971 Disable,
972 Reserve,
973 Commit
974 };
975
976 static ReservedSpace reserve_memory(size_t reserve_size_aligned, TestLargePages mode) {
977 switch(mode) {
978 default:
979 case Default:
980 case Reserve:
981 return ReservedSpace(reserve_size_aligned);
982 case Disable:
983 case Commit:
984 return ReservedSpace(reserve_size_aligned,
985 os::vm_allocation_granularity(),
986 /* large */ false, /* exec */ false);
987 }
988 }
989
990 static bool initialize_virtual_space(VirtualSpace& vs, ReservedSpace rs, TestLargePages mode) {
991 switch(mode) {
992 default:
993 case Default:
994 case Reserve:
995 return vs.initialize(rs, 0);
996 case Disable:
997 return vs.initialize_with_granularity(rs, 0, os::vm_page_size());
998 case Commit:
999 return vs.initialize_with_granularity(rs, 0, os::page_size_for_region_unaligned(rs.size(), 1));
1000 }
1001 }
1002
1003 public:
1004 static void test_virtual_space_actual_committed_space(size_t reserve_size, size_t commit_size,
1005 TestLargePages mode = Default) {
1006 size_t granularity = os::vm_allocation_granularity();
1007 size_t reserve_size_aligned = align_size_up(reserve_size, granularity);
1008
1009 ReservedSpace reserved = reserve_memory(reserve_size_aligned, mode);
1010
1011 assert(reserved.is_reserved(), "Must be");
1012
1013 VirtualSpace vs;
1014 bool initialized = initialize_virtual_space(vs, reserved, mode);
1015 assert(initialized, "Failed to initialize VirtualSpace");
1016
1017 vs.expand_by(commit_size, false);
1018
1019 if (vs.special()) {
1020 assert_equals(vs.actual_committed_size(), reserve_size_aligned);
1021 } else {
1022 assert_ge(vs.actual_committed_size(), commit_size);
1023 // Approximate the commit granularity.
1024 // Make sure that we don't commit using large pages
1025 // if large pages has been disabled for this VirtualSpace.
1026 size_t commit_granularity = (mode == Disable || !UseLargePages) ?
1027 os::vm_page_size() : os::large_page_size();
1028 assert_lt(vs.actual_committed_size(), commit_size + commit_granularity);
1029 }
1030
1031 reserved.release();
1032 }
1033
1034 static void test_virtual_space_actual_committed_space_one_large_page() {
1035 if (!UseLargePages) {
1036 return;
1037 }
1038
1039 size_t large_page_size = os::large_page_size();
1040
1041 ReservedSpace reserved(large_page_size, large_page_size, true, false);
1042
1043 assert(reserved.is_reserved(), "Must be");
1044
1045 VirtualSpace vs;
1046 bool initialized = vs.initialize(reserved, 0);
1047 assert(initialized, "Failed to initialize VirtualSpace");
1048
1049 vs.expand_by(large_page_size, false);
1050
1051 assert_equals(vs.actual_committed_size(), large_page_size);
1052
1053 reserved.release();
1054 }
1055
1056 static void test_virtual_space_actual_committed_space() {
1057 test_virtual_space_actual_committed_space(4 * K, 0);
1058 test_virtual_space_actual_committed_space(4 * K, 4 * K);
1059 test_virtual_space_actual_committed_space(8 * K, 0);
1060 test_virtual_space_actual_committed_space(8 * K, 4 * K);
1061 test_virtual_space_actual_committed_space(8 * K, 8 * K);
1062 test_virtual_space_actual_committed_space(12 * K, 0);
1063 test_virtual_space_actual_committed_space(12 * K, 4 * K);
1064 test_virtual_space_actual_committed_space(12 * K, 8 * K);
1065 test_virtual_space_actual_committed_space(12 * K, 12 * K);
1066 test_virtual_space_actual_committed_space(64 * K, 0);
1067 test_virtual_space_actual_committed_space(64 * K, 32 * K);
1068 test_virtual_space_actual_committed_space(64 * K, 64 * K);
1069 test_virtual_space_actual_committed_space(2 * M, 0);
1070 test_virtual_space_actual_committed_space(2 * M, 4 * K);
1071 test_virtual_space_actual_committed_space(2 * M, 64 * K);
1072 test_virtual_space_actual_committed_space(2 * M, 1 * M);
1073 test_virtual_space_actual_committed_space(2 * M, 2 * M);
1074 test_virtual_space_actual_committed_space(10 * M, 0);
1075 test_virtual_space_actual_committed_space(10 * M, 4 * K);
1076 test_virtual_space_actual_committed_space(10 * M, 8 * K);
1077 test_virtual_space_actual_committed_space(10 * M, 1 * M);
1078 test_virtual_space_actual_committed_space(10 * M, 2 * M);
1079 test_virtual_space_actual_committed_space(10 * M, 5 * M);
1080 test_virtual_space_actual_committed_space(10 * M, 10 * M);
1081 }
1082
1083 static void test_virtual_space_disable_large_pages() {
1084 if (!UseLargePages) {
1085 return;
1086 }
1087 // These test cases verify that if we force VirtualSpace to disable large pages
1088 test_virtual_space_actual_committed_space(10 * M, 0, Disable);
1089 test_virtual_space_actual_committed_space(10 * M, 4 * K, Disable);
1090 test_virtual_space_actual_committed_space(10 * M, 8 * K, Disable);
1091 test_virtual_space_actual_committed_space(10 * M, 1 * M, Disable);
1092 test_virtual_space_actual_committed_space(10 * M, 2 * M, Disable);
1093 test_virtual_space_actual_committed_space(10 * M, 5 * M, Disable);
1094 test_virtual_space_actual_committed_space(10 * M, 10 * M, Disable);
1095
1096 test_virtual_space_actual_committed_space(10 * M, 0, Reserve);
1097 test_virtual_space_actual_committed_space(10 * M, 4 * K, Reserve);
1098 test_virtual_space_actual_committed_space(10 * M, 8 * K, Reserve);
1099 test_virtual_space_actual_committed_space(10 * M, 1 * M, Reserve);
1100 test_virtual_space_actual_committed_space(10 * M, 2 * M, Reserve);
1101 test_virtual_space_actual_committed_space(10 * M, 5 * M, Reserve);
1102 test_virtual_space_actual_committed_space(10 * M, 10 * M, Reserve);
1103
1104 test_virtual_space_actual_committed_space(10 * M, 0, Commit);
1105 test_virtual_space_actual_committed_space(10 * M, 4 * K, Commit);
1106 test_virtual_space_actual_committed_space(10 * M, 8 * K, Commit);
1107 test_virtual_space_actual_committed_space(10 * M, 1 * M, Commit);
1108 test_virtual_space_actual_committed_space(10 * M, 2 * M, Commit);
1109 test_virtual_space_actual_committed_space(10 * M, 5 * M, Commit);
1110 test_virtual_space_actual_committed_space(10 * M, 10 * M, Commit);
1111 }
1112
1113 static void test_virtual_space() {
1114 test_virtual_space_actual_committed_space();
1115 test_virtual_space_actual_committed_space_one_large_page();
1116 test_virtual_space_disable_large_pages();
1117 }
1118 };
1119
1120 void TestVirtualSpace_test() {
1121 TestVirtualSpace::test_virtual_space();
1122 }
1123
1124 #endif // PRODUCT
1125
1126 #endif