# HG changeset patch # User goetz # Date 1414601578 -3600 # Node ID 8dcc736582d558190f896c7020c7b6448febec06 # Parent 4522428f522022fe01b8e0c88f149ed00380d659 8064457: Introduce compressed oops mode "disjoint base" and improve compressed heap handling. diff --git a/src/share/vm/memory/universe.cpp b/src/share/vm/memory/universe.cpp --- a/src/share/vm/memory/universe.cpp +++ b/src/share/vm/memory/universe.cpp @@ -691,103 +691,6 @@ // NarrowOopHeapBaseMin + heap_size < 32Gb // HeapBased - Use compressed oops with heap base + encoding. -// 4Gb -static const uint64_t UnscaledOopHeapMax = (uint64_t(max_juint) + 1); -// 32Gb -// OopEncodingHeapMax == UnscaledOopHeapMax << LogMinObjAlignmentInBytes; - -char* Universe::preferred_heap_base(size_t heap_size, size_t alignment, NARROW_OOP_MODE mode) { - assert(is_size_aligned((size_t)OopEncodingHeapMax, alignment), "Must be"); - assert(is_size_aligned((size_t)UnscaledOopHeapMax, alignment), "Must be"); - assert(is_size_aligned(heap_size, alignment), "Must be"); - - uintx heap_base_min_address_aligned = align_size_up(HeapBaseMinAddress, alignment); - - size_t base = 0; -#ifdef _LP64 - if (UseCompressedOops) { - assert(mode == UnscaledNarrowOop || - mode == ZeroBasedNarrowOop || - mode == HeapBasedNarrowOop, "mode is invalid"); - const size_t total_size = heap_size + heap_base_min_address_aligned; - // Return specified base for the first request. - if (!FLAG_IS_DEFAULT(HeapBaseMinAddress) && (mode == UnscaledNarrowOop)) { - base = heap_base_min_address_aligned; - - // If the total size is small enough to allow UnscaledNarrowOop then - // just use UnscaledNarrowOop. - } else if ((total_size <= OopEncodingHeapMax) && (mode != HeapBasedNarrowOop)) { - if ((total_size <= UnscaledOopHeapMax) && (mode == UnscaledNarrowOop) && - (Universe::narrow_oop_shift() == 0)) { - // Use 32-bits oops without encoding and - // place heap's top on the 4Gb boundary - base = (UnscaledOopHeapMax - heap_size); - } else { - // Can't reserve with NarrowOopShift == 0 - Universe::set_narrow_oop_shift(LogMinObjAlignmentInBytes); - - if (mode == UnscaledNarrowOop || - mode == ZeroBasedNarrowOop && total_size <= UnscaledOopHeapMax) { - - // Use zero based compressed oops with encoding and - // place heap's top on the 32Gb boundary in case - // total_size > 4Gb or failed to reserve below 4Gb. - uint64_t heap_top = OopEncodingHeapMax; - - // For small heaps, save some space for compressed class pointer - // space so it can be decoded with no base. - if (UseCompressedClassPointers && !UseSharedSpaces && - OopEncodingHeapMax <= 32*G) { - - uint64_t class_space = align_size_up(CompressedClassSpaceSize, alignment); - assert(is_size_aligned((size_t)OopEncodingHeapMax-class_space, - alignment), "difference must be aligned too"); - uint64_t new_top = OopEncodingHeapMax-class_space; - - if (total_size <= new_top) { - heap_top = new_top; - } - } - - // Align base to the adjusted top of the heap - base = heap_top - heap_size; - } - } - } else { - // UnscaledNarrowOop encoding didn't work, and no base was found for ZeroBasedOops or - // HeapBasedNarrowOop encoding was requested. So, can't reserve below 32Gb. - Universe::set_narrow_oop_shift(LogMinObjAlignmentInBytes); - } - - // Set narrow_oop_base and narrow_oop_use_implicit_null_checks - // used in ReservedHeapSpace() constructors. - // The final values will be set in initialize_heap() below. - if ((base != 0) && ((base + heap_size) <= OopEncodingHeapMax)) { - // Use zero based compressed oops - Universe::set_narrow_oop_base(NULL); - // Don't need guard page for implicit checks in indexed - // addressing mode with zero based Compressed Oops. - Universe::set_narrow_oop_use_implicit_null_checks(true); - } else { - // Set to a non-NULL value so the ReservedSpace ctor computes - // the correct no-access prefix. - // The final value will be set in initialize_heap() below. - Universe::set_narrow_oop_base((address)UnscaledOopHeapMax); -#if defined(_WIN64) || defined(AIX) - if (UseLargePages) { - // Cannot allocate guard pages for implicit checks in indexed - // addressing mode when large pages are specified on windows. - Universe::set_narrow_oop_use_implicit_null_checks(false); - } -#endif // _WIN64 - } - } -#endif - - assert(is_ptr_aligned((char*)base, alignment), "Must be"); - return (char*)base; // also return NULL (don't care) for 32-bit VM -} - jint Universe::initialize_heap() { if (UseParallelGC) { @@ -841,30 +744,13 @@ // See needs_explicit_null_check. // Only set the heap base for compressed oops because it indicates // compressed oops for pstack code. - if (((uint64_t)Universe::heap()->reserved_region().end() > OopEncodingHeapMax)) { - // Can't reserve heap below 32Gb. - // keep the Universe::narrow_oop_base() set in Universe::reserve_heap() + if ((uint64_t)Universe::heap()->reserved_region().end() > UnscaledOopHeapMax) { + // Didn't reserve heap below 4Gb. Must shift. Universe::set_narrow_oop_shift(LogMinObjAlignmentInBytes); -#ifdef AIX - // There is no protected page before the heap. This assures all oops - // are decoded so that NULL is preserved, so this page will not be accessed. - Universe::set_narrow_oop_use_implicit_null_checks(false); -#endif - } else { + } + if ((uint64_t)Universe::heap()->reserved_region().end() <= OopEncodingHeapMax) { + // Did reserve heap below 32Gb. Can use base == 0; Universe::set_narrow_oop_base(0); -#ifdef _WIN64 - if (!Universe::narrow_oop_use_implicit_null_checks()) { - // Don't need guard page for implicit checks in indexed addressing - // mode with zero based Compressed Oops. - Universe::set_narrow_oop_use_implicit_null_checks(true); - } -#endif // _WIN64 - if((uint64_t)Universe::heap()->reserved_region().end() > UnscaledOopHeapMax) { - // Can't reserve heap below 4Gb. - Universe::set_narrow_oop_shift(LogMinObjAlignmentInBytes); - } else { - Universe::set_narrow_oop_shift(0); - } } Universe::set_narrow_ptrs_base(Universe::narrow_oop_base()); @@ -900,22 +786,75 @@ tty->print(", Compressed Oops mode: %s", narrow_oop_mode_to_string(narrow_oop_mode())); if (Universe::narrow_oop_base() != 0) { - tty->print(":" PTR_FORMAT, Universe::narrow_oop_base()); + tty->print(": " PTR_FORMAT, Universe::narrow_oop_base()); } if (Universe::narrow_oop_shift() != 0) { tty->print(", Oop shift amount: %d", Universe::narrow_oop_shift()); } + if (!Universe::narrow_oop_use_implicit_null_checks()) { + tty->print(", no protected page in front of the heap"); + } + tty->cr(); tty->cr(); } -// Reserve the Java heap, which is now the same for all GCs. +#define SIZE_64K ((uint64_t) 0x10000ULL) +#define SIZE_256M ((uint64_t) 0x10000000ULL) +#define SIZE_32G ((uint64_t) 0x800000000ULL) + +// Helper for heap allocation. Returns an array with addresses +// (OS-specific) which are suited for disjoint base mode. Array is +// NULL terminated. +static char** get_attach_addresses_for_disjoint_mode() { + static uintptr_t addresses[] = { +#ifdef _LP64 + 2 * SIZE_32G, + 3 * SIZE_32G, + 4 * SIZE_32G, + 8 * SIZE_32G, + 10 * SIZE_32G, + 1 * SIZE_64K * SIZE_32G, + 2 * SIZE_64K * SIZE_32G, + 3 * SIZE_64K * SIZE_32G, + 4 * SIZE_64K * SIZE_32G, + 16 * SIZE_64K * SIZE_32G, + 32 * SIZE_64K * SIZE_32G, + 34 * SIZE_64K * SIZE_32G, +#endif + 0 + }; + + // Sort out addresses smaller than HeapBaseMinAddress. This assumes + // the array is sorted. + uint i = 0; + while (addresses[i] != 0 && + (addresses[i] < OopEncodingHeapMax || addresses[i] < HeapBaseMinAddress)) { + i++; + } + uint start = i; + + // Avoid more steps than requested. + i = 0; + while (addresses[start+i] != 0) { + if (i == HeapSearchSteps) { + addresses[start+i] = 0; + break; + } + i++; + } + + return (char**) &addresses[start]; +} + ReservedSpace Universe::reserve_heap(size_t heap_size, size_t alignment) { + assert(alignment <= Arguments::conservative_max_heap_alignment(), err_msg("actual alignment "SIZE_FORMAT" must be within maximum heap alignment "SIZE_FORMAT, alignment, Arguments::conservative_max_heap_alignment())); + size_t total_reserved = align_size_up(heap_size, alignment); assert(!UseCompressedOops || (total_reserved <= (OopEncodingHeapMax - os::vm_page_size())), "heap size is too big for compressed oops"); @@ -925,46 +864,126 @@ || UseParallelGC || use_large_pages, "Wrong alignment to use large pages"); - char* addr = Universe::preferred_heap_base(total_reserved, alignment, Universe::UnscaledNarrowOop); - - ReservedHeapSpace total_rs(total_reserved, alignment, use_large_pages, addr); + // Address where to allocate the heap. NULL: anywhere. + char* addr = NULL; + size_t disjoint_noaccess_prefix = 0; if (UseCompressedOops) { - if (addr != NULL && !total_rs.is_reserved()) { - // Failed to reserve at specified address - the requested memory - // region is taken already, for example, by 'java' launcher. - // Try again to reserver heap higher. - addr = Universe::preferred_heap_base(total_reserved, alignment, Universe::ZeroBasedNarrowOop); + // Try to get a heap by: + // 0) if HeapBaseMinAddress is set, try this address first. + // 1) get heap for unscaled (base = 0, shift = 0) + // 2) failing that, get heap for zerobased (base = 0, shift != 0) + // 3) failing that, get heap for disjointbase (base != 0, shift != 0) + // 4) failing that, any heap will do. - ReservedHeapSpace total_rs0(total_reserved, alignment, - use_large_pages, addr); + // Loop over compressed oop modes; try to obtain a fitting memory range; + // if success, release it again and let ReservedHeapSpace attempt to + // allocate in the same range. + for (int i = 0; i <= 4; i++) { + disjoint_noaccess_prefix = 0; + switch (i) { + case 0: + // Attempt to alloc at user-given address. + if (!FLAG_IS_DEFAULT(HeapBaseMinAddress)) { + addr = os::attempt_reserve_memory_at(total_reserved, (char *)HeapBaseMinAddress); + if (is_disjoint_heap_base_address((address)addr)) { + disjoint_noaccess_prefix = ReservedHeapSpace::noaccess_prefix_size(alignment); + } + } + break; + case 1: + // Attempt to alloc for unscaled. + addr = os::attempt_reserve_memory_in_range(total_reserved, alignment, + (char*) HeapBaseMinAddress, + (char*) UnscaledOopHeapMax, + HeapSearchSteps); + break; + case 2: + { + // zerobased: Attempt to allocate in the lower 32G. + // But leave room for the compressed class pointers. + char* zerobased_max = (char*)OopEncodingHeapMax; - if (addr != NULL && !total_rs0.is_reserved()) { - // Failed to reserve at specified address again - give up. - addr = Universe::preferred_heap_base(total_reserved, alignment, Universe::HeapBasedNarrowOop); - assert(addr == NULL, ""); + // For small heaps, save some space for compressed class pointer + // space so it can be decoded with no base. + if (UseCompressedClassPointers && !UseSharedSpaces && + OopEncodingHeapMax <= 32*G) { + uint64_t class_space = align_size_up(CompressedClassSpaceSize, alignment); + assert(is_size_aligned((size_t)OopEncodingHeapMax-class_space, + alignment), "difference must be aligned too"); + zerobased_max = (char*) OopEncodingHeapMax - class_space; + } - ReservedHeapSpace total_rs1(total_reserved, alignment, - use_large_pages, addr); - total_rs = total_rs1; - } else { - total_rs = total_rs0; + addr = os::attempt_reserve_memory_in_range(total_reserved, alignment, + (char*) MAX2((char*)UnscaledOopHeapMax, (char*)HeapBaseMinAddress), + (char*) zerobased_max, + HeapSearchSteps); + } + break; + case 3: + // disjointbase. Here we just try a bushel of OS-dependend known + // disjoint-based friendly addresses. + { + char** addresses = get_attach_addresses_for_disjoint_mode(); + addr = os::attempt_reserve_memory_at_multiple(total_reserved, addresses); + disjoint_noaccess_prefix = ReservedHeapSpace::noaccess_prefix_size(alignment); + } + break; + case 4: + addr = 0; + break; + default: + ShouldNotReachHere(); } - } + + // If we could not find space for the current mode, try the next mode. + if (!addr && i < 4) { + continue; + } + + // If we did find space, release space; ReservedHeapSpace will allocate + // again. + if (addr) { + os::release_memory(addr, total_reserved); + break; // Quit the for loop. + } + + } // for loop } - if (!total_rs.is_reserved()) { - vm_exit_during_initialization(err_msg("Could not reserve enough space for " SIZE_FORMAT "KB object heap", total_reserved/K)); + // now create the space + ReservedHeapSpace total_rs(total_reserved, alignment, + use_large_pages, addr + disjoint_noaccess_prefix); + + if (addr != NULL && !total_rs.is_reserved()) { + // Try arbitrary position. + ReservedHeapSpace total_rs1(total_reserved, alignment, use_large_pages, NULL); + disjoint_noaccess_prefix = 0; + total_rs = total_rs1; + } + + if (total_rs.is_reserved()) { + // we are good. + + if (UseCompressedOops) { + // Universe::initialize_heap() will reset this to NULL if unscaled + // or zero-based narrow oops are actually used. + // SAPJVM GL 2014-09-22 + // Else heap start and base MUST differ, so that NULL can be encoded nonambigous. + address base = (address)(total_rs.base() - ReservedHeapSpace::noaccess_prefix_size(alignment)); + Universe::set_narrow_oop_base(base); + } + return total_rs; } - if (UseCompressedOops) { - // Universe::initialize_heap() will reset this to NULL if unscaled - // or zero-based narrow oops are actually used. - address base = (address)(total_rs.base() - os::vm_page_size()); - Universe::set_narrow_oop_base(base); - } - return total_rs; + vm_exit_during_initialization( + err_msg("Could not reserve enough space for " SIZE_FORMAT "KB object heap", + total_reserved/K)); + + // satisfy compiler + ShouldNotReachHere(); + return ReservedHeapSpace(0, 0, false, 0); } @@ -982,6 +1001,8 @@ return "32-bit"; case ZeroBasedNarrowOop: return "Zero based"; + case DisjointBaseNarrowOop: + return "Non-zero disjoint base"; case HeapBasedNarrowOop: return "Non-zero based"; } @@ -992,6 +1013,10 @@ Universe::NARROW_OOP_MODE Universe::narrow_oop_mode() { + if (narrow_oop_base_disjoint()) { + return DisjointBaseNarrowOop; + } + if (narrow_oop_base() != 0) { return HeapBasedNarrowOop; } diff --git a/src/share/vm/memory/universe.hpp b/src/share/vm/memory/universe.hpp --- a/src/share/vm/memory/universe.hpp +++ b/src/share/vm/memory/universe.hpp @@ -90,7 +90,6 @@ enum VerifyOption { VerifyOption_Default = 0, - // G1 VerifyOption_G1UsePrevMarking = VerifyOption_Default, VerifyOption_G1UseNextMarking = VerifyOption_G1UsePrevMarking + 1, @@ -102,8 +101,8 @@ friend class MarkSweep; friend class oopDesc; friend class ClassLoader; - friend class Arguments; friend class SystemDictionary; + friend class ReservedSpace; friend class VMStructs; friend class VM_PopulateDumpSharedSpace; friend class Metaspace; @@ -349,17 +348,37 @@ // NarrowOopHeapBaseMin + heap_size < 4Gb // 1 - Use zero based compressed oops with encoding when // NarrowOopHeapBaseMin + heap_size < 32Gb - // 2 - Use compressed oops with heap base + encoding. + // 2 - Use compressed oops with disjoint heap base if + // base is 32G-aligned and base > 0. This allows certain + // optimizations in encoding/decoding. + // 3 - Use compressed oops with heap base + encoding. enum NARROW_OOP_MODE { UnscaledNarrowOop = 0, ZeroBasedNarrowOop = 1, - HeapBasedNarrowOop = 2 + DisjointBaseNarrowOop = 2, + HeapBasedNarrowOop = 3, + AnyNarrowOopMode = 4 }; static NARROW_OOP_MODE narrow_oop_mode(); static const char* narrow_oop_mode_to_string(NARROW_OOP_MODE mode); static char* preferred_heap_base(size_t heap_size, size_t alignment, NARROW_OOP_MODE mode); static char* preferred_metaspace_base(size_t heap_size, NARROW_OOP_MODE mode); - static address narrow_oop_base() { return _narrow_oop._base; } + static address narrow_oop_base() { return _narrow_oop._base; } + // Test whether bits of addr and possible offsets into the heap overlap. + static bool is_disjoint_heap_base_address(address addr) { + return (((uint64_t)(intptr_t)addr) & + (((uint64_t)UCONST64(0xFFFFffffFFFFffff)) >> (32-LogMinObjAlignmentInBytes))) == 0; + } + // Check for disjoint base compressed oops. + static bool narrow_oop_base_disjoint() { + return _narrow_oop._base != NULL && is_disjoint_heap_base_address(_narrow_oop._base); + } + // Check for real heapbased compressed oops. + // We must subtract the base as the bits overlap. + // If we negate above function, we also get unscaled and zerobased. + static bool narrow_oop_base_overlaps() { + return _narrow_oop._base != NULL && !is_disjoint_heap_base_address(_narrow_oop._base); + } static bool is_narrow_oop_base(void* addr) { return (narrow_oop_base() == (address)addr); } static int narrow_oop_shift() { return _narrow_oop._shift; } static bool narrow_oop_use_implicit_null_checks() { return _narrow_oop._use_implicit_null_checks; } diff --git a/src/share/vm/opto/matcher.hpp b/src/share/vm/opto/matcher.hpp --- a/src/share/vm/opto/matcher.hpp +++ b/src/share/vm/opto/matcher.hpp @@ -433,6 +433,13 @@ // NullCheck oop_reg // inline static bool gen_narrow_oop_implicit_null_checks() { + // Advice matcher to perform null checks on the narrow oop side. + // Implicit checks are not possible on the uncompressed oop side anyway + // (at least not for read accesses). + // Performs significantly better (especially on Power 6). + if (!os::zero_page_read_protected()) { + return true; + } return Universe::narrow_oop_use_implicit_null_checks() && (narrow_oop_use_complex_address() || Universe::narrow_oop_base() != NULL); diff --git a/src/share/vm/runtime/arguments.cpp b/src/share/vm/runtime/arguments.cpp --- a/src/share/vm/runtime/arguments.cpp +++ b/src/share/vm/runtime/arguments.cpp @@ -1562,15 +1562,6 @@ FLAG_SET_ERGO(bool, UseCompressedOops, true); } #endif -#ifdef _WIN64 - if (UseLargePages && UseCompressedOops) { - // Cannot allocate guard pages for implicit checks in indexed addressing - // mode, when large pages are specified on windows. - // This flag could be switched ON if narrow oop base address is set to 0, - // see code in Universe::initialize_heap(). - Universe::set_narrow_oop_use_implicit_null_checks(false); - } -#endif // _WIN64 } else { if (UseCompressedOops && !FLAG_IS_DEFAULT(UseCompressedOops)) { warning("Max heap size too large for Compressed Oops"); @@ -4182,6 +4173,10 @@ PropertyList_add(plist, new_p); } +void Arguments::PropertyList_add(SystemProperty *element) { + PropertyList_add(&_system_properties, element); +} + // This add maintains unique property key in the list. void Arguments::PropertyList_unique_add(SystemProperty** plist, const char* k, char* v, jboolean append) { if (plist == NULL) diff --git a/src/share/vm/runtime/arguments.hpp b/src/share/vm/runtime/arguments.hpp --- a/src/share/vm/runtime/arguments.hpp +++ b/src/share/vm/runtime/arguments.hpp @@ -571,6 +571,7 @@ static void init_version_specific_system_properties(); // Property List manipulation + static void PropertyList_add(SystemProperty *element); static void PropertyList_add(SystemProperty** plist, SystemProperty *element); static void PropertyList_add(SystemProperty** plist, const char* k, char* v); static void PropertyList_unique_add(SystemProperty** plist, const char* k, char* v) { diff --git a/src/share/vm/runtime/globals.hpp b/src/share/vm/runtime/globals.hpp --- a/src/share/vm/runtime/globals.hpp +++ b/src/share/vm/runtime/globals.hpp @@ -531,6 +531,11 @@ product_pd(uintx, HeapBaseMinAddress, \ "OS specific low limit for heap base address") \ \ + product(uintx, HeapSearchSteps, 1 PPC64_ONLY(+19), \ + "Heap allocation steps through preferred address regions to find" \ + " where it can allocate the heap. Number of steps to take per " \ + "region.") \ + \ diagnostic(bool, PrintCompressedOopsMode, false, \ "Print compressed oops base address and encoding mode") \ \ diff --git a/src/share/vm/runtime/os.cpp b/src/share/vm/runtime/os.cpp --- a/src/share/vm/runtime/os.cpp +++ b/src/share/vm/runtime/os.cpp @@ -1533,6 +1533,115 @@ return result; } +// A convenience function which attempts to reserve memory at a number +// of given addresses. An address array is given and first the first, +// then the second is tried and so on. +// Function returns NULL if memory could not be obtained at any of the +// given addresses. +// +// parameters +// bytes - size of block +// addr - array of addresses, NULL terminated +// +char* os::attempt_reserve_memory_at_multiple(size_t bytes, char* addr[]) { + for (int i = 0; addr[i]; i ++) { + char* const attach_point = addr[i]; + assert(attach_point >= (char *)HeapBaseMinAddress, "Flag support broken"); + char* const addr = os::attempt_reserve_memory_at(bytes, attach_point); + if (addr) { + // Note: depending on platform and OS, it is not guaranteed that + // os::attempt_reserve_memory_at only returns requested addr or nothing :/ + if (addr != attach_point) { + os::release_memory(addr, bytes); + } else { + return addr; + } + } + } + + return NULL; +} + +// A convenience function which attempts to reserve memory +// in a given memory range. +char* os::attempt_reserve_memory_in_range(size_t size, size_t alignment, + char* range_from, char* range_to, int num_attempts) { + // sanity checks + if (range_from == 0) { + range_from = (char*) 1; + } + + if (size == 0) { + return NULL; + } + + if (range_to <= range_from) { + return NULL; + } else { + size_t d = range_to - range_from; + if (d < size) { + return NULL; + } + } + + // The necessary attach point alignment (for mmap or shmat to have a + // chance of attaching). + const size_t os_attach_point_alignment = + AIX_ONLY(SIZE_256M) // Known shm boundary alignment. + NOT_AIX(os::vm_allocation_granularity()); + + const size_t attach_point_alignment = + alignment > 0 ? + lcm(alignment, os_attach_point_alignment) : os_attach_point_alignment; + + // Calc address range within we try to attach (range of possible start addresses). + char* const highest_start = (char *)align_ptr_down(range_to - size, attach_point_alignment); + char* const lowest_start = (char *)align_ptr_up(range_from, attach_point_alignment); + const size_t attach_range = highest_start - lowest_start; + + // Default is 20 ... + if (num_attempts <= 0) { + num_attempts = 20; + } + + // ... but cap at possible + const uint64_t num_attempts_possible = + (attach_range / attach_point_alignment) + 1; // at least one is possible even for 0 sized attach range + + if (num_attempts_possible < (uint64_t)num_attempts) { + num_attempts = num_attempts_possible; + } + + const size_t stepsize = + align_size_up(attach_range / num_attempts, attach_point_alignment); + + // Build up attach point array. + char** attach_points = (char**) os::malloc((num_attempts + 1) * sizeof(char*), mtInternal); + assert(attach_points, "OOM"); + + // Fill attach points array. + char* attach_point = highest_start; + int i = 0; + while (i < num_attempts && attach_point >= lowest_start) { + attach_points[i] = attach_point; + i++; + // Handle overflow correctly! + if ((uintptr_t)attach_point > stepsize) { + attach_point -= stepsize; + } else { + attach_point = 0; + } + } + + attach_points[i] = NULL; + + char* addr = os::attempt_reserve_memory_at_multiple(size, attach_points); + + os::free(attach_points); + + return addr; +} + void os::split_reserved_memory(char *base, size_t size, size_t split, bool realloc) { pd_split_reserved_memory(base, size, split, realloc); diff --git a/src/share/vm/runtime/os.hpp b/src/share/vm/runtime/os.hpp --- a/src/share/vm/runtime/os.hpp +++ b/src/share/vm/runtime/os.hpp @@ -297,6 +297,10 @@ size_t alignment_hint, MEMFLAGS flags); static char* reserve_memory_aligned(size_t size, size_t alignment); static char* attempt_reserve_memory_at(size_t bytes, char* addr); + static char* attempt_reserve_memory_at_multiple(size_t bytes, char* addr[]); + static char* attempt_reserve_memory_in_range(size_t size, size_t alignment, + char* range_from, char* range_to, + int num_attempts); static void split_reserved_memory(char *base, size_t size, size_t split, bool realloc); static bool commit_memory(char* addr, size_t bytes, bool executable); diff --git a/src/share/vm/runtime/virtualspace.cpp b/src/share/vm/runtime/virtualspace.cpp --- a/src/share/vm/runtime/virtualspace.cpp +++ b/src/share/vm/runtime/virtualspace.cpp @@ -70,6 +70,26 @@ if (base != NULL) { // Different reserve address may be acceptable in other cases // but for compressed oops heap should be at requested address. + + // Or at least in the requested mode. + if ((uint64_t)base >= HeapBaseMinAddress) { + if ((uint64_t)requested_address + size < UnscaledOopHeapMax) { + // Requested unscaled mode. + if ((uint64_t)base + size < UnscaledOopHeapMax) { + // Reserved unscaled mode. + if (PrintCompressedOopsMode) { + tty->print("base: %p, req_addr: %p, base+size: %p but fulfills unscaled criteria.\n", base, requested_address, base+size); + } + return false; + } + } else if ((uint64_t)requested_address + size < OopEncodingHeapMax && + (uint64_t)base + size < OopEncodingHeapMax) { + // Requested and reserved zerobased mode. + tty->print("base: %p, req_addr: %p, base+size: %p but fulfills zerobased criteria.\n", base, requested_address, base+size); + return false; + } + } + assert(UseCompressedOops, "currently requested address used only for compressed oops"); if (PrintCompressedOopsMode) { tty->cr(); @@ -282,10 +302,6 @@ } void ReservedSpace::protect_noaccess_prefix(const size_t size) { - assert( (_noaccess_prefix != 0) == (UseCompressedOops && _base != NULL && - (Universe::narrow_oop_base() != NULL) && - Universe::narrow_oop_use_implicit_null_checks()), - "noaccess_prefix should be used only with non zero based compressed oops"); // If there is no noaccess prefix, return. if (_noaccess_prefix == 0) return; @@ -293,15 +309,22 @@ assert(_noaccess_prefix >= (size_t)os::vm_page_size(), "must be at least page size big"); - // Protect memory at the base of the allocated region. - // If special, the page was committed (only matters on windows) - if (!os::protect_memory(_base, _noaccess_prefix, os::MEM_PROT_NONE, - _special)) { - fatal("cannot protect protection page"); - } - if (PrintCompressedOopsMode) { - tty->cr(); - tty->print_cr("Protected page at the reserved heap base: " PTR_FORMAT " / " INTX_FORMAT " bytes", _base, _noaccess_prefix); + if (true + WIN64_ONLY(&& !UseLargePages) + AIX_ONLY(&& os::vm_page_size() != SIZE_64K)) { + // Protect memory at the base of the allocated region. + // If special, the page was committed (only matters on windows) + if (!os::protect_memory(_base, _noaccess_prefix, os::MEM_PROT_NONE, + _special)) { + fatal("cannot protect protection page"); + } + if (PrintCompressedOopsMode) { + tty->cr(); + tty->print_cr("Protected page at the reserved heap base: " PTR_FORMAT " / " INTX_FORMAT " bytes", _base, _noaccess_prefix); + } + assert(Universe::narrow_oop_use_implicit_null_checks() == true, "not initialized?"); + } else { + Universe::set_narrow_oop_use_implicit_null_checks(false); } _base += _noaccess_prefix; @@ -314,9 +337,9 @@ bool large, char* requested_address) : ReservedSpace(size, alignment, large, requested_address, - (UseCompressedOops && (Universe::narrow_oop_base() != NULL) && - Universe::narrow_oop_use_implicit_null_checks()) ? - lcm(os::vm_page_size(), alignment) : 0) { + (UseCompressedOops && (requested_address == NULL || requested_address+size > (char*)OopEncodingHeapMax) ? + noaccess_prefix_size(alignment) : 0)) { + if (base() > 0) { MemTracker::record_virtual_memory_type((address)base(), mtJavaHeap); } @@ -326,6 +349,9 @@ protect_noaccess_prefix(size); } +size_t ReservedHeapSpace::noaccess_prefix_size(size_t alignment) { + return lcm(os::vm_page_size(), alignment); +} // Reserve space for code segment. Same as Java heap only we mark this as // executable. ReservedCodeSpace::ReservedCodeSpace(size_t r_size, diff --git a/src/share/vm/runtime/virtualspace.hpp b/src/share/vm/runtime/virtualspace.hpp --- a/src/share/vm/runtime/virtualspace.hpp +++ b/src/share/vm/runtime/virtualspace.hpp @@ -104,6 +104,7 @@ // Constructor ReservedHeapSpace(size_t size, size_t forced_base_alignment, bool large, char* requested_address); + static size_t noaccess_prefix_size(size_t alignment); }; // Class encapsulating behavior specific memory space for Code diff --git a/src/share/vm/utilities/globalDefinitions.hpp b/src/share/vm/utilities/globalDefinitions.hpp --- a/src/share/vm/utilities/globalDefinitions.hpp +++ b/src/share/vm/utilities/globalDefinitions.hpp @@ -124,7 +124,9 @@ extern int BytesPerHeapOop; extern int BitsPerHeapOop; -// Oop encoding heap max +// 4Gb +static const uint64_t UnscaledOopHeapMax = 0x100000000; +// Oop encoding heap max: UnscaledOopHeapMax << LogMinObjAlignmentInBytes extern uint64_t OopEncodingHeapMax; const int BitsPerJavaInteger = 32; diff --git a/test/runtime/CompressedOops/UseCompressedOops.java b/test/runtime/CompressedOops/UseCompressedOops.java --- a/test/runtime/CompressedOops/UseCompressedOops.java +++ b/test/runtime/CompressedOops/UseCompressedOops.java @@ -63,6 +63,12 @@ .shouldContain("Oop shift amount: 3") .shouldHaveExitValue(0); + // Larger than 3gb heap and HeapBaseMinAddress=1g should result in zero based with shift 3 + testCompressedOops("-XX:+UseCompressedOops", "-Xmx3200m", "-XX:HeapBaseMinAddress=1g") + .shouldContain("Zero based") + .shouldContain("Oop shift amount: 3") + .shouldHaveExitValue(0); + // Small heap above 4gb should result in zero based with shift 3 testCompressedOops("-XX:+UseCompressedOops", "-Xmx32m", "-XX:HeapBaseMinAddress=4g") .shouldContain("Zero based") @@ -71,6 +77,12 @@ // Small heap above 32gb should result in non-zero based with shift 3 testCompressedOops("-XX:+UseCompressedOops", "-Xmx32m", "-XX:HeapBaseMinAddress=32g") + .shouldContain("Non-zero disjoint base") + .shouldContain("Oop shift amount: 3") + .shouldHaveExitValue(0); + + // Small heap above 32gb should result in non-zero based with shift 3 + testCompressedOops("-XX:+UseCompressedOops", "-Xmx32m", "-XX:HeapBaseMinAddress=72704m") .shouldContain("Non-zero based") .shouldContain("Oop shift amount: 3") .shouldHaveExitValue(0); @@ -79,7 +91,7 @@ // in non-zero based with shift 4 testCompressedOops("-XX:+UseCompressedOops", "-Xmx32g", "-XX:ObjectAlignmentInBytes=16", "-XX:HeapBaseMinAddress=64g") - .shouldContain("Non-zero based") + .shouldContain("Non-zero disjoint base") .shouldContain("Oop shift amount: 4") .shouldHaveExitValue(0);