--- old/src/hotspot/share/memory/filemap.cpp 2018-08-14 19:22:55.329371680 -0700 +++ new/src/hotspot/share/memory/filemap.cpp 2018-08-14 19:22:54.997358992 -0700 @@ -35,12 +35,15 @@ #include "logging/logStream.hpp" #include "logging/logMessage.hpp" #include "memory/filemap.hpp" +#include "memory/heapShared.inline.hpp" +#include "memory/iterator.inline.hpp" #include "memory/metadataFactory.hpp" #include "memory/metaspaceClosure.hpp" #include "memory/metaspaceShared.hpp" #include "memory/oopFactory.hpp" #include "oops/compressedOops.inline.hpp" #include "oops/objArrayOop.hpp" +#include "oops/oop.inline.hpp" #include "prims/jvmtiExport.hpp" #include "runtime/arguments.hpp" #include "runtime/java.hpp" @@ -51,6 +54,7 @@ #include "utilities/defaultStream.hpp" #if INCLUDE_G1GC #include "gc/g1/g1CollectedHeap.hpp" +#include "gc/g1/heapRegion.hpp" #endif # include @@ -188,6 +192,9 @@ _shared_path_table_size = mapinfo->_shared_path_table_size; _shared_path_table = mapinfo->_shared_path_table; _shared_path_entry_size = mapinfo->_shared_path_entry_size; + if (MetaspaceShared::is_heap_object_archiving_allowed()) { + _g1_reserved = G1CollectedHeap::heap()->g1_reserved(); + } // The following fields are for sanity checks for whether this archive // will function correctly with this JVM and the bootclasspath it's @@ -525,8 +532,7 @@ } size_t len = lseek(fd, 0, SEEK_END); - struct FileMapInfo::FileMapHeader::space_info* si = - &_header->_space[MetaspaceShared::last_valid_region]; + FileMapHeader::space_info* si = space_at(MetaspaceShared::last_valid_region); // The last space might be empty if (si->_file_offset > len || len - si->_file_offset < si->_used) { fail_continue("The shared archive file has been truncated."); @@ -607,7 +613,7 @@ void FileMapInfo::write_region(int region, char* base, size_t size, bool read_only, bool allow_exec) { - struct FileMapInfo::FileMapHeader::space_info* si = &_header->_space[region]; + FileMapHeader::space_info* si = space_at(region); if (_file_open) { guarantee(si->_file_offset == _file_offset, "file offset mismatch."); @@ -661,13 +667,14 @@ // "_" represented unused spaced in the heap region. // // -// |ah0 | ah1 | ah2| ...... | ahn | +// |ah0 | ah1 | ah2| ...... | ahn| // |XXXXXX|__ |XXXXX|XXXX|XXXXXXXX|XXXX| // |<-r0->| |<- r1 ----------------->| // ^^^ // | // +-- gap size_t FileMapInfo::write_archive_heap_regions(GrowableArray *heap_mem, + GrowableArray *oopmaps, int first_region_id, int max_num_regions) { assert(max_num_regions <= 2, "Only support maximum 2 memory regions"); @@ -692,6 +699,10 @@ log_info(cds)("Archive heap region %d " INTPTR_FORMAT " - " INTPTR_FORMAT " = " SIZE_FORMAT_W(8) " bytes", i, p2i(start), p2i(start + size), size); write_region(i, start, size, false, false); + if (size > 0) { + space_at(i)->_oopmap = oopmaps->at(arr_idx)._oopmap; + space_at(i)->_oopmap_size_in_bits = oopmaps->at(arr_idx)._oopmap_size_in_bits; + } } return total_size; } @@ -760,7 +771,7 @@ // Remap the shared readonly space to shared readwrite, private. bool FileMapInfo::remap_shared_readonly_as_readwrite() { int idx = MetaspaceShared::ro; - struct FileMapInfo::FileMapHeader::space_info* si = &_header->_space[idx]; + FileMapHeader::space_info* si = space_at(idx); if (!si->_read_only) { // the space is already readwrite so we are done return true; @@ -812,7 +823,7 @@ char* FileMapInfo::map_region(int i, char** top_ret) { assert(!MetaspaceShared::is_heap_region(i), "sanity"); - struct FileMapInfo::FileMapHeader::space_info* si = &_header->_space[i]; + FileMapHeader::space_info* si = space_at(i); size_t used = si->_used; size_t alignment = os::vm_allocation_granularity(); size_t size = align_up(used, alignment); @@ -846,12 +857,49 @@ return base; } + +address FileMapInfo::FileMapHeader::space_info::decode_start_address(bool with_current_oop_encoding_mode) { + if (with_current_oop_encoding_mode) { + return (address)CompressedOops::decode_not_null(offset()); + } else { + return (address)HeapShared::decode_with_archived_oop_encoding_mode(offset()); + } +} + static MemRegion *string_ranges = NULL; static MemRegion *open_archive_heap_ranges = NULL; static int num_string_ranges = 0; static int num_open_archive_heap_ranges = 0; #if INCLUDE_CDS_JAVA_HEAP +bool FileMapInfo::has_heap_regions() { + return (_header->_space[MetaspaceShared::first_string]._used > 0); +} + +// Returns the address range where the heap regions would occupy using the +// current oop encoding mode. +MemRegion FileMapInfo::get_heap_regions_range_with_current_oop_encoding_mode() { + address start = (address) max_uintx; + address end = NULL; + + for (int i = MetaspaceShared::first_string; i <= MetaspaceShared::last_valid_region; i++) { + FileMapHeader::space_info* si = space_at(i); + size_t size = si->_used; + if (size > 0) { + address s = si->start_address_with_current_oop_encoding_mode(); + address e = s + size; + if (start > s) { + start = s; + } + if (end < e) { + end = e; + } + } + } + assert(end != NULL, "must have at least one used heap region"); + return MemRegion((HeapWord*)start, (HeapWord*)end); +} + // // Map the shared string objects and open archive heap objects to the runtime // java heap. @@ -866,56 +914,110 @@ // During runtime execution, out-going references to any other java heap // regions may be added. GC may mark and update references in the mapped // open archive objects. -void FileMapInfo::map_heap_regions() { - if (MetaspaceShared::is_heap_object_archiving_allowed()) { - log_info(cds)("Archived narrow_oop_mode = %d, narrow_oop_base = " PTR_FORMAT ", narrow_oop_shift = %d", - narrow_oop_mode(), p2i(narrow_oop_base()), narrow_oop_shift()); - log_info(cds)("Archived narrow_klass_base = " PTR_FORMAT ", narrow_klass_shift = %d", - p2i(narrow_klass_base()), narrow_klass_shift()); - - // Check that all the narrow oop and klass encodings match the archive - if (narrow_oop_mode() != Universe::narrow_oop_mode() || - narrow_oop_base() != Universe::narrow_oop_base() || - narrow_oop_shift() != Universe::narrow_oop_shift() || - narrow_klass_base() != Universe::narrow_klass_base() || - narrow_klass_shift() != Universe::narrow_klass_shift()) { - if (log_is_enabled(Info, cds) && _header->_space[MetaspaceShared::first_string]._used > 0) { - log_info(cds)("Cached heap data from the CDS archive is being ignored. " - "The current CompressedOops/CompressedClassPointers encoding differs from " - "that archived due to heap size change. The archive was dumped using max heap " - "size " UINTX_FORMAT "M.", max_heap_size()/M); - log_info(cds)("Current narrow_oop_mode = %d, narrow_oop_base = " PTR_FORMAT ", narrow_oop_shift = %d", - Universe::narrow_oop_mode(), p2i(Universe::narrow_oop_base()), - Universe::narrow_oop_shift()); - log_info(cds)("Current narrow_klass_base = " PTR_FORMAT ", narrow_klass_shift = %d", - p2i(Universe::narrow_klass_base()), Universe::narrow_klass_shift()); - } - } else { - // First, map string regions as closed archive heap regions. - // GC does not write into the regions. - if (map_heap_data(&string_ranges, - MetaspaceShared::first_string, - MetaspaceShared::max_strings, - &num_string_ranges)) { - StringTable::set_shared_string_mapped(); - - // Now, map open_archive heap regions, GC can write into the regions. - if (map_heap_data(&open_archive_heap_ranges, - MetaspaceShared::first_open_archive_heap_region, - MetaspaceShared::max_open_archive_heap_region, - &num_open_archive_heap_ranges, - true /* open */)) { - MetaspaceShared::set_open_archive_heap_region_mapped(); - } - } - } +void FileMapInfo::map_heap_regions_impl() { + if (!MetaspaceShared::is_heap_object_archiving_allowed()) { + log_info(cds)("CDS heap data is being ignored. UseG1GC, " + "UseCompressedOops and UseCompressedClassPointers are required."); + return; + } + + MemRegion g1_reserved = G1CollectedHeap::heap()->g1_reserved(); + + log_info(cds)("CDS archive was created with max heap size = " UINTX_FORMAT "M, and the following configuration:", + max_heap_size()/M); + log_info(cds)(" narrow_klass_base = " PTR_FORMAT ", narrow_klass_shift = %d", + p2i(narrow_klass_base()), narrow_klass_shift()); + log_info(cds)(" narrow_oop_mode = %d, narrow_oop_base = " PTR_FORMAT ", narrow_oop_shift = %d", + narrow_oop_mode(), p2i(narrow_oop_base()), narrow_oop_shift()); + + log_info(cds)("The current max heap size = " UINTX_FORMAT "M, HeapRegion::GrainBytes = " UINTX_FORMAT, + g1_reserved.byte_size()/M, HeapRegion::GrainBytes); + log_info(cds)(" narrow_klass_base = " PTR_FORMAT ", narrow_klass_shift = %d", + p2i(Universe::narrow_klass_base()), Universe::narrow_klass_shift()); + log_info(cds)(" narrow_oop_mode = %d, narrow_oop_base = " PTR_FORMAT ", narrow_oop_shift = %d", + Universe::narrow_oop_mode(), p2i(Universe::narrow_oop_base()), Universe::narrow_oop_shift()); + + if (narrow_klass_base() != Universe::narrow_klass_base() || + narrow_klass_shift() != Universe::narrow_klass_shift()) { + log_info(cds)("CDS heap data cannot be used because the archive was created with an incompatible heap size."); + return; + } + + if (narrow_oop_mode() != Universe::narrow_oop_mode() || + narrow_oop_base() != Universe::narrow_oop_base() || + narrow_oop_shift() != Universe::narrow_oop_shift()) { + log_info(cds)("CDS heap data need to be relocated because the archive was created with an incompatible heap size."); + _heap_pointers_need_patching = true; } else { - if (log_is_enabled(Info, cds) && _header->_space[MetaspaceShared::first_string]._used > 0) { - log_info(cds)("Cached heap data from the CDS archive is being ignored. UseG1GC, " - "UseCompressedOops and UseCompressedClassPointers are required."); + MemRegion range = get_heap_regions_range_with_current_oop_encoding_mode(); + if (!g1_reserved.contains(range)) { + log_info(cds)("CDS heap data need to be relocated because"); + log_info(cds)("the desired range " PTR_FORMAT " - " PTR_FORMAT, p2i(range.start()), p2i(range.end())); + log_info(cds)("is outside of the heap " PTR_FORMAT " - " PTR_FORMAT, p2i(g1_reserved.start()), p2i(g1_reserved.end())); + _heap_pointers_need_patching = true; } } + ptrdiff_t delta = 0; + if (_heap_pointers_need_patching) { + // dumptime heap end ------------v + // [ |archived heap regions| ] runtime heap end ------v + // [ |archived heap regions| ] + // |<-----delta-------------------->| + // + // At dump time, the archived heap regions were near the top of the heap. + // At run time, they may not be inside the heap, so we move them so + // that they are now near the top of the runtime time. This can be done by + // the simple math of adding the delta as shown above. + address dumptime_heap_end = (address)_header->_g1_reserved.end(); + address runtime_heap_end = (address)g1_reserved.end(); + delta = runtime_heap_end - dumptime_heap_end; + } + + log_info(cds)("CDS heap data relocation delta = " INTX_FORMAT " bytes", delta); + HeapShared::init_narrow_oop_decoding(narrow_oop_base() + delta, narrow_oop_shift()); + + FileMapHeader::space_info* si = space_at(MetaspaceShared::first_string); + address relocated_strings_bottom = si->start_address_with_archived_oop_encoding_mode(); + if (!is_aligned(relocated_strings_bottom + delta, HeapRegion::GrainBytes)) { + // Align the bottom of the string regions at G1 region boundary. This will avoid + // the situation where the highest open region and the lowest string region sharing + // the same G1 region. Otherwise we will fail to map the open regions. + size_t align = size_t(relocated_strings_bottom) % HeapRegion::GrainBytes; + delta -= align; + assert(is_aligned(relocated_strings_bottom + delta, HeapRegion::GrainBytes), "must be"); + + log_info(cds)("CDS heap data need to be relocated lower by a further " UINTX_FORMAT + " bytes to be aligned with HeapRegion::GrainBytes", align); + + HeapShared::init_narrow_oop_decoding(narrow_oop_base() + delta, narrow_oop_shift()); + _heap_pointers_need_patching = true; + } + + // First, map string regions as closed archive heap regions. + // GC does not write into the regions. + if (map_heap_data(&string_ranges, + MetaspaceShared::first_string, + MetaspaceShared::max_strings, + &num_string_ranges)) { + StringTable::set_shared_string_mapped(); + + // Now, map open_archive heap regions, GC can write into the regions. + if (map_heap_data(&open_archive_heap_ranges, + MetaspaceShared::first_open_archive_heap_region, + MetaspaceShared::max_open_archive_heap_region, + &num_open_archive_heap_ranges, + true /* open */)) { + MetaspaceShared::set_open_archive_heap_region_mapped(); + } + } +} + +void FileMapInfo::map_heap_regions() { + if (has_heap_regions()) { + map_heap_regions_impl(); + } + if (!StringTable::shared_string_mapped()) { assert(string_ranges == NULL && num_string_ranges == 0, "sanity"); } @@ -928,19 +1030,19 @@ bool FileMapInfo::map_heap_data(MemRegion **heap_mem, int first, int max, int* num, bool is_open_archive) { MemRegion * regions = new MemRegion[max]; - struct FileMapInfo::FileMapHeader::space_info* si; + FileMapHeader::space_info* si; int region_num = 0; for (int i = first; i < first + max; i++) { - si = &_header->_space[i]; - size_t used = si->_used; - if (used > 0) { - size_t size = used; - char* requested_addr = (char*)((void*)CompressedOops::decode_not_null( - (narrowOop)si->_addr._offset)); - regions[region_num] = MemRegion((HeapWord*)requested_addr, size / HeapWordSize); + si = space_at(i); + size_t size = si->_used; + if (size > 0) { + HeapWord* start = (HeapWord*)si->start_address_with_archived_oop_encoding_mode(); + regions[region_num] = MemRegion(start, size / HeapWordSize); region_num ++; + log_info(cds)("Trying to map heap data: region[%d] at " INTPTR_FORMAT ", size = %8d bytes", + i, p2i(start), int(size)); } } @@ -950,16 +1052,14 @@ // Check that ranges are within the java heap if (!G1CollectedHeap::heap()->check_archive_addresses(regions, region_num)) { - log_info(cds)("UseSharedSpaces: Unable to allocate region, " - "range is not within java heap."); + log_info(cds)("UseSharedSpaces: Unable to allocate region, range is not within java heap."); return false; } // allocate from java heap if (!G1CollectedHeap::heap()->alloc_archive_regions( regions, region_num, is_open_archive)) { - log_info(cds)("UseSharedSpaces: Unable to allocate region, " - "java heap range is already in use."); + log_info(cds)("UseSharedSpaces: Unable to allocate region, java heap range is already in use."); return false; } @@ -967,7 +1067,7 @@ // for mapped regions as they are part of the reserved java heap, which is // already recorded. for (int i = 0; i < region_num; i++) { - si = &_header->_space[first + i]; + si = space_at(first + i); char* addr = (char*)regions[i].start(); char* base = os::map_memory(_fd, _full_path, si->_file_offset, addr, regions[i].byte_size(), si->_read_only, @@ -975,7 +1075,9 @@ if (base == NULL || base != addr) { // dealloc the regions from java heap dealloc_archive_heap_regions(regions, region_num); - log_info(cds)("UseSharedSpaces: Unable to map at required address in java heap."); + log_info(cds)("UseSharedSpaces: Unable to map at required address in java heap. " + INTPTR_FORMAT ", size = " UINTX_FORMAT " bytes", + p2i(addr), regions[i].byte_size()); return false; } } @@ -1003,6 +1105,31 @@ return true; } +void FileMapInfo::patch_archived_heap_embedded_pointers() { + if (!_heap_pointers_need_patching) { + return; + } + + patch_archived_heap_embedded_pointers(string_ranges, + num_string_ranges, + MetaspaceShared::first_string); + + patch_archived_heap_embedded_pointers(open_archive_heap_ranges, + num_open_archive_heap_ranges, + MetaspaceShared::first_open_archive_heap_region); +} + +void FileMapInfo::patch_archived_heap_embedded_pointers(MemRegion* ranges, int num_ranges, + int first_region_idx) { + for (int i=0; i_oopmap, + si->_oopmap_size_in_bits); + } +} + +// This internally allocates objects using SystemDictionary::Object_klass(), so it +// must be called after the well-known classes are resolved. void FileMapInfo::fixup_mapped_heap_regions() { // If any string regions were found, call the fill routine to make them parseable. // Note that string_ranges may be non-NULL even if no ranges were found. @@ -1057,7 +1184,7 @@ void FileMapInfo::unmap_region(int i) { assert(!MetaspaceShared::is_heap_region(i), "sanity"); - struct FileMapInfo::FileMapHeader::space_info* si = &_header->_space[i]; + FileMapHeader::space_info* si = space_at(i); size_t used = si->_used; size_t size = align_up(used, os::vm_allocation_granularity()); @@ -1086,6 +1213,7 @@ FileMapInfo* FileMapInfo::_current_info = NULL; +bool FileMapInfo::_heap_pointers_need_patching = false; Array* FileMapInfo::_shared_path_table = NULL; int FileMapInfo::_shared_path_table_size = 0; size_t FileMapInfo::_shared_path_entry_size = 0x1234baad; @@ -1117,8 +1245,9 @@ char* FileMapInfo::FileMapHeader::region_addr(int idx) { if (MetaspaceShared::is_heap_region(idx)) { + assert(DumpSharedSpaces, "The following doesn't work at runtime"); return _space[idx]._used > 0 ? - (char*)((void*)CompressedOops::decode_not_null((narrowOop)_space[idx]._addr._offset)) : NULL; + (char*)_space[idx].start_address_with_current_oop_encoding_mode() : NULL; } else { return _space[idx]._addr._base; } @@ -1231,27 +1360,18 @@ return false; } -void FileMapInfo::print_shared_spaces() { - tty->print_cr("Shared Spaces:"); - for (int i = 0; i < MetaspaceShared::n_regions; i++) { - struct FileMapInfo::FileMapHeader::space_info* si = &_header->_space[i]; - char *base = _header->region_addr(i); - tty->print(" %s " INTPTR_FORMAT "-" INTPTR_FORMAT, - shared_region_name[i], - p2i(base), p2i(base + si->_used)); - } -} - // Unmap mapped regions of shared space. void FileMapInfo::stop_sharing_and_unmap(const char* msg) { FileMapInfo *map_info = FileMapInfo::current_info(); if (map_info) { map_info->fail_continue("%s", msg); for (int i = 0; i < MetaspaceShared::num_non_heap_spaces; i++) { - char *addr = map_info->_header->region_addr(i); - if (addr != NULL && !MetaspaceShared::is_heap_region(i)) { - map_info->unmap_region(i); - map_info->_header->_space[i]._addr._base = NULL; + if (!MetaspaceShared::is_heap_region(i)) { + char *addr = map_info->_header->region_addr(i); + if (addr != NULL) { + map_info->unmap_region(i); + map_info->_header->_space[i]._addr._base = NULL; + } } } // Dealloc the archive heap regions only without unmapping. The regions are part