--- old/src/hotspot/share/gc/g1/g1Allocator.inline.hpp 2018-11-19 17:18:48.854052500 -0800 +++ new/src/hotspot/share/gc/g1/g1Allocator.inline.hpp 2018-11-19 17:18:47.959092100 -0800 @@ -97,7 +97,7 @@ } _archive_check_enabled = true; - size_t length = static_cast (Universe::heap())->max_reserved_capacity(); + size_t length = G1CollectedHeap::heap()->max_reserved_capacity(); _closed_archive_region_map.initialize((HeapWord*)Universe::heap()->base(), (HeapWord*)Universe::heap()->base() + length, HeapRegion::GrainBytes); --- old/src/hotspot/share/gc/g1/g1Arguments.cpp 2018-11-19 17:19:00.227828000 -0800 +++ new/src/hotspot/share/gc/g1/g1Arguments.cpp 2018-11-19 17:18:59.322512700 -0800 @@ -147,5 +147,9 @@ } CollectedHeap* G1Arguments::create_heap() { + if(AllocateOldGenAt != NULL) { + return create_heap_with_policy(); + } return create_heap_with_policy(); + } --- old/src/hotspot/share/gc/g1/g1CollectedHeap.cpp 2018-11-19 17:19:11.572376700 -0800 +++ new/src/hotspot/share/gc/g1/g1CollectedHeap.cpp 2018-11-19 17:19:10.639281100 -0800 @@ -1025,6 +1025,8 @@ abandon_collection_set(collection_set()); tear_down_region_sets(false /* free_list_only */); + + hrm()->prepare_for_full_collection_start(); } void G1CollectedHeap::verify_before_full_collection(bool explicit_gc) { @@ -1036,6 +1038,8 @@ } void G1CollectedHeap::prepare_heap_for_mutators() { + hrm()->prepare_for_full_collection_end(); + // Delete metaspaces for unloaded class loaders and clean up loader_data graph ClassLoaderDataGraph::purge(); MetaspaceUtils::verify_metrics(); @@ -1486,7 +1490,7 @@ _bot(NULL), _listener(), _hrm(NULL), - _is_hetero_heap(false), + _is_hetero_heap(AllocateOldGenAt != NULL), _allocator(NULL), _verifier(NULL), _summary_bytes_used(0), @@ -1619,7 +1623,7 @@ guarantee(HeapWordSize == wordSize, "HeapWordSize must equal wordSize"); size_t init_byte_size = collector_policy()->initial_heap_byte_size(); - size_t max_byte_size = collector_policy()->max_heap_byte_size(); + size_t max_byte_size = g1_collector_policy()->heap_reservation_size_bytes(); size_t heap_alignment = collector_policy()->heap_alignment(); // Ensure that the sizes are properly aligned. @@ -1639,10 +1643,6 @@ // address that was requested (i.e. the preferred heap base). // If this happens then we could end up using a non-optimal // compressed oops mode. - if (AllocateOldGenAt != NULL) { - _is_hetero_heap = true; - max_byte_size *= 2; - } ReservedSpace heap_rs = Universe::reserve_heap(max_byte_size, heap_alignment); @@ -1693,6 +1693,11 @@ HeapRegion::GrainBytes, 1, mtJavaHeap); + if(heap_storage == NULL) { + vm_shutdown_during_initialization("Could not initialize G1 heap"); + return JNI_ERR; + } + os::trace_page_sizes("Heap", collector_policy()->min_heap_byte_size(), max_byte_size, @@ -1723,12 +1728,8 @@ G1RegionToSpaceMapper* next_bitmap_storage = create_aux_memory_mapper("Next Bitmap", bitmap_size, G1CMBitMap::heap_map_factor()); - if (is_hetero_heap()) { - _hrm = new HeapRegionManagerForHeteroHeap((uint)((max_byte_size / 2) / HeapRegion::GrainBytes /*heap size as num of regions*/)); - } - else { - _hrm = new HeapRegionManager(); - } + _hrm = HeapRegionManager::create_manager(this, collector_policy()); + _hrm->initialize(heap_storage, prev_bitmap_storage, next_bitmap_storage, bot_storage, cardtable_storage, card_counts_storage); _card_table->initialize(cardtable_storage); // Do later initialization work for concurrent refinement. @@ -1782,6 +1783,11 @@ // Perform any initialization actions delegated to the policy. g1_policy()->init(this, &_collection_set); + // Now we know the target length of young list. So adjust the heap to provision that many regions on dram. + if (is_hetero_heap()) { + static_cast(hrm())->adjust_dram_regions((uint)g1_policy()->young_list_target_length(), workers()); + } + jint ecode = initialize_concurrent_refinement(); if (ecode != JNI_OK) { return ecode; @@ -1920,6 +1926,10 @@ return _collector_policy; } +G1CollectorPolicy* G1CollectedHeap::g1_collector_policy() const { + return _collector_policy; +} + SoftRefPolicy* G1CollectedHeap::soft_ref_policy() { return &_soft_ref_policy; } @@ -2519,6 +2529,10 @@ if (full) { // Update the number of full collections that have been completed. increment_old_marking_cycles_completed(false /* concurrent */); + // Now we know the target length of young list. So adjust the heap to provision that many regions on dram. + if (is_hetero_heap()) { + static_cast(hrm())->adjust_dram_regions((uint)g1_policy()->young_list_target_length(), workers()); + } } // We are at the end of the GC. Total collections has already been increased. --- old/src/hotspot/share/gc/g1/g1CollectedHeap.hpp 2018-11-19 17:19:23.089011600 -0800 +++ new/src/hotspot/share/gc/g1/g1CollectedHeap.hpp 2018-11-19 17:19:22.189577700 -0800 @@ -43,7 +43,7 @@ #include "gc/g1/g1SurvivorRegions.hpp" #include "gc/g1/g1YCTypes.hpp" #include "gc/g1/heapRegionManager.hpp" -#include "gc/g1/heapRegionManagerForHeteroHeap.hpp" +#include "gc/g1/heterogeneousHeapRegionManager.hpp" #include "gc/g1/heapRegionSet.hpp" #include "gc/shared/barrierSet.hpp" #include "gc/shared/collectedHeap.hpp" @@ -964,6 +964,7 @@ G1CollectionSet* collection_set() { return &_collection_set; } virtual CollectorPolicy* collector_policy() const; + virtual G1CollectorPolicy* g1_collector_policy() const; virtual SoftRefPolicy* soft_ref_policy(); --- old/src/hotspot/share/gc/g1/g1CollectedHeap.inline.hpp 2018-11-19 17:19:34.367539000 -0800 +++ new/src/hotspot/share/gc/g1/g1CollectedHeap.inline.hpp 2018-11-19 17:19:33.431736700 -0800 @@ -60,7 +60,7 @@ inline HeapRegion* G1CollectedHeap::region_at(uint index) const { return _hrm->at(index); } // Return the region with the given index, or NULL if unmapped. It assumes the index is valid. -inline HeapRegion* G1CollectedHeap::region_at_or_null(uint index) const { return _hrm.at_or_null(index); } +inline HeapRegion* G1CollectedHeap::region_at_or_null(uint index) const { return _hrm->at_or_null(index); } inline HeapRegion* G1CollectedHeap::next_region_in_humongous(HeapRegion* hr) const { return _hrm->next_region_in_humongous(hr); --- old/src/hotspot/share/gc/g1/g1CollectorPolicy.cpp 2018-11-19 17:19:46.191766400 -0800 +++ new/src/hotspot/share/gc/g1/g1CollectorPolicy.cpp 2018-11-19 17:19:45.308032300 -0800 @@ -55,3 +55,19 @@ size_t page_size = UseLargePages ? os::large_page_size() : os::vm_page_size(); _heap_alignment = MAX3(card_table_alignment, _space_alignment, page_size); } + +size_t G1CollectorPolicy::heap_reservation_size_bytes() { + return _max_heap_byte_size; +} + +G1HeteroCollectorPolicy::G1HeteroCollectorPolicy() : _heap_reservation_size_bytes(MaxHeapSize*2) { +} + +void G1HeteroCollectorPolicy::initialize_size_info() { + G1CollectorPolicy::initialize_size_info(); + _heap_reservation_size_bytes = 2 * _max_heap_byte_size; +} + +size_t G1HeteroCollectorPolicy::heap_reservation_size_bytes() { + return _heap_reservation_size_bytes; +} \ No newline at end of file --- old/src/hotspot/share/gc/g1/g1CollectorPolicy.hpp 2018-11-19 17:19:57.413186700 -0800 +++ new/src/hotspot/share/gc/g1/g1CollectorPolicy.hpp 2018-11-19 17:19:56.532387900 -0800 @@ -38,6 +38,17 @@ public: G1CollectorPolicy(); + virtual size_t heap_reservation_size_bytes(); +}; + +class G1HeteroCollectorPolicy : public G1CollectorPolicy { + size_t _heap_reservation_size_bytes; + +protected: + void initialize_size_info(); +public: + G1HeteroCollectorPolicy(); + size_t heap_reservation_size_bytes(); }; #endif // SHARE_VM_GC_G1_G1COLLECTORPOLICY_HPP --- old/src/hotspot/share/gc/g1/g1FullCollector.cpp 2018-11-19 17:20:09.151890200 -0800 +++ new/src/hotspot/share/gc/g1/g1FullCollector.cpp 2018-11-19 17:20:08.233035500 -0800 @@ -165,11 +165,6 @@ } void G1FullCollector::collect() { - - if (_heap->is_hetero_heap()) { - static_cast (_heap->_hrm)->prepare_for_full_collection_start(); - } - phase1_mark_live_objects(); verify_after_marking(); @@ -181,10 +176,6 @@ phase3_adjust_pointers(); phase4_do_compaction(); - - if (_heap->is_hetero_heap()) { - static_cast (_heap->_hrm)->prepare_for_full_collection_end(); - } } void G1FullCollector::complete_collection() { @@ -202,6 +193,7 @@ _heap->prepare_heap_for_mutators(); _heap->g1_policy()->record_full_collection_end(); + _heap->gc_epilogue(true); _heap->verify_after_full_collection(); --- old/src/hotspot/share/gc/g1/g1Policy.cpp 2018-11-19 17:20:20.433241800 -0800 +++ new/src/hotspot/share/gc/g1/g1Policy.cpp 2018-11-19 17:20:19.525029900 -0800 @@ -219,10 +219,6 @@ YoungTargetLengths young_lengths = young_list_target_lengths(rs_lengths); _young_list_target_length = young_lengths.first; - // Resize dram regions set if called after full collection end. - if(_g1h->is_hetero_heap() && (Thread::current()->is_VM_thread() || Heap_lock->owned_by_self())) { - static_cast (_g1h->hrm())->resize_dram_regions(_young_list_target_length, _g1h->workers()); - } return young_lengths.second; } --- old/src/hotspot/share/gc/g1/g1RegionToSpaceMapper.cpp 2018-11-19 17:20:31.579841400 -0800 +++ new/src/hotspot/share/gc/g1/g1RegionToSpaceMapper.cpp 2018-11-19 17:20:30.663169400 -0800 @@ -22,14 +22,17 @@ * */ +#include "logging/log.hpp" #include "precompiled.hpp" #include "gc/g1/g1BiasedArray.hpp" #include "gc/g1/g1RegionToSpaceMapper.hpp" #include "memory/allocation.inline.hpp" #include "memory/virtualspace.hpp" +#include "runtime/java.hpp" #include "services/memTracker.hpp" #include "utilities/align.hpp" #include "utilities/bitMap.inline.hpp" +#include "utilities/formatBuffer.hpp" G1RegionToSpaceMapper::G1RegionToSpaceMapper(ReservedSpace rs, size_t used_size, @@ -170,23 +173,24 @@ } } -void G1RegionToHeteroSpaceMapper::map_nvdimm_space(ReservedSpace rs) { +static bool map_nvdimm_space(ReservedSpace rs) { assert(AllocateOldGenAt != NULL, ""); int _backing_fd = os::create_file_for_heap(AllocateOldGenAt); if (_backing_fd == -1) { - vm_exit_during_initialization( - err_msg("Could not create file for Old generation at location %s", AllocateOldGenAt)); + return false; + log_error(gc, init)("Could not create file for Old generation at location %s", AllocateOldGenAt); } // commit this memory in nv-dimm char* ret = os::attempt_reserve_memory_at(rs.size(), rs.base(), _backing_fd); - //char* ret = os::replace_existing_mapping_with_file_mapping(rs.base(), rs.size(), _backing_fd); + if (ret != rs.base()) { if (ret != NULL) { os::unmap_memory(rs.base(), rs.size()); } - vm_exit_during_initialization( - err_msg("Error in mapping Old Gen to given AllocateOldGenAt = %s", AllocateOldGenAt)); + log_error(gc, init)("Error in mapping Old Gen to given AllocateOldGenAt = %s", AllocateOldGenAt); + return false; } + return true; } G1RegionToHeteroSpaceMapper::G1RegionToHeteroSpaceMapper(ReservedSpace rs, @@ -196,22 +200,19 @@ size_t commit_factor, MemoryType type) : G1RegionToSpaceMapper(rs, actual_size, page_size, alloc_granularity, commit_factor, type), - _num_committed_dram(0), _num_committed_nvdimm(0) { + _num_committed_dram(0), _num_committed_nvdimm(0), _success(false) { assert(actual_size == 2 * MaxHeapSize, "For 2-way heterogenuous heap, reserved space is two times MaxHeapSize"); - // Since we need to split the reserved space in half and map second half to file in NV-DIMM, we need to release the reserved memory first + // Since we need to re-map the reserved space - 'Xmx' to nv-dimm and 'Xmx' to dram, we need to release the reserved memory first. // Because on some OSes (e.g. Windows) you cannot do a file mapping on memory reserved with regular mapping. os::release_memory(rs.base(), rs.size()); - - /* Toggle HeteroHeap - // We map first part of size Xmx to DRAM. - ReservedSpace rs_dram = rs.first_part(MaxHeapSize); - // Second half of reserved memory is mapped to NV-DIMM. - ReservedSpace rs_nvdimm = rs.last_part(MaxHeapSize);*/ - // We map first part of size Xmx to NVDIMM. + // First half of size Xmx is for nv-dimm. ReservedSpace rs_nvdimm = rs.first_part(MaxHeapSize); - // Second half of reserved memory is mapped to DRAM. + assert(rs_nvdimm.base() == rs.base(), "We should get the same base address"); + + // Second half of reserved memory is mapped to dram. ReservedSpace rs_dram = rs.last_part(MaxHeapSize); + assert(rs_dram.size() == rs_nvdimm.size() && rs_nvdimm.size() == MaxHeapSize, "They all should be same"); // Reserve dram memory @@ -220,11 +221,15 @@ if (base != NULL) { os::release_memory(base, rs_dram.size()); } - vm_exit_during_initialization(err_msg("Error in allocating heap")); + log_error(gc, init)("Error in re-mapping memory on dram during G1 heterogenous memory initialization"); + return; } // We reserve and commit this entire space to NV-DIMM. - map_nvdimm_space(rs_nvdimm); + if (!map_nvdimm_space(rs_nvdimm)) { + log_error(gc, init)("Error in re-mapping memory to nv-dimm during G1 heterogenous memory initialization"); + return; + } if (alloc_granularity >= (page_size * commit_factor)) { _dram_mapper = new G1RegionsLargerThanCommitSizeMapper(rs_dram, rs_dram.size(), page_size, alloc_granularity, commit_factor, type); @@ -233,19 +238,14 @@ _dram_mapper = new G1RegionsSmallerThanCommitSizeMapper(rs_dram, rs_dram.size(), page_size, alloc_granularity, commit_factor, type); } -/* Toggle HeteroHeap - _start_index_of_nvdimm = (uint)(rs_dram.size() / alloc_granularity); - _start_index_of_dram = 0; */ _start_index_of_nvdimm = 0; _start_index_of_dram = (uint)(rs_nvdimm.size() / alloc_granularity); + _success = true; } void G1RegionToHeteroSpaceMapper::commit_regions(uint start_idx, size_t num_regions, WorkGang* pretouch_gang) { uint end_idx = (start_idx + (uint)num_regions - 1); - /* Toggle HeteroHeap - uint num_nvdimm = end_idx >= _start_index_of_nvdimm ? MIN2((end_idx - _start_index_of_nvdimm + 1), (uint)num_regions) : 0; - uint num_dram = (uint)num_regions - num_nvdimm;*/ uint num_dram = end_idx >= _start_index_of_dram ? MIN2((end_idx - _start_index_of_dram + 1), (uint)num_regions) : 0; uint num_nvdimm = (uint)num_regions - num_dram; @@ -254,8 +254,6 @@ _num_committed_nvdimm += num_nvdimm; } if (num_dram > 0) { - /* Toggle HeteroHeap - _dram_mapper->commit_regions(start_idx, num_dram, pretouch_gang); */ _dram_mapper->commit_regions(start_idx > _start_index_of_dram ? (start_idx - _start_index_of_dram) : 0, num_dram, pretouch_gang); _num_committed_dram += num_dram; } @@ -263,9 +261,6 @@ void G1RegionToHeteroSpaceMapper::uncommit_regions(uint start_idx, size_t num_regions) { uint end_idx = (start_idx + (uint)num_regions - 1); - /* Toggle HeterHeap - uint num_nvdimm = end_idx >= _start_index_of_nvdimm ? MIN2((end_idx - _start_index_of_nvdimm + 1), (uint)num_regions) : 0; - uint num_dram = (uint)num_regions - num_nvdimm;*/ uint num_dram = end_idx >= _start_index_of_dram ? MIN2((end_idx - _start_index_of_dram + 1), (uint)num_regions) : 0; uint num_nvdimm = (uint)num_regions - num_dram; @@ -280,11 +275,11 @@ } } -uint G1RegionToHeteroSpaceMapper::num_committed_dram() { +uint G1RegionToHeteroSpaceMapper::num_committed_dram() const { return _num_committed_dram; } -uint G1RegionToHeteroSpaceMapper::num_committed_nvdimm() { +uint G1RegionToHeteroSpaceMapper::num_committed_nvdimm() const { return _num_committed_nvdimm; } @@ -295,7 +290,12 @@ size_t commit_factor, MemoryType type) { if (AllocateOldGenAt != NULL) { - return new G1RegionToHeteroSpaceMapper(rs, actual_size, page_size, region_granularity, commit_factor, type); + G1RegionToHeteroSpaceMapper* mapper = new G1RegionToHeteroSpaceMapper(rs, actual_size, page_size, region_granularity, commit_factor, type); + if (!mapper->success()) { + delete mapper; + return NULL; + } + return (G1RegionToSpaceMapper*)mapper; } else { return create_mapper(rs, actual_size, page_size, region_granularity, commit_factor, type); } --- old/src/hotspot/share/gc/g1/g1RegionToSpaceMapper.hpp 2018-11-19 17:20:42.905878900 -0800 +++ new/src/hotspot/share/gc/g1/g1RegionToSpaceMapper.hpp 2018-11-19 17:20:41.968547700 -0800 @@ -106,13 +106,13 @@ uint _num_committed_nvdimm; uint _start_index_of_nvdimm; uint _start_index_of_dram; - - void map_nvdimm_space(ReservedSpace rs); + bool _success; public: G1RegionToHeteroSpaceMapper(ReservedSpace rs, size_t used_size, size_t page_size, size_t region_granularity, size_t commit_factor, MemoryType type); - uint num_committed_dram(); - uint num_committed_nvdimm(); + uint num_committed_dram() const; + uint num_committed_nvdimm() const; + bool success() { return _success; } virtual void commit_regions(uint start_idx, size_t num_regions = 1, WorkGang* pretouch_workers = NULL); virtual void uncommit_regions(uint start_idx, size_t num_regions = 1); --- old/src/hotspot/share/gc/g1/g1_globals.hpp 2018-11-19 17:20:54.229201000 -0800 +++ new/src/hotspot/share/gc/g1/g1_globals.hpp 2018-11-19 17:20:53.277383500 -0800 @@ -302,6 +302,15 @@ "Verify the code root lists attached to each heap region.") \ \ develop(bool, G1VerifyBitmaps, false, \ - "Verifies the consistency of the marking bitmaps") + "Verifies the consistency of the marking bitmaps") \ + \ + experimental(uintx, G1YoungExpansionBufferPercent, 10, \ + "When heterogenous heap is enabled by AllocateOldGenAt " \ + "option, after every GC, young gen is re-sized which " \ + "involves system calls to commit/uncommit memory. To " \ + "reduce these calls, we keep a buffer of extra regions to " \ + "absorb small changes in young gen length. This flag takes " \ + "the buffer size as an percentage of young gen length") \ + range(0, 100) #endif // SHARE_VM_GC_G1_G1_GLOBALS_HPP --- old/src/hotspot/share/gc/g1/heapRegionManager.cpp 2018-11-19 17:21:05.453914600 -0800 +++ new/src/hotspot/share/gc/g1/heapRegionManager.cpp 2018-11-19 17:21:04.551167400 -0800 @@ -27,7 +27,9 @@ #include "gc/g1/g1ConcurrentRefine.hpp" #include "gc/g1/heapRegion.hpp" #include "gc/g1/heapRegionManager.inline.hpp" +#include "gc/g1/heterogeneousHeapRegionManager.hpp" #include "gc/g1/heapRegionSet.inline.hpp" +#include "gc/shared/collectorPolicy.hpp" #include "memory/allocation.hpp" #include "utilities/bitMap.inline.hpp" @@ -54,18 +56,25 @@ }; HeapRegionManager::HeapRegionManager() : - _regions(), _heap_mapper(NULL), _prev_bitmap_mapper(NULL), _next_bitmap_mapper(NULL), _bot_mapper(NULL), _cardtable_mapper(NULL), _card_counts_mapper(NULL), - _free_list("Free list", new MasterFreeRegionListChecker()), _available_map(mtGC), _num_committed(0), - _allocated_heapregions_length(0) + _allocated_heapregions_length(0), + _regions(), _heap_mapper(NULL), + _free_list("Free list", new MasterFreeRegionListChecker()) { } +HeapRegionManager* HeapRegionManager::create_manager(G1CollectedHeap* heap, CollectorPolicy* policy) { + if (heap->is_hetero_heap()) { + return new HeterogeneousHeapRegionManager((uint)(policy->max_heap_byte_size() / HeapRegion::GrainBytes) /*heap size as num of regions*/); + } + return new HeapRegionManager(); +} + void HeapRegionManager::initialize(G1RegionToSpaceMapper* heap_storage, G1RegionToSpaceMapper* prev_bitmap, G1RegionToSpaceMapper* next_bitmap, --- old/src/hotspot/share/gc/g1/heapRegionManager.hpp 2018-11-19 17:21:16.719366200 -0800 +++ new/src/hotspot/share/gc/g1/heapRegionManager.hpp 2018-11-19 17:21:15.768424800 -0800 @@ -71,20 +71,12 @@ friend class VMStructs; friend class HeapRegionClaimer; - protected: - G1HeapRegionTable _regions; - G1RegionToSpaceMapper* _heap_mapper; - private: G1RegionToSpaceMapper* _prev_bitmap_mapper; G1RegionToSpaceMapper* _next_bitmap_mapper; G1RegionToSpaceMapper* _bot_mapper; G1RegionToSpaceMapper* _cardtable_mapper; G1RegionToSpaceMapper* _card_counts_mapper; - protected: - FreeRegionList _free_list; - private: - // Each bit in this bitmap indicates that the corresponding region is available // for allocation. CHeapBitMap _available_map; @@ -98,10 +90,6 @@ HeapWord* heap_bottom() const { return _regions.bottom_address_mapped(); } HeapWord* heap_end() const {return _regions.end_address_mapped(); } - protected: - void make_regions_available(uint index, uint num_regions = 1, WorkGang* pretouch_gang = NULL); - void uncommit_regions(uint index, size_t num_regions = 1); - private: // Pass down commit calls to the VirtualSpace. void commit_regions(uint index, size_t num_regions = 1, WorkGang* pretouch_gang = NULL); @@ -121,17 +109,25 @@ // the heap. Returns the length of the sequence found. If this value is zero, no // sequence could be found, otherwise res_idx contains the start index of this range. uint find_empty_from_idx_reverse(uint start_idx, uint* res_idx) const; + +protected: + G1HeapRegionTable _regions; + G1RegionToSpaceMapper* _heap_mapper; + FreeRegionList _free_list; + void make_regions_available(uint index, uint num_regions = 1, WorkGang* pretouch_gang = NULL); + void uncommit_regions(uint index, size_t num_regions = 1); + // Allocate a new HeapRegion for the given index. + HeapRegion* new_heap_region(uint hrm_index); #ifdef ASSERT public: bool is_free(HeapRegion* hr) const; #endif -protected: - // Allocate a new HeapRegion for the given index. - HeapRegion* new_heap_region(uint hrm_index); public: // Empty constructor, we'll initialize it with the initialize() method. HeapRegionManager(); + static HeapRegionManager* create_manager(G1CollectedHeap* heap, CollectorPolicy* policy); + void initialize(G1RegionToSpaceMapper* heap_storage, G1RegionToSpaceMapper* prev_bitmap, G1RegionToSpaceMapper* next_bitmap, @@ -139,6 +135,11 @@ G1RegionToSpaceMapper* cardtable, G1RegionToSpaceMapper* card_counts); + // Prepare heap regions before and after full collection. + // Nothing to be done in this class. + virtual void prepare_for_full_collection_start() {} + virtual void prepare_for_full_collection_end() {} + // Return the "dummy" region used for G1AllocRegion. This is currently a hardwired // new HeapRegion that owns HeapRegion at index 0. Since at the moment we commit // the heap from the lowest address, this region (and its associated data --- old/src/hotspot/share/gc/g1/heapRegionSet.cpp 2018-11-19 17:21:27.883379600 -0800 +++ new/src/hotspot/share/gc/g1/heapRegionSet.cpp 2018-11-19 17:21:26.967232900 -0800 @@ -237,12 +237,11 @@ uint FreeRegionList::num_of_regions_in_range(uint start, uint end) const { HeapRegion* cur = _head; uint num = 0; - bool started = false; - while (cur != NULL && cur->hrm_index() <= end) { - if (!started && cur->hrm_index() >= start) { - started = true; - } - if(started) { + while (cur != NULL) { + uint index = cur->hrm_index(); + if (index > end) { + break; + } else if (index >= start) { num++; } cur = cur->next(); --- old/src/hotspot/share/gc/g1/heapRegionType.hpp 2018-11-19 17:21:39.065690600 -0800 +++ new/src/hotspot/share/gc/g1/heapRegionType.hpp 2018-11-19 17:21:38.110132800 -0800 @@ -77,6 +77,7 @@ PreMatureOldMask = OldMask + 1, OldTag = OldMask, PreMatureOldTag = PreMatureOldMask, + // Archive regions are regions with immutable content (i.e. not reclaimed, and // not allocated into during regular operation). They differ in the kind of references // allowed for the contained objects: --- old/src/hotspot/share/gc/g1/vmStructs_g1.hpp 2018-11-19 17:21:50.241566400 -0800 +++ new/src/hotspot/share/gc/g1/vmStructs_g1.hpp 2018-11-19 17:21:49.343830100 -0800 @@ -83,6 +83,7 @@ declare_constant(HeapRegionType::StartsHumongousTag) \ declare_constant(HeapRegionType::ContinuesHumongousTag) \ declare_constant(HeapRegionType::OldMask) \ + declare_constant(HeapRegionType::PreMatureOldMask) \ declare_constant(BarrierSet::G1BarrierSet) \ declare_constant(G1CardTable::g1_young_gen) --- old/src/hotspot/share/gc/shared/gcArguments.cpp 2018-11-19 17:22:01.595121100 -0800 +++ new/src/hotspot/share/gc/shared/gcArguments.cpp 2018-11-19 17:22:00.663552000 -0800 @@ -28,6 +28,7 @@ #include "runtime/arguments.hpp" #include "runtime/globals.hpp" #include "runtime/globals_extension.hpp" +#include "utilities/defaultStream.hpp" #include "utilities/macros.hpp" void GCArguments::initialize() { @@ -54,3 +55,18 @@ FLAG_SET_CMDLINE(bool, ClassUnloadingWithConcurrentMark, false); } } + +bool GCArguments::check_args_consistency() { + bool status = true; + if (!FLAG_IS_DEFAULT(AllocateHeapAt) && !FLAG_IS_DEFAULT(AllocateOldGenAt)) { + jio_fprintf(defaultStream::error_stream(), + "AllocateHeapAt and AllocateOldGenAt cannot be used together.\n"); + status = false; + } + if (!FLAG_IS_DEFAULT(AllocateOldGenAt) && (UseSerialGC || UseConcMarkSweepGC || UseEpsilonGC || UseZGC)) { + jio_fprintf(defaultStream::error_stream(), + "AllocateOldGenAt not supported for selected GC.\n"); + status = false; + } + return status; +} --- old/src/hotspot/share/gc/shared/gcArguments.hpp 2018-11-19 17:22:12.817845800 -0800 +++ new/src/hotspot/share/gc/shared/gcArguments.hpp 2018-11-19 17:22:11.895336600 -0800 @@ -39,6 +39,7 @@ virtual void initialize(); virtual size_t conservative_max_heap_alignment() = 0; virtual CollectedHeap* create_heap() = 0; + static bool check_args_consistency(); }; #endif // SHARE_GC_SHARED_GCARGUMENTS_HPP --- old/src/hotspot/share/prims/whitebox.cpp 2018-11-19 17:22:23.952045800 -0800 +++ new/src/hotspot/share/prims/whitebox.cpp 2018-11-19 17:22:23.004245600 -0800 @@ -82,6 +82,7 @@ #include "gc/g1/g1ConcurrentMark.hpp" #include "gc/g1/g1ConcurrentMarkThread.hpp" #include "gc/g1/heapRegionRemSet.hpp" +#include "gc/g1/heterogeneousHeapRegionManager.hpp" #endif // INCLUDE_G1GC #if INCLUDE_PARALLELGC #include "gc/parallel/parallelScavengeHeap.inline.hpp" @@ -370,6 +371,10 @@ return !gch->is_in_young(p); WB_END +WB_ENTRY(jlong, WB_GetHeapBase(JNIEnv* env, jobject o)) + return (jlong)Universe::heap()->base(); +WB_END + WB_ENTRY(jlong, WB_GetObjectSize(JNIEnv* env, jobject o, jobject obj)) oop p = JNIHandles::resolve(obj); return Universe::heap()->obj_size(p) * HeapWordSize; @@ -497,6 +502,60 @@ THROW_MSG_0(vmSymbols::java_lang_UnsupportedOperationException(), "WB_G1RegionSize: G1 GC is not enabled"); WB_END +WB_ENTRY(jlong, WB_G1DramReservedStart(JNIEnv* env, jobject o)) + if (UseG1GC) { + G1CollectedHeap* g1h = G1CollectedHeap::heap(); + if (g1h->is_hetero_heap()) { + uint start_region = static_cast(g1h->hrm())->start_index_of_dram(); + return (jlong)(Universe::heap()->base() + start_region * HeapRegion::GrainBytes); + } else { + return (jlong)Universe::heap()->base(); + } + } + THROW_MSG_0(vmSymbols::java_lang_UnsupportedOperationException(), "WB_G1DramReservedStart: G1 GC is not enabled"); +WB_END + +WB_ENTRY(jlong, WB_G1DramReservedEnd(JNIEnv* env, jobject o)) +if (UseG1GC) { + G1CollectedHeap* g1h = G1CollectedHeap::heap(); + if (g1h->is_hetero_heap()) { + uint end_region = static_cast(g1h->hrm())->end_index_of_dram(); + return (jlong)(Universe::heap()->base() + (end_region + 1) * HeapRegion::GrainBytes - 1); + } + else { + return (jlong)Universe::heap()->base() + Universe::heap()->collector_policy()->max_heap_byte_size(); + } +} +THROW_MSG_0(vmSymbols::java_lang_UnsupportedOperationException(), "WB_G1DramReservedEnd: G1 GC is not enabled"); +WB_END + +WB_ENTRY(jlong, WB_G1NvdimmReservedStart(JNIEnv* env, jobject o)) +if (UseG1GC) { + G1CollectedHeap* g1h = G1CollectedHeap::heap(); + if (g1h->is_hetero_heap()) { + uint start_region = static_cast(g1h->hrm())->start_index_of_nvdimm(); + return (jlong)(Universe::heap()->base() + start_region * HeapRegion::GrainBytes); + } else { + THROW_MSG_0(vmSymbols::java_lang_UnsupportedOperationException(), "WB_G1NvdimmReservedStart: G1 heap is not heterogeneous"); + } +} +THROW_MSG_0(vmSymbols::java_lang_UnsupportedOperationException(), "WB_G1NvdimmReservedStart: G1 GC is not enabled"); +WB_END + +WB_ENTRY(jlong, WB_G1NvdimmReservedEnd(JNIEnv* env, jobject o)) +if (UseG1GC) { + G1CollectedHeap* g1h = G1CollectedHeap::heap(); + if (g1h->is_hetero_heap()) { + uint end_region = static_cast(g1h->hrm())->start_index_of_nvdimm(); + return (jlong)(Universe::heap()->base() + (end_region + 1) * HeapRegion::GrainBytes - 1); + } + else { + THROW_MSG_0(vmSymbols::java_lang_UnsupportedOperationException(), "WB_G1NvdimmReservedEnd: G1 heap is not heterogeneous"); + } +} +THROW_MSG_0(vmSymbols::java_lang_UnsupportedOperationException(), "WB_G1NvdimmReservedEnd: G1 GC is not enabled"); +WB_END + #endif // INCLUDE_G1GC #if INCLUDE_PARALLELGC @@ -2015,6 +2074,7 @@ static JNINativeMethod methods[] = { {CC"getObjectAddress0", CC"(Ljava/lang/Object;)J", (void*)&WB_GetObjectAddress }, {CC"getObjectSize0", CC"(Ljava/lang/Object;)J", (void*)&WB_GetObjectSize }, + {CC"getHeapBase", CC"()J", (void*)&WB_GetHeapBase }, {CC"isObjectInOldGen0", CC"(Ljava/lang/Object;)Z", (void*)&WB_isObjectInOldGen }, {CC"getHeapOopSize", CC"()I", (void*)&WB_GetHeapOopSize }, {CC"getVMPageSize", CC"()I", (void*)&WB_GetVMPageSize }, @@ -2049,6 +2109,10 @@ {CC"g1NumMaxRegions", CC"()J", (void*)&WB_G1NumMaxRegions }, {CC"g1NumFreeRegions", CC"()J", (void*)&WB_G1NumFreeRegions }, {CC"g1RegionSize", CC"()I", (void*)&WB_G1RegionSize }, + {CC"g1DramReservedStart", CC"()J", (void*)&WB_G1DramReservedStart }, + {CC"g1DramReservedEnd", CC"()J", (void*)&WB_G1DramReservedEnd }, + {CC"g1NvdimmReservedStart", CC"()J", (void*)&WB_G1NvdimmReservedStart }, + {CC"g1NvdimmReservedEnd", CC"()J", (void*)&WB_G1NvdimmReservedEnd }, {CC"g1StartConcMarkCycle", CC"()Z", (void*)&WB_G1StartMarkCycle }, {CC"g1AuxiliaryMemoryUsage", CC"()Ljava/lang/management/MemoryUsage;", (void*)&WB_G1AuxiliaryMemoryUsage }, --- old/src/hotspot/share/runtime/arguments.cpp 2018-11-19 17:22:35.208170400 -0800 +++ new/src/hotspot/share/runtime/arguments.cpp 2018-11-19 17:22:34.274073200 -0800 @@ -1622,6 +1622,10 @@ } void Arguments::set_use_compressed_oops() { +if (AllocateOldGenAt != NULL) { + FLAG_SET_ERGO(bool, UseCompressedOops, false); + return; +} #ifndef ZERO #ifdef _LP64 // MaxHeapSize is not set up properly at this point, but @@ -2063,17 +2067,7 @@ } } - if(!FLAG_IS_DEFAULT(AllocateHeapAt) && !FLAG_IS_DEFAULT(AllocateOldGenAt)) { - jio_fprintf(defaultStream::error_stream(), - "AllocateHeapAt and AllocateOldGenAt cannot be used together.\n"); - status = false; - } - - if (!FLAG_IS_DEFAULT(AllocateOldGenAt) && (UseSerialGC || UseConcMarkSweepGC || UseEpsilonGC || UseZGC)) { - jio_fprintf(defaultStream::error_stream(), - "AllocateOldGenAt not supported for selected GC.\n"); - status = false; - } + status = status && GCArguments::check_args_consistency(); return status; } --- old/src/hotspot/share/runtime/globals.hpp 2018-11-19 17:22:46.762646100 -0800 +++ new/src/hotspot/share/runtime/globals.hpp 2018-11-19 17:22:45.886082000 -0800 @@ -2604,15 +2604,7 @@ "Use platform unstable time where supported for timestamps only") \ \ experimental(ccstr, AllocateOldGenAt, NULL, \ - "Directory to use for allocating old generation") \ - \ - experimental(uintx, G1YoungExpansionBufferPerc, 10, \ - "When heterogenous heap is enabled by AllocateOldGenAt " \ - "option, after every GC, yg gen is re-sized which involves " \ - "system calls to commit/uncommit memory. To reduce these " \ - "calls, we keep a buffer of extra regions to absorb small " \ - "changes in yg gen length. This flag takes the buffer " \ - "size as an percentage of young gen length") + "Directory to use for allocating old generation") #define VM_FLAGS(develop, \ develop_pd, \ --- old/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/g1/G1CollectedHeap.java 2018-11-19 17:22:58.176309600 -0800 +++ new/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/g1/G1CollectedHeap.java 2018-11-19 17:22:57.255471300 -0800 @@ -47,7 +47,7 @@ public class G1CollectedHeap extends CollectedHeap { // HeapRegionManager _hrm; - static private long hrmFieldOffset; + static private AddressField hrmField; // MemRegion _g1_reserved; static private long g1ReservedFieldOffset; // size_t _summary_bytes_used; @@ -72,7 +72,7 @@ static private synchronized void initialize(TypeDataBase db) { Type type = db.lookupType("G1CollectedHeap"); - hrmFieldOffset = type.getField("_hrm").getOffset(); + hrmField = type.getAddressField("_hrm"); summaryBytesUsedField = type.getCIntegerField("_summary_bytes_used"); g1mmField = type.getAddressField("_g1mm"); oldSetFieldOffset = type.getField("_old_set").getOffset(); @@ -93,7 +93,7 @@ } public HeapRegionManager hrm() { - Address hrmAddr = addr.addOffsetTo(hrmFieldOffset); + Address hrmAddr = hrmField.getValue(addr); return (HeapRegionManager) VMObjectFactory.newObject(HeapRegionManager.class, hrmAddr); } --- old/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/g1/HeapRegion.java 2018-11-19 17:23:09.519374200 -0800 +++ new/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/g1/HeapRegion.java 2018-11-19 17:23:08.611810800 -0800 @@ -122,6 +122,10 @@ return type.isOld(); } + public boolean isPreMatureOld() { + return type.isPreMatureOld(); + } + public static long getPointerSize() { return pointerSize; } --- old/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/g1/HeapRegionType.java 2018-11-19 17:23:20.664221500 -0800 +++ new/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/g1/HeapRegionType.java 2018-11-19 17:23:19.752289200 -0800 @@ -48,6 +48,7 @@ private static int pinnedMask; private static int archiveMask; private static int oldMask; + private static int preMatureOldMask; private static CIntegerField tagField; private int tag; @@ -74,6 +75,7 @@ humongousMask = db.lookupIntConstant("HeapRegionType::HumongousMask"); pinnedMask = db.lookupIntConstant("HeapRegionType::PinnedMask"); oldMask = db.lookupIntConstant("HeapRegionType::OldMask"); + preMatureOldMask = db.lookupIntConstant("HeapRegionType::PreMatureOldMask"); } public boolean isFree() { @@ -116,6 +118,10 @@ return (tagField.getValue(addr) & oldMask) != 0; } + public boolean isPreMatureOld() { + return (tagField.getValue(addr) & preMatureOldMask) != 0; + } + public HeapRegionType(Address addr) { super(addr); } @@ -145,6 +151,9 @@ if (isOld()) { return "Old"; } + if (isPreMatureOld()) { + return "PreMature Old"; + } return "Unknown Region Type"; } } --- old/test/lib/sun/hotspot/WhiteBox.java 2018-11-19 17:23:32.160086800 -0800 +++ new/test/lib/sun/hotspot/WhiteBox.java 2018-11-19 17:23:31.204404600 -0800 @@ -182,6 +182,10 @@ public native long g1NumMaxRegions(); public native long g1NumFreeRegions(); public native int g1RegionSize(); + public native long g1DramReservedStart(); + public native long g1DramReservedEnd(); + public native long g1NvdimmReservedStart(); + public native long g1NvdimmReservedEnd(); public native MemoryUsage g1AuxiliaryMemoryUsage(); private native Object[] parseCommandLine0(String commandline, char delim, DiagnosticCommand[] args); public Object[] parseCommandLine(String commandline, char delim, DiagnosticCommand[] args) { --- /dev/null 2018-11-19 17:23:44.000000000 -0800 +++ new/src/hotspot/share/gc/g1/heterogeneousHeapRegionManager.cpp 2018-11-19 17:23:42.592511200 -0800 @@ -0,0 +1,446 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/g1/g1CollectedHeap.inline.hpp" +#include "gc/g1/g1ConcurrentRefine.hpp" +#include "gc/g1/heapRegion.hpp" +#include "gc/g1/heapRegionManager.inline.hpp" +#include "gc/g1/heterogeneousHeapRegionManager.hpp" +#include "gc/g1/heapRegionSet.inline.hpp" +#include "memory/allocation.hpp" + +// expand_by() is called to grow the heap. We grow into nvdimm now. +// Dram regions are committed later as needed during mutator region allocation or +// when young list target length is determined after gc cycle. +uint HeterogeneousHeapRegionManager::expand_by(uint num_regions, WorkGang* pretouch_workers) { + uint num_expanded = expand_nvdimm(MIN2(num_regions, max_expandable_length() - total_regions_committed()), pretouch_workers); + assert(total_regions_committed() <= max_expandable_length(), "must be"); + return num_expanded; +} + +// Expands heap starting from 'start' index. The question is should we expand from one memory (e.g. nvdimm) to another (e.g. dram). +// Looking at the code, expand_at() is called for humongous allocation where 'start' is in nv-dimm. +// So we only allocate regions in the same kind of memory as 'start'. +uint HeterogeneousHeapRegionManager::expand_at(uint start, uint num_regions, WorkGang* pretouch_workers) { + if (num_regions == 0) { + return 0; + } + uint target_num_regions = MIN2(num_regions, max_expandable_length() - total_regions_committed()); + uint end = is_in_nvdimm(start) ? end_index_of_nvdimm() : end_index_of_dram(); + uint num_expanded = expand_in_range(start, end, target_num_regions, pretouch_workers); + assert(total_regions_committed() <= max_expandable_length(), "must be"); + return num_expanded; +} + +// This function ensures that there are 'expected_num_regions' committed regions in dram. +// If new regions are committed, it un-commits that many regions from nv-dimm. +// If there are already more regions committed in dram, extra regions are un-committed. +void HeterogeneousHeapRegionManager::adjust_dram_regions(uint expected_num_regions, WorkGang* pretouch_workers) { + + assert(total_regions_committed() <= max_expandable_length(), "must be"); + if(expected_num_regions > free_list_dram_length()) { + // If we are going to expand DRAM, we expand a little more so that we can absorb small variations in Young gen sizing. + uint targeted_dram_regions = expected_num_regions * (1 + (double)G1YoungExpansionBufferPerc / 100); + uint to_be_made_available = targeted_dram_regions - free_list_dram_length(); + +#ifdef ASSERT + uint total_committed_before = total_regions_committed(); +#endif + uint can_be_made_available = shrink_nvdimm(to_be_made_available); + uint ret = expand_dram(can_be_made_available, pretouch_workers); +#ifdef ASSERT + assert(ret == can_be_made_available, "should be equal"); + assert(total_committed_before == total_regions_committed(), "invariant not met"); + assert(total_regions_committed() <= _max_regions, "post-condition"); +#endif + } else { + uint to_be_released = free_list_dram_length() - expected_num_regions; + // if number of extra DRAM regions is small, do not shrink. + if (to_be_released < expected_num_regions * G1YoungExpansionBufferPerc / 100) { + return; + } + +#ifdef ASSERT + uint total_committed_before = total_regions_committed(); +#endif + uint ret = shrink_dram(to_be_released); + assert(ret == to_be_released, "Should be able to shrink by given amount"); + ret = expand_nvdimm(to_be_released, pretouch_workers); +#ifdef ASSERT + assert(ret == to_be_released, "Should be able to expand by given amount"); + assert(total_committed_before == total_regions_committed(), "invariant not met"); + assert(total_regions_committed() <= _max_regions, "post-condition"); +#endif + } + assert(total_regions_committed() <= max_expandable_length(), "must be"); +} + +uint HeterogeneousHeapRegionManager::total_regions_committed() const { + return num_committed_dram() + num_committed_nvdimm(); +} + +uint HeterogeneousHeapRegionManager::num_committed_dram() const { + // This class does not keep count of committed regions in dram and nv-dimm. + // G1RegionToHeteroSpaceMapper keeps this information. + return static_cast(_heap_mapper)->num_committed_dram(); +} + +uint HeterogeneousHeapRegionManager::num_committed_nvdimm() const { + // See comment for num_committed_dram() + return static_cast(_heap_mapper)->num_committed_nvdimm(); +} + +// Return maximum number of regions that heap can expand to. +uint HeterogeneousHeapRegionManager::max_expandable_length() const { + return _max_regions; +} + +uint HeterogeneousHeapRegionManager::find_unavailable_in_range(uint start_idx, uint end_idx, uint* res_idx) const { + guarantee(res_idx != NULL, "checking"); + guarantee(start_idx <= (max_length() + 1), "checking"); + + uint num_regions = 0; + + uint cur = start_idx; + while (cur <= end_idx && is_available(cur)) { + cur++; + } + if (cur == end_idx + 1) { + return num_regions; + } + *res_idx = cur; + while (cur <= end_idx && !is_available(cur)) { + cur++; + } + num_regions = cur - *res_idx; + +#ifdef ASSERT + for (uint i = *res_idx; i < (*res_idx + num_regions); i++) { + assert(!is_available(i), "just checking"); + } + assert(cur == end_idx + 1 || num_regions == 0 || is_available(cur), + "The region at the current position %u must be available or at the end", cur); +#endif + return num_regions; +} + +uint HeterogeneousHeapRegionManager::expand_dram(uint num_regions, WorkGang* pretouch_workers) { + return expand_in_range(start_index_of_dram(), end_index_of_dram(), num_regions, pretouch_workers); +} + +uint HeterogeneousHeapRegionManager::expand_nvdimm(uint num_regions, WorkGang* pretouch_workers) { + return expand_in_range(start_index_of_nvdimm(), end_index_of_nvdimm(), num_regions, pretouch_workers); +} + +// Follows same logic as expand_at() form HeapRegionManager. +uint HeterogeneousHeapRegionManager::expand_in_range(uint start, uint end, uint num_regions, WorkGang* pretouch_gang) { + + uint so_far = 0; + uint chunk_start = 0; + uint num_last_found = 0; + while (so_far < num_regions && + (num_last_found = find_unavailable_in_range(start, end, &chunk_start)) > 0) { + uint to_commit = MIN2(num_regions - so_far, num_last_found); + make_regions_available(chunk_start, to_commit, pretouch_gang); + so_far += to_commit; + start = chunk_start + to_commit + 1; + } + + return so_far; +} + +// Shrink in the range of indexes which are reserved for dram. +uint HeterogeneousHeapRegionManager::shrink_dram(uint num_regions, bool update_free_list) { + return shrink_in_range(start_index_of_dram(), end_index_of_dram(), num_regions, update_free_list); +} + +// Shrink in the range of indexes which are reserved for nv-dimm. +uint HeterogeneousHeapRegionManager::shrink_nvdimm(uint num_regions, bool update_free_list) { + return shrink_in_range(start_index_of_nvdimm(), end_index_of_nvdimm(), num_regions, update_free_list); +} + +// Find empty regions in given range, un-commit them and return the count. +uint HeterogeneousHeapRegionManager::shrink_in_range(uint start, uint end, uint num_regions, bool update_free_list) { + + if (num_regions == 0) { + return 0; + } + uint so_far = 0; + uint idx_last_found = 0; + uint num_last_found; + while (so_far < num_regions && + (num_last_found = find_empty_in_range_reverse(start, end, &idx_last_found)) > 0) { + uint to_uncommit = MIN2(num_regions - so_far, num_last_found); + if(update_free_list) { + _free_list.remove_starting_at(at(idx_last_found + num_last_found - to_uncommit), to_uncommit); + } + uncommit_regions(idx_last_found + num_last_found - to_uncommit, to_uncommit); + so_far += to_uncommit; + end = idx_last_found; + } + return so_far; +} + +uint HeterogeneousHeapRegionManager::find_empty_in_range_reverse(uint start_idx, uint end_idx, uint* res_idx) { + guarantee(res_idx != NULL, "checking"); + guarantee(start_idx < max_length(), "checking"); + guarantee(end_idx < max_length(), "checking"); + if(start_idx > end_idx) { + return 0; + } + + uint num_regions_found = 0; + + jlong cur = end_idx; + while (cur >= start_idx && !(is_available(cur) && at(cur)->is_empty())) { + cur--; + } + if (cur == start_idx - 1) { + return num_regions_found; + } + jlong old_cur = cur; + // cur indexes the first empty region + while (cur >= start_idx && is_available(cur) && at(cur)->is_empty()) { + cur--; + } + *res_idx = cur + 1; + num_regions_found = old_cur - cur; + +#ifdef ASSERT + for (uint i = *res_idx; i < (*res_idx + num_regions_found); i++) { + assert(at(i)->is_empty(), "just checking"); + } +#endif + return num_regions_found; +} + +HeapRegion* HeterogeneousHeapRegionManager::allocate_free_region(bool is_old) { + // old region is allocated from nv-dimm, non-old region from dram + // assumption: dram regions take higher indexes + assert(total_regions_committed() <= max_expandable_length(), "must be"); + bool from_head = is_old ? true : false; + HeapRegion* hr = _free_list.remove_region(from_head); + + if (hr != NULL && ( (is_old && !is_in_nvdimm(hr->hrm_index())) || (!is_old && !is_in_dram(hr->hrm_index())) ) ) { + _free_list.add_ordered(hr); + hr = NULL; + } + +#ifdef ASSERT + uint total_committed_before = total_regions_committed(); +#endif + + if (hr == NULL) { + if (!is_old) { + uint ret = shrink_nvdimm(1); + if (ret == 1) { + ret = expand_dram(1, NULL); + assert(ret == 1, "We should be able to commit one region"); + hr = _free_list.remove_region(from_head); + } + } + else { /*is_old*/ + uint ret = shrink_dram(1); + if (ret == 1) { + ret = expand_nvdimm(1, NULL); + assert(ret == 1, "We should be able to commit one region"); + hr = _free_list.remove_region(from_head); + } + } + } +#ifdef ASSERT + assert(total_committed_before == total_regions_committed(), "invariant not met"); + assert(total_regions_committed() <= max_expandable_length(), "post-condition"); +#endif + + if (hr != NULL) { + assert(hr->next() == NULL, "Single region should not have next"); + assert(is_available(hr->hrm_index()), "Must be committed"); + } + assert(total_regions_committed() <= max_expandable_length(), "must be"); + return hr; +} + +uint HeterogeneousHeapRegionManager::find_contiguous_only_empty(size_t num) { + return find_contiguous(start_index_of_nvdimm(), end_index_of_nvdimm(), num, true); +} + +uint HeterogeneousHeapRegionManager::find_contiguous_empty_or_unavailable(size_t num) { + return find_contiguous(start_index_of_nvdimm(), end_index_of_nvdimm(), num, false); +} + +uint HeterogeneousHeapRegionManager::find_contiguous(size_t start, size_t end, size_t num, bool empty_only) { + uint found = 0; + size_t length_found = 0; + uint cur = (uint)start; + uint length_unavailable = 0; + + while (length_found < num && cur <= end) { + HeapRegion* hr = _regions.get_by_index(cur); + if ((!empty_only && !is_available(cur)) || (is_available(cur) && hr != NULL && hr->is_empty())) { + // This region is a potential candidate for allocation into. + if (!is_available(cur)) { + length_unavailable++; + } + length_found++; + } + else { + // This region is not a candidate. The next region is the next possible one. + found = cur + 1; + length_found = 0; + } + cur++; + } + + if (length_found == num) { + for (uint i = found; i < (found + num); i++) { + HeapRegion* hr = _regions.get_by_index(i); + // sanity check + guarantee((!empty_only && !is_available(i)) || (is_available(i) && hr != NULL && hr->is_empty()), + "Found region sequence starting at " UINT32_FORMAT ", length " SIZE_FORMAT + " that is not empty at " UINT32_FORMAT ". Hr is " PTR_FORMAT, found, num, i, p2i(hr)); + } + if (!empty_only && length_unavailable > (max_expandable_length() - total_regions_committed())) { + // if 'length_unavailable' number of regions will be made available, we will exceed max regions. + return G1_NO_HRM_INDEX; + } + return found; + } + else { + return G1_NO_HRM_INDEX; + } +} + +uint HeterogeneousHeapRegionManager::find_highest_free(bool* expanded) { + // Loop downwards from the highest dram region index, looking for an + // entry which is either free or not yet committed. If not yet + // committed, expand_at that index. + uint curr = end_index_of_dram(); + while (true) { + HeapRegion *hr = _regions.get_by_index(curr); + if (hr == NULL && !(total_regions_committed() < _max_regions)) { + uint res = shrink_nvdimm(1); + if (res == 1) { + res = expand_in_range(curr, curr, 1, NULL); + assert(res == 1, "We should be able to expand since shrink was successful"); + *expanded = true; + return curr; + } + } + else { + if (hr->is_free()) { + *expanded = false; + return curr; + } + } + if (curr == start_index_of_dram()) { + return G1_NO_HRM_INDEX; + } + curr--; + } +} + +// We need to override this since region 0 which serves are dummy region in base class may not be available here. +// This is a corner condition when either number of regions is small. When adaptive sizing is used, initial heap size +// could be just one region. This region is commited in dram to be used for young generation, leaving region 0 (which is in nvdimm) +// unavailable. +HeapRegion* HeterogeneousHeapRegionManager::get_dummy_region() { + uint curr = 0; + + while (curr < _regions.length()) { + if (is_available(curr)) { + return new_heap_region(curr); + } + curr++; + } + assert(false, "We should always find a region available for dummy region"); + return NULL; +} + +// First shrink in dram, then in nv-dimm. +uint HeterogeneousHeapRegionManager::shrink_by(uint num_regions) { + // This call is made at end of full collection. Before making this call the region sets are tore down (tear_down_region_sets()). + // So shrink() calls below do not need to remove uncomitted regions from free list. + uint ret = shrink_dram(num_regions, false /* update_free_list */); + ret += shrink_nvdimm(num_regions - ret, false /* update_free_list */); + return ret; +} + +void HeterogeneousHeapRegionManager::verify() { + HeapRegionManager::verify(); +} + +uint HeterogeneousHeapRegionManager::free_list_dram_length() const { + return _free_list.num_of_regions_in_range(start_index_of_dram(), end_index_of_dram()); +} + +uint HeterogeneousHeapRegionManager::free_list_nvdimm_length() const { + return _free_list.num_of_regions_in_range(start_index_of_nvdimm(), end_index_of_nvdimm()); +} + +bool HeterogeneousHeapRegionManager::is_in_nvdimm(uint index) const { + return index >= start_index_of_nvdimm() && index <= end_index_of_nvdimm(); +} + +bool HeterogeneousHeapRegionManager::is_in_dram(uint index) const { + return index >= start_index_of_dram() && index <= end_index_of_dram(); +} + +// We have to make sure full collection copies all surviving objects to NV-DIMM. +// We might not have enough regions in nvdimm_set, so we need to make more regions on NV-DIMM available for full collection. +// Note: by doing this we are breaking the in-variant that total number of committed regions is equal to current heap size. +// After full collection ends, we will re-establish this in-variant by freeing DRAM regions. +void HeterogeneousHeapRegionManager::prepare_for_full_collection_start() { + _total_commited_before_full_gc = total_regions_committed(); + expand_nvdimm(num_committed_dram(), NULL); + remove_all_free_regions(); +} + +// We need to bring back the total committed regions to before full collection start. +// All regular regions (not pinned regions) in DRAM should be free. +// We shrink all free regions in DRAM and if needed from NV-DIMM (when there are pinned DRAM regions) +void HeterogeneousHeapRegionManager::prepare_for_full_collection_end() { + uint shrink_size = total_regions_committed() - _total_commited_before_full_gc; + uint so_far = 0; + uint idx_last_found = 0; + uint num_last_found; + uint end = (uint)_regions.length() - 1; + while (so_far < shrink_size && + (num_last_found = find_empty_in_range_reverse(0, end, &idx_last_found)) > 0) { + uint to_uncommit = MIN2(shrink_size - so_far, num_last_found); + uncommit_regions(idx_last_found + num_last_found - to_uncommit, to_uncommit); + so_far += to_uncommit; + end = idx_last_found; + } + assert(so_far == shrink_size, "We should be able to shrink this many regions"); +} + +uint HeterogeneousHeapRegionManager::start_index_of_dram() const { return _max_regions;} + +uint HeterogeneousHeapRegionManager::end_index_of_dram() const { return 2*_max_regions - 1; } + +uint HeterogeneousHeapRegionManager::start_index_of_nvdimm() const { return 0; } + +uint HeterogeneousHeapRegionManager::end_index_of_nvdimm() const { return _max_regions - 1; } --- /dev/null 2018-11-19 17:23:53.000000000 -0800 +++ new/src/hotspot/share/gc/g1/heterogeneousHeapRegionManager.hpp 2018-11-19 17:23:50.833082400 -0800 @@ -0,0 +1,131 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_G1_HeterogeneousHeapRegionManager_HPP +#define SHARE_VM_GC_G1_HeterogeneousHeapRegionManager_HPP + +// This class manages heap regions on heterogenous memory comprising of dram and nv-dimm. +// Regions in dram (dram_set) are used for young objects and archive regions (CDS). +// Regions in nv-dimm (nvdimm_set) are used for old objects and humongous objects. +// At any point there are some regions committed on dram and some on nv-dimm with the following guarantees: +// 1. The total number of regions committed in dram and nv-dimm equals the current size of heap. +// 2. Consequently, total number of regions committed is less than or equal to Xmx. +// 3. To maintain the guarantee stated by 1., whenever one set grows (new regions committed), the other set shrinks (regions un-committed). +// 3a. If more dram regions are needed (young generation expansion), corresponding number of regions in nv-dimm are un-committed. +// 3b. When old generation or humongous set grows, and new regions need to be committed to nv-dimm, corresponding number of regions +// are un-committed in dram. +class HeterogeneousHeapRegionManager : public HeapRegionManager { + const uint _max_regions; + uint _max_dram_regions; + uint _max_nvdimm_regions; + uint _total_commited_before_full_gc; + uint _start_index_of_nvdimm; + + uint total_regions_committed() const; + uint num_committed_dram() const; + uint num_committed_nvdimm() const; + + // Similar to find_unavailable_from_idx() function from base class, difference is this function searches in range [start, end]. + uint find_unavailable_in_range(uint start_idx, uint end_idx, uint* res_idx) const; + + // Expand into dram. Maintains the invariant that total number of committed regions is less than current heap size. + uint expand_dram(uint num_regions, WorkGang* pretouch_workers); + + // Expand into nv-dimm. + uint expand_nvdimm(uint num_regions, WorkGang* pretouch_workers); + + // Expand by finding unavailable regions in [start, end] range. + uint expand_in_range(uint start, uint end, uint num_regions, WorkGang* pretouch_workers); + + // Shrink dram set of regions. + uint shrink_dram(uint num_regions, bool update_free_list = true); + + // Shrink nv-dimm set of regions. + uint shrink_nvdimm(uint num_regions, bool update_free_list = true); + + // Shrink regions from [start, end] range. + uint shrink_in_range(uint start, uint end, uint num_regions, bool update_free_list = true); + + // Similar to find_empty_from_idx_reverse() in base class. Only here it searches in a range. + uint find_empty_in_range_reverse(uint start_idx, uint end_idx, uint* res_idx); + + // Similar to find_contiguous() in base class, with [start, end] range + uint find_contiguous(size_t start, size_t end, size_t num, bool empty_only); + + uint free_list_dram_length() const; + uint free_list_nvdimm_length() const; + + // is region with given index in nv-dimm? + bool is_in_nvdimm(uint index) const; + bool is_in_dram(uint index) const; + +public: + + // Empty constructor, we'll initialize it with the initialize() method. + HeterogeneousHeapRegionManager(uint num_regions) : _max_regions(num_regions), _max_dram_regions(0), + _max_nvdimm_regions(0), _start_index_of_nvdimm(0) + {} + + uint start_index_of_nvdimm() const; + uint start_index_of_dram() const; + uint end_index_of_nvdimm() const; + uint end_index_of_dram() const; + + // Override. + HeapRegion* get_dummy_region(); + + // Adjust dram_set to provision 'expected_num_regions' regions. + void adjust_dram_regions(uint expected_num_regions, WorkGang* pretouch_workers); + + // Prepare heap regions before and after full collection. + void prepare_for_full_collection_start(); + void prepare_for_full_collection_end(); + + virtual HeapRegion* allocate_free_region(bool is_old); + + // Return maximum number of regions that heap can expand to. + uint max_expandable_length() const; + + // Override. Expand in nv-dimm. + uint expand_by(uint num_regions, WorkGang* pretouch_workers); + + // Override. + uint expand_at(uint start, uint num_regions, WorkGang* pretouch_workers); + + // Override. This function is called for humongous allocation, so we need to find empty regions in nv-dimm. + uint find_contiguous_only_empty(size_t num); + + // Override. This function is called for humongous allocation, so we need to find empty or unavailable regions in nv-dimm. + uint find_contiguous_empty_or_unavailable(size_t num); + + // Overrides base class implementation to find highest free region in dram. + uint find_highest_free(bool* expanded); + + // Override. This fuction is called to shrink the heap, we shrink in dram first then in nv-dimm. + uint shrink_by(uint num_regions_to_remove); + + void verify(); +}; + +#endif --- /dev/null 2018-11-19 17:24:01.000000000 -0800 +++ new/test/hotspot/jtreg/gc/8202286/TestAllocateOldGenAt.java 2018-11-19 17:23:59.228066800 -0800 @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +/* @test TestAllocateOldGenAt.java + * @key gc + * @summary Test to check allocation of Java Heap with AllocateOldGenAt option + * @library /test/lib + * @modules java.base/jdk.internal.misc + */ + +import jdk.test.lib.JDKToolFinder; +import jdk.test.lib.process.ProcessTools; +import jdk.test.lib.process.OutputAnalyzer; +import java.util.ArrayList; +import java.util.Collections; + +public class TestAllocateOldGenAt { + private static ArrayList commonOpts; + + public static void main(String args[]) throws Exception { + commonOpts = new ArrayList(); + + String testVmOptsStr = System.getProperty("test.java.opts"); + if (!testVmOptsStr.isEmpty()) { + String[] testVmOpts = testVmOptsStr.split(" "); + Collections.addAll(commonOpts, testVmOpts); + } + String test_dir = System.getProperty("test.dir", "."); + Collections.addAll(commonOpts, new String[] {"-XX:+UnlockExperimentalVMOptions", + "-XX:AllocateOldGenAt=" + test_dir, + "-Xmx32m", + "-Xms32m", + "-version"}); + + runTest("-XX:+UseG1GC"); + //runTest("-XX:+UseParallelOldGC"); + } + + private static void runTest(String... extraFlags) throws Exception { + ArrayList testOpts = new ArrayList(); + Collections.addAll(testOpts, commonOpts.toArray(new String[commonOpts.size()])); + Collections.addAll(testOpts, extraFlags); + + System.out.print("Testing:\n" + JDKToolFinder.getJDKTool("java")); + for (int i = 0; i < testOpts.size(); i += 1) { + System.out.print(" " + testOpts.get(i)); + } + System.out.println(); + + ProcessBuilder pb = + ProcessTools.createJavaProcessBuilder(testOpts.toArray(new String[testOpts.size()])); + OutputAnalyzer output = new OutputAnalyzer(pb.start()); + + System.out.println("Output:\n" + output.getOutput()); + + output.shouldHaveExitValue(0); + + } +} --- /dev/null 2018-11-19 17:24:09.000000000 -0800 +++ new/test/hotspot/jtreg/gc/8202286/TestAllocateOldGenAtError.java 2018-11-19 17:24:07.549826700 -0800 @@ -0,0 +1,93 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +/* @test TestAllocateOldGenAtError.java + * @key gc + * @summary Test to check correct handling of non-existent directory passed to AllocateOldGenAt option + * @library /test/lib + * @modules java.base/jdk.internal.misc + * @run main TestAllocateOldGenAtError -XX:+UseG1GC + * @run main TestAllocateOldGenAtError -XX:+UseParallelOldGC + */ + +import java.io.File; +import jdk.test.lib.JDKToolFinder; +import jdk.test.lib.process.ProcessTools; +import jdk.test.lib.process.OutputAnalyzer; +import java.util.ArrayList; +import java.util.Collections; +import java.util.UUID; + +public class TestAllocateOldGenAtError { + private static ArrayList commonOpts; + + public static void main(String args[]) throws Exception { + commonOpts = new ArrayList(); + + String testVmOptsStr = System.getProperty("test.java.opts"); + if (!testVmOptsStr.isEmpty()) { + String[] testVmOpts = testVmOptsStr.split(" "); + Collections.addAll(commonOpts, testVmOpts); + } + String test_dir = System.getProperty("test.dir", "."); + + File f = null; + do { + f = new File(test_dir, UUID.randomUUID().toString()); + } while(f.exists()); + + Collections.addAll(commonOpts, new String[] {"-XX:+UnlockExperimentalVMOptions", + "-XX:AllocateOldGenAt=" + f.getName(), + "-Xlog:gc+heap=info", + "-Xmx32m", + "-Xms32m", + "-version"}); + + runTest("-XX:+UseG1GC"); + //runTest("-XX:+UseParallelOldGC"); + } + + private static void runTest(String... extraFlags) throws Exception { + ArrayList testOpts = new ArrayList(); + Collections.addAll(testOpts, commonOpts.toArray(new String[commonOpts.size()])); + Collections.addAll(testOpts, extraFlags); + + System.out.print("Testing:\n" + JDKToolFinder.getJDKTool("java")); + for (int i = 0; i < testOpts.size(); i += 1) { + System.out.print(" " + testOpts.get(i)); + } + System.out.println(); + + ProcessBuilder pb = + ProcessTools.createJavaProcessBuilder(testOpts.toArray(new String[testOpts.size()])); + OutputAnalyzer output = new OutputAnalyzer(pb.start()); + + System.out.println("Output:\n" + output.getOutput()); + + output.shouldContain("Could not initialize G1 heap"); + output.shouldContain("Error occurred during initialization of VM"); + output.shouldNotHaveExitValue(0); + + } +} + --- /dev/null 2018-11-19 17:24:18.000000000 -0800 +++ new/test/hotspot/jtreg/gc/8202286/TestAllocateOldGenAtMultiple.java 2018-11-19 17:24:15.914680700 -0800 @@ -0,0 +1,91 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +/* @test TestAllocateOldGenAtMultiple.java + * @key gc + * @summary Test to check allocation of Java Heap with AllocateOldGenAt option. Has multiple sub-tests to cover different code paths. + * @library /test/lib + * @modules java.base/jdk.internal.misc + * @requires vm.bits == "64" + * @run main TestAllocateOldGenAtMultiple -XX:+UseG1GC + */ + +import jdk.test.lib.JDKToolFinder; +import jdk.test.lib.process.ProcessTools; +import jdk.test.lib.process.OutputAnalyzer; +import java.util.ArrayList; +import java.util.Collections; + +public class TestAllocateOldGenAtMultiple { + public static void main(String args[]) throws Exception { + ArrayList vmOpts = new ArrayList(); + String[] testVmOpts = null; + + String test_dir = System.getProperty("test.dir", "."); + + String testVmOptsStr = System.getProperty("test.java.opts"); + if (!testVmOptsStr.isEmpty()) { + testVmOpts = testVmOptsStr.split(" "); + } + + // Extra options for each of the sub-tests + String[] extraOptsList = new String[] { + "-Xmx32m -Xms32m -XX:+UseCompressedOops", // 1. With compressedoops enabled. + "-Xmx32m -Xms32m -XX:-UseCompressedOops", // 2. With compressedoops disabled. + "-Xmx32m -Xms32m -XX:HeapBaseMinAddress=3g", // 3. With user specified HeapBaseMinAddress. + "-Xmx4g -Xms4g", // 4. With larger heap size (UnscaledNarrowOop not possible). + "-Xmx4g -Xms4g -XX:+UseLargePages", // 5. Set UseLargePages. + "-Xmx4g -Xms4g -XX:+UseNUMA" // 6. Set UseNUMA. + }; + + for(String extraOpts : extraOptsList) { + vmOpts.clear(); + if(testVmOpts != null) { + Collections.addAll(vmOpts, testVmOpts); + } + // Add extra options specific to the sub-test. + String[] extraOptsArray = extraOpts.split(" "); + if(extraOptsArray != null) { + Collections.addAll(vmOpts, extraOptsArray); + } + // Add common options + Collections.addAll(vmOpts, new String[] {"-XX:+UnlockExperimentalVMOptions", + "-XX:AllocateOldGenAt=" + test_dir, + "-version"}); + + System.out.print("Testing:\n" + JDKToolFinder.getJDKTool("java")); + for (int i = 0; i < vmOpts.size(); i += 1) { + System.out.print(" " + vmOpts.get(i)); + } + System.out.println(); + + ProcessBuilder pb = + ProcessTools.createJavaProcessBuilder(vmOpts.toArray(new String[vmOpts.size()])); + OutputAnalyzer output = new OutputAnalyzer(pb.start()); + + System.out.println("Output:\n" + output.getOutput()); + + output.shouldHaveExitValue(0); + } + } +} --- /dev/null 2018-11-19 17:24:26.000000000 -0800 +++ new/test/hotspot/jtreg/gc/8202286/TestHumongousObjectsOnNvdimm.java 2018-11-19 17:24:24.224710500 -0800 @@ -0,0 +1,117 @@ +/* + * Copyright (c) 2018 Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +/* + * @test TestHumongousObjectsOnNvdimm + * @summary Check that humongous objects reside in nv-dimm + * @library /test/lib / + * @build sun.hotspot.WhiteBox + * @run driver ClassFileInstaller sun.hotspot.WhiteBox + * @run main TestHumongousObjectsOnNvdimm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions + * -XX:+WhiteBoxAPI + */ + +import jdk.test.lib.process.OutputAnalyzer; +import jdk.test.lib.process.ProcessTools; +import jdk.test.lib.Asserts; +import sun.hotspot.WhiteBox; + +import java.util.ArrayList; +import java.util.List; +import java.util.Collections; +import gc.testlibrary.Helpers; + +/** + * Test spawns HumongousObjectTest in a separate VM and expects that it + * completes without a RuntimeException. + */ +public class TestHumongousObjectsOnNvdimm { + + private static ArrayList testOpts; + + public static void main(String args[]) throws Exception { + testOpts = new ArrayList(); + + String[] common_options = new String[] { + "-Xbootclasspath/a:.", + "-XX:+UnlockExperimentalVMOptions", + "-XX:+UnlockDiagnosticVMOptions", + "-XX:+WhiteBoxAPI", + "-XX:AllocateOldGenAt="+System.getProperty("test.dir", "."), + "-Xms10M", "-Xmx10M", + "-XX:G1HeapRegionSize=1m" + }; + + String testVmOptsStr = System.getProperty("test.java.opts"); + if (!testVmOptsStr.isEmpty()) { + String[] testVmOpts = testVmOptsStr.split(" "); + Collections.addAll(testOpts, testVmOpts); + } + Collections.addAll(testOpts, common_options); + + // Test with G1 GC + runTest("-XX:+UseG1GC"); + } + + private static void runTest(String... extraFlags) throws Exception { + Collections.addAll(testOpts, extraFlags); + testOpts.add(HumongousObjectTest.class.getName()); + System.out.println(testOpts); + + ProcessBuilder pb = ProcessTools.createJavaProcessBuilder(false, + testOpts.toArray(new String[testOpts.size()])); + + OutputAnalyzer output = new OutputAnalyzer(pb.start()); + System.out.println(output.getStdout()); + output.shouldHaveExitValue(0); + } +} + +/** + * This class tests that a humongous object resides in NVDIMM. + */ +class HumongousObjectTest { + private static final WhiteBox WB = WhiteBox.getWhiteBox(); + + private static void validateObject(Object o) { + Asserts.assertTrue(WB.isObjectInOldGen(o), + "Object is supposed to be in OldGen"); + + long obj_addr = WB.getObjectAddress(o); + long nvdimm_heap_start = WB.g1NvdimmReservedStart(); + long nvdimm_heap_end = WB.g1NvdimmReservedEnd(); + + Asserts.assertTrue(WB.g1BelongsToHumongousRegion(obj_addr), "Object address should be in Humongous set"); + Asserts.assertTrue(obj_addr >= nvdimm_heap_start && obj_addr < nvdimm_heap_end, + "Humongous object does not reside in NVDIMM"); + } + + public static void main(String args[]) throws Exception { + // allocate an humongous object + int byteArrayMemoryOverhead = Helpers.detectByteArrayAllocationOverhead(); + int MinByteArrayHumongousSize = (WB.g1RegionSize() / 2) - byteArrayMemoryOverhead + 1; + byte[] obj = new byte[MinByteArrayHumongousSize]; + + validateObject(obj); + } +} --- /dev/null 2018-11-19 17:24:34.000000000 -0800 +++ new/test/hotspot/jtreg/gc/8202286/TestOldObjectsOnNvdimm.java 2018-11-19 17:24:32.491161100 -0800 @@ -0,0 +1,119 @@ +/* + * Copyright (c) 2018 Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +/* + * @test TestOldObjectsOnNvdimm + * @summary Check that objects in old generation reside in dram. + * @library /test/lib + * @build sun.hotspot.WhiteBox + * @run driver ClassFileInstaller sun.hotspot.WhiteBox + * @run main TestOldObjectsOnNvdimm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions + * -XX:+WhiteBoxAPI + */ + +import jdk.test.lib.process.OutputAnalyzer; +import jdk.test.lib.process.ProcessTools; +import jdk.test.lib.Asserts; +import sun.hotspot.WhiteBox; + +import java.util.ArrayList; +import java.util.List; +import java.util.Collections; + +/* + * Test spawns OldObjectTest in a separate VM and expects that it + * completes without a RuntimeException. + */ +public class TestOldObjectsOnNvdimm { + + public static final int ALLOCATION_SIZE = 100; + private static ArrayList testOpts; + + public static void main(String args[]) throws Exception { + testOpts = new ArrayList(); + + String[] common_options = new String[] { + "-Xbootclasspath/a:.", + "-XX:+UnlockExperimentalVMOptions", + "-XX:+UnlockDiagnosticVMOptions", + "-XX:+WhiteBoxAPI", + "-XX:AllocateOldGenAt="+System.getProperty("test.dir", "."), + "-Xms10M", "-Xmx10M", + "-XX:MaxTenuringThreshold=1" // Promote objects to Old Gen + }; + + String testVmOptsStr = System.getProperty("test.java.opts"); + if (!testVmOptsStr.isEmpty()) { + String[] testVmOpts = testVmOptsStr.split(" "); + Collections.addAll(testOpts, testVmOpts); + } + Collections.addAll(testOpts, common_options); + + // Test with G1 GC + runTest("-XX:+UseG1GC"); + // Test with ParallelOld GC + // runTest("-XX:+UseParallelOldGC"); + } + + private static void runTest(String... extraFlags) throws Exception { + Collections.addAll(testOpts, extraFlags); + testOpts.add(OldObjectTest.class.getName()); + System.out.println(testOpts); + + ProcessBuilder pb = ProcessTools.createJavaProcessBuilder(false, + testOpts.toArray(new String[testOpts.size()])); + + OutputAnalyzer output = new OutputAnalyzer(pb.start()); + System.out.println(output.getStdout()); + output.shouldHaveExitValue(0); + } +} + +/* + * This class tests that object is in Old generation after tenuring and resides in NVDIMM. + * The necessary condition for this test is running in VM with the following flags: + * -XX:AllocateOldGenAt=, -XX:MaxTenuringThreshold=1 + */ +class OldObjectTest { + private static final WhiteBox WB = WhiteBox.getWhiteBox(); + + private static void validateOldObject(Object o) { + Asserts.assertTrue(WB.isObjectInOldGen(o), + "Object is supposed to be in OldGen"); + + long oldObj_addr = WB.getObjectAddress(o); + long nvdimm_heap_start = WB.g1NvdimmReservedStart(); + long nvdimm_heap_end = WB.g1NvdimmReservedEnd(); + + Asserts.assertTrue(oldObj_addr >= nvdimm_heap_start && oldObj_addr <= nvdimm_heap_end, + "Old object does not reside in NVDIMM"); + } + + public static void main(String args[]) throws Exception { + // allocate an object and perform Young GCs to promote it to Old + byte[] oldObj = new byte[TestOldObjectsOnNvdimm.ALLOCATION_SIZE]; + WB.youngGC(); + WB.youngGC(); + validateOldObject(oldObj); + } +} --- /dev/null 2018-11-19 17:24:43.000000000 -0800 +++ new/test/hotspot/jtreg/gc/8202286/TestYoungObjectsOnDram.java 2018-11-19 17:24:40.791936100 -0800 @@ -0,0 +1,123 @@ +/* + * Copyright (c) 2018 Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +/* + * @test TestYoungObjectsOnDram + * @summary Check that objects in young generation reside in dram. + * @library /test/lib + * @build sun.hotspot.WhiteBox + * @run driver ClassFileInstaller sun.hotspot.WhiteBox + * @run main TestYoungObjectsOnDram -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions + * -XX:+WhiteBoxAPI + */ + +import jdk.test.lib.process.OutputAnalyzer; +import jdk.test.lib.process.ProcessTools; +import jdk.test.lib.Asserts; +import sun.hotspot.WhiteBox; + +import java.util.ArrayList; +import java.util.List; +import java.util.Collections; + +/** + * Test spawns YoungObjectTest in a separate VM and expects that it + * completes without a RuntimeException. + */ +public class TestYoungObjectsOnDram { + + public static final int ALLOCATION_SIZE = 100; + private static ArrayList testOpts; + + public static void main(String args[]) throws Exception { + testOpts = new ArrayList(); + + String[] common_options = new String[] { + "-Xbootclasspath/a:.", + "-XX:+UnlockExperimentalVMOptions", + "-XX:+UnlockDiagnosticVMOptions", + "-XX:+WhiteBoxAPI", + "-XX:AllocateOldGenAt="+System.getProperty("test.dir", "."), + "-XX:SurvivorRatio=1", // Survivor-to-eden ratio is 1:1 + "-Xms10M", "-Xmx10M", + "-XX:InitialTenuringThreshold=15" // avoid promotion of objects to Old Gen + }; + + String testVmOptsStr = System.getProperty("test.java.opts"); + if (!testVmOptsStr.isEmpty()) { + String[] testVmOpts = testVmOptsStr.split(" "); + Collections.addAll(testOpts, testVmOpts); + } + Collections.addAll(testOpts, common_options); + + // Test with G1 GC + runTest("-XX:+UseG1GC"); + // Test with ParallelOld GC + //runTest("-XX:+UseParallelOldGC"); + } + + private static void runTest(String... extraFlags) throws Exception { + Collections.addAll(testOpts, extraFlags); + testOpts.add(YoungObjectTest.class.getName()); + System.out.println(testOpts); + + ProcessBuilder pb = ProcessTools.createJavaProcessBuilder(false, + testOpts.toArray(new String[testOpts.size()])); + + OutputAnalyzer output = new OutputAnalyzer(pb.start()); + System.out.println(output.getStdout()); + output.shouldHaveExitValue(0); + } +} + +/** + * This class tests that newly created object is in Young generation and resides in DRAM. + * The necessary condition for this test is running in VM with the following flags: + * -XX:AllocateOldGenAt=, -XX:InitialTenuringThreshold=15, -XX:SurvivorRatio=1 + */ +class YoungObjectTest { + private static final WhiteBox WB = WhiteBox.getWhiteBox(); + + private static void validateYoungObject(Object o) { + Asserts.assertTrue(!WB.isObjectInOldGen(o), + "Object is supposed to be in YoungGen"); + + long youngObj_addr = WB.getObjectAddress(o); + long dram_heap_start = WB.g1DramReservedStart(); + long dram_heap_end = WB.g1DramReservedEnd(); + + Asserts.assertTrue(youngObj_addr >= dram_heap_start && youngObj_addr <= dram_heap_end, + "Young object does not reside in DRAM"); + } + + public static void main(String args[]) throws Exception { + // allocate an object + byte[] youngObj = new byte[TestYoungObjectsOnDram.ALLOCATION_SIZE]; + validateYoungObject(youngObj); + + // Start a Young GC and check that object is still in DRAM. + // We have used -XX:InitialTenuringThreshold=15 to invoke this test + WB.youngGC(); + validateYoungObject(youngObj); + } +} --- old/src/hotspot/share/gc/g1/heapRegionManagerForHeteroHeap.cpp 2018-11-19 17:24:50.023722500 -0800 +++ /dev/null 2018-11-19 17:24:51.000000000 -0800 @@ -1,446 +0,0 @@ -/* - * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "gc/g1/g1CollectedHeap.inline.hpp" -#include "gc/g1/g1ConcurrentRefine.hpp" -#include "gc/g1/heapRegion.hpp" -#include "gc/g1/heapRegionManager.inline.hpp" -#include "gc/g1/heapRegionManagerForHeteroHeap.hpp" -#include "gc/g1/heapRegionSet.inline.hpp" -#include "memory/allocation.hpp" - -// expand_by() is called to grow the heap. We grow into nvdimm now. -// Dram regions are committed later as needed during mutator region allocation or -// when young list target length is determined after gc cycle. -uint HeapRegionManagerForHeteroHeap::expand_by(uint num_regions, WorkGang* pretouch_workers) { - uint num_expanded = expand_nvdimm(MIN2(num_regions, max_expandable_length() - total_regions_committed()), pretouch_workers); - assert(total_regions_committed() <= max_expandable_length(), "must be"); - return num_expanded; -} - -// Expands heap starting from 'start' index. The question is should we expand from one memory (e.g. nvdimm) to another (e.g. dram). -// Looking at the code, expand_at() is called for humongous allocation where 'start' is in nv-dimm. -// So we only allocate regions in the same kind of memory as 'start'. -uint HeapRegionManagerForHeteroHeap::expand_at(uint start, uint num_regions, WorkGang* pretouch_workers) { - if (num_regions == 0) { - return 0; - } - num_regions = MIN2(num_regions, max_expandable_length() - total_regions_committed()); - uint end = is_in_nvdimm(start) ? end_index_of_nvdimm() : end_index_of_dram(); - uint num_expanded = expand_in_range(start, end, num_regions, pretouch_workers); - assert(total_regions_committed() <= max_expandable_length(), "must be"); - return num_expanded; -} - -// This function ensures that there are 'expected_num_regions' committed regions in dram. -// If new regions are committed, it un-commits that many regions from nv-dimm. -// If there are already more regions committed in dram, extra regions are un-committed. -void HeapRegionManagerForHeteroHeap::resize_dram_regions(uint expected_num_regions, WorkGang* pretouch_workers) { - - assert(total_regions_committed() <= max_expandable_length(), "must be"); - if(expected_num_regions > free_list_dram_length()) { - // If we are going to expand DRAM, we expand a little more so that we can absorb small variations in Young gen sizing. - uint targeted_dram_regions = expected_num_regions * (1 + (double)G1YoungExpansionBufferPerc / 100); - uint to_be_made_available = targeted_dram_regions - free_list_dram_length(); - -#ifdef ASSERT - uint total_committed_before = total_regions_committed(); -#endif - uint can_be_made_available = shrink_nvdimm(to_be_made_available); - uint ret = expand_dram(can_be_made_available, pretouch_workers); -#ifdef ASSERT - assert(ret == can_be_made_available, "should be equal"); - assert(total_committed_before == total_regions_committed(), "invariant not met"); - assert(total_regions_committed() <= _max_regions, "post-condition"); -#endif - } else { - uint to_be_released = free_list_dram_length() - expected_num_regions; - // if number of extra DRAM regions is small, do not shrink. - if (to_be_released < expected_num_regions * G1YoungExpansionBufferPerc / 100) { - return; - } - -#ifdef ASSERT - uint total_committed_before = total_regions_committed(); -#endif - uint ret = shrink_dram(to_be_released); - assert(ret == to_be_released, "Should be able to shrink by given amount"); - ret = expand_nvdimm(to_be_released, pretouch_workers); -#ifdef ASSERT - assert(ret == to_be_released, "Should be able to expand by given amount"); - assert(total_committed_before == total_regions_committed(), "invariant not met"); - assert(total_regions_committed() <= _max_regions, "post-condition"); -#endif - } - assert(total_regions_committed() <= max_expandable_length(), "must be"); -} - -uint HeapRegionManagerForHeteroHeap::total_regions_committed() const { - return num_committed_dram() + num_committed_nvdimm(); -} - -uint HeapRegionManagerForHeteroHeap::num_committed_dram() const { - // This class does not keep count of committed regions in dram and nv-dimm. - // G1RegionToHeteroSpaceMapper keeps this information. - return static_cast(_heap_mapper)->num_committed_dram(); -} - -uint HeapRegionManagerForHeteroHeap::num_committed_nvdimm() const { - // See comment for num_committed_dram() - return static_cast(_heap_mapper)->num_committed_nvdimm(); -} - -// Return maximum number of regions that heap can expand to. -uint HeapRegionManagerForHeteroHeap::max_expandable_length() const { - return _max_regions; -} - -uint HeapRegionManagerForHeteroHeap::find_unavailable_in_range(uint start_idx, uint end_idx, uint* res_idx) const { - guarantee(res_idx != NULL, "checking"); - guarantee(start_idx <= (max_length() + 1), "checking"); - - uint num_regions = 0; - - uint cur = start_idx; - while (cur <= end_idx && is_available(cur)) { - cur++; - } - if (cur == end_idx + 1) { - return num_regions; - } - *res_idx = cur; - while (cur <= end_idx && !is_available(cur)) { - cur++; - } - num_regions = cur - *res_idx; - -#ifdef ASSERT - for (uint i = *res_idx; i < (*res_idx + num_regions); i++) { - assert(!is_available(i), "just checking"); - } - assert(cur == end_idx + 1 || num_regions == 0 || is_available(cur), - "The region at the current position %u must be available or at the end", cur); -#endif - return num_regions; -} - -uint HeapRegionManagerForHeteroHeap::expand_dram(uint num_regions, WorkGang* pretouch_workers) { - return expand_in_range(start_index_of_dram(), end_index_of_dram(), num_regions, pretouch_workers); -} - -uint HeapRegionManagerForHeteroHeap::expand_nvdimm(uint num_regions, WorkGang* pretouch_workers) { - return expand_in_range(start_index_of_nvdimm(), end_index_of_nvdimm(), num_regions, pretouch_workers); -} - -// Follows same logic as expand_at() form HeapRegionManager. -uint HeapRegionManagerForHeteroHeap::expand_in_range(uint start, uint end, uint num_regions, WorkGang* pretouch_gang) { - - uint so_far = 0; - uint chunk_start = 0; - uint num_last_found = 0; - while (so_far < num_regions && - (num_last_found = find_unavailable_in_range(start, end, &chunk_start)) > 0) { - uint to_commit = MIN2(num_regions - so_far, num_last_found); - make_regions_available(chunk_start, to_commit, pretouch_gang); - so_far += to_commit; - start = chunk_start + to_commit + 1; - } - - return so_far; -} - -// Shrink in the range of indexes which are reserved for dram. -uint HeapRegionManagerForHeteroHeap::shrink_dram(uint num_regions, bool update_free_list) { - return shrink_in_range(start_index_of_dram(), end_index_of_dram(), num_regions, update_free_list); -} - -// Shrink in the range of indexes which are reserved for nv-dimm. -uint HeapRegionManagerForHeteroHeap::shrink_nvdimm(uint num_regions, bool update_free_list) { - return shrink_in_range(start_index_of_nvdimm(), end_index_of_nvdimm(), num_regions, update_free_list); -} - -// Find empty regions in given range, un-commit them and return the count. -uint HeapRegionManagerForHeteroHeap::shrink_in_range(uint start, uint end, uint num_regions, bool update_free_list) { - - if (num_regions == 0) { - return 0; - } - uint so_far = 0; - uint idx_last_found = 0; - uint num_last_found; - while (so_far < num_regions && - (num_last_found = find_empty_in_range_reverse(start, end, &idx_last_found)) > 0) { - uint to_uncommit = MIN2(num_regions - so_far, num_last_found); - if(update_free_list) { - _free_list.remove_starting_at(at(idx_last_found + num_last_found - to_uncommit), to_uncommit); - } - uncommit_regions(idx_last_found + num_last_found - to_uncommit, to_uncommit); - so_far += to_uncommit; - end = idx_last_found; - } - return so_far; -} - -uint HeapRegionManagerForHeteroHeap::find_empty_in_range_reverse(uint start_idx, uint end_idx, uint* res_idx) { - guarantee(res_idx != NULL, "checking"); - guarantee(start_idx < max_length(), "checking"); - guarantee(end_idx < max_length(), "checking"); - if(start_idx > end_idx) { - return 0; - } - - uint num_regions_found = 0; - - jlong cur = end_idx; - while (cur >= start_idx && !(is_available(cur) && at(cur)->is_empty())) { - cur--; - } - if (cur == start_idx - 1) { - return num_regions_found; - } - jlong old_cur = cur; - // cur indexes the first empty region - while (cur >= start_idx && (is_available(cur) && at(cur)->is_empty())) { - cur--; - } - *res_idx = cur + 1; - num_regions_found = old_cur - cur; - -#ifdef ASSERT - for (uint i = *res_idx; i < (*res_idx + num_regions_found); i++) { - assert(at(i)->is_empty(), "just checking"); - } -#endif - return num_regions_found; -} - -HeapRegion* HeapRegionManagerForHeteroHeap::allocate_free_region(bool is_old) { - // old region is allocated from nv-dimm, non-old region from dram - // assumption: dram regions take higher indexes - assert(total_regions_committed() <= max_expandable_length(), "must be"); - bool from_head = is_old ? true : false; - HeapRegion* hr = _free_list.remove_region(from_head); - - if (hr != NULL && ( (is_old && !is_in_nvdimm(hr->hrm_index())) || (!is_old && !is_in_dram(hr->hrm_index())) ) ) { - _free_list.add_ordered(hr); - hr = NULL; - } - -#ifdef ASSERT - uint total_committed_before = total_regions_committed(); -#endif - - if (hr == NULL) { - if (!is_old) { - uint ret = shrink_nvdimm(1); - if (ret == 1) { - ret = expand_dram(1, NULL); - assert(ret == 1, "We should be able to commit one region"); - hr = _free_list.remove_region(from_head); - } - } - else { /*is_old*/ - uint ret = shrink_dram(1); - if (ret == 1) { - ret = expand_nvdimm(1, NULL); - assert(ret == 1, "We should be able to commit one region"); - hr = _free_list.remove_region(from_head); - } - } - } -#ifdef ASSERT - assert(total_committed_before == total_regions_committed(), "invariant not met"); - assert(total_regions_committed() <= max_expandable_length(), "post-condition"); -#endif - - if (hr != NULL) { - assert(hr->next() == NULL, "Single region should not have next"); - assert(is_available(hr->hrm_index()), "Must be committed"); - } - assert(total_regions_committed() <= max_expandable_length(), "must be"); - return hr; -} - -uint HeapRegionManagerForHeteroHeap::find_contiguous_only_empty(size_t num) { - return find_contiguous(start_index_of_nvdimm(), end_index_of_nvdimm(), num, true); -} - -uint HeapRegionManagerForHeteroHeap::find_contiguous_empty_or_unavailable(size_t num) { - return find_contiguous(start_index_of_nvdimm(), end_index_of_nvdimm(), num, false); -} - -uint HeapRegionManagerForHeteroHeap::find_contiguous(size_t start, size_t end, size_t num, bool empty_only) { - uint found = 0; - size_t length_found = 0; - uint cur = (uint)start; - uint length_unavailable = 0; - - while (length_found < num && cur <= end) { - HeapRegion* hr = _regions.get_by_index(cur); - if ((!empty_only && !is_available(cur)) || (is_available(cur) && hr != NULL && hr->is_empty())) { - // This region is a potential candidate for allocation into. - if (!is_available(cur)) { - length_unavailable++; - } - length_found++; - } - else { - // This region is not a candidate. The next region is the next possible one. - found = cur + 1; - length_found = 0; - } - cur++; - } - - if (length_found == num) { - for (uint i = found; i < (found + num); i++) { - HeapRegion* hr = _regions.get_by_index(i); - // sanity check - guarantee((!empty_only && !is_available(i)) || (is_available(i) && hr != NULL && hr->is_empty()), - "Found region sequence starting at " UINT32_FORMAT ", length " SIZE_FORMAT - " that is not empty at " UINT32_FORMAT ". Hr is " PTR_FORMAT, found, num, i, p2i(hr)); - } - if (!empty_only && length_unavailable > (max_expandable_length() - total_regions_committed())) { - // if 'length_unavailable' number of regions will be made available, we will exceed max regions. - return G1_NO_HRM_INDEX; - } - return found; - } - else { - return G1_NO_HRM_INDEX; - } -} - -uint HeapRegionManagerForHeteroHeap::find_highest_free(bool* expanded) { - // Loop downwards from the highest dram region index, looking for an - // entry which is either free or not yet committed. If not yet - // committed, expand_at that index. - uint curr = end_index_of_dram(); - while (true) { - HeapRegion *hr = _regions.get_by_index(curr); - if (hr == NULL && !(total_regions_committed() < _max_regions)) { - uint res = shrink_nvdimm(1); - if (res == 1) { - res = expand_in_range(curr, curr, 1, NULL); - assert(res == 1, "We should be able to expand since shrink was successful"); - *expanded = true; - return curr; - } - } - else { - if (hr->is_free()) { - *expanded = false; - return curr; - } - } - if (curr == start_index_of_dram()) { - return G1_NO_HRM_INDEX; - } - curr--; - } -} - -// We need to override this since region 0 which serves are dummy region in base class may not be available here. -// This is a corner condition when either number of regions is small. When adaptive sizing is used, initial heap size -// could be just one region. This region is commited in dram to be used for young generation, leaving region 0 (which is in nvdimm) -// unavailable. -HeapRegion* HeapRegionManagerForHeteroHeap::get_dummy_region() { - uint curr = 0; - - while (curr < _regions.length()) { - if (is_available(curr)) { - return new_heap_region(curr); - } - curr++; - } - assert(false, "We should always find a region available for dummy region"); - return NULL; -} - -// First shrink in dram, then in nv-dimm. -uint HeapRegionManagerForHeteroHeap::shrink_by(uint num_regions) { - // This call is made at end of full collection. Before making this call the region sets are tore down (tear_down_region_sets()). - // So shrink() calls below do not need to remove uncomitted regions from free list. - uint ret = shrink_dram(num_regions, false /* update_free_list */); - ret += shrink_nvdimm(num_regions - ret, false /* update_free_list */); - return ret; -} - -void HeapRegionManagerForHeteroHeap::verify() { - HeapRegionManager::verify(); -} - -uint HeapRegionManagerForHeteroHeap::free_list_dram_length() const { - return _free_list.num_of_regions_in_range(start_index_of_dram(), end_index_of_dram()); -} - -uint HeapRegionManagerForHeteroHeap::free_list_nvdimm_length() const { - return _free_list.num_of_regions_in_range(start_index_of_nvdimm(), end_index_of_nvdimm()); -} - -bool HeapRegionManagerForHeteroHeap::is_in_nvdimm(uint index) const { - return index >= start_index_of_nvdimm() && index <= end_index_of_nvdimm(); -} - -bool HeapRegionManagerForHeteroHeap::is_in_dram(uint index) const { - return index >= start_index_of_dram() && index <= end_index_of_dram(); -} - -// We have to make sure full collection copies all surviving objects to NV-DIMM. -// We might not have enough regions in nvdimm_set, so we need to make more regions on NV-DIMM available for full collection. -// Note: by doing this we are breaking the in-variant that total number of committed regions is equal to current heap size. -// After full collection ends, we will re-establish this in-variant by freeing DRAM regions. -void HeapRegionManagerForHeteroHeap::prepare_for_full_collection_start() { - _total_commited_before_full_gc = total_regions_committed(); - expand_nvdimm(num_committed_dram(), NULL); - remove_all_free_regions(); -} - -// We need to bring back the total committed regions to before full collection start. -// All regular regions (not pinned regions) in DRAM should be free. -// We shrink all free regions in DRAM and if needed from NV-DIMM (when there are pinned DRAM regions) -void HeapRegionManagerForHeteroHeap::prepare_for_full_collection_end() { - uint shrink_size = total_regions_committed() - _total_commited_before_full_gc; - uint so_far = 0; - uint idx_last_found = 0; - uint num_last_found; - uint end = (uint)_regions.length() - 1; - while (so_far < shrink_size && - (num_last_found = find_empty_in_range_reverse(0, end, &idx_last_found)) > 0) { - uint to_uncommit = MIN2(shrink_size - so_far, num_last_found); - uncommit_regions(idx_last_found + num_last_found - to_uncommit, to_uncommit); - so_far += to_uncommit; - end = idx_last_found; - } - assert(so_far == shrink_size, "We should be able to shrink this many regions"); -} - -uint HeapRegionManagerForHeteroHeap::start_index_of_dram() const { return _max_regions;} - -uint HeapRegionManagerForHeteroHeap::end_index_of_dram() const { return 2*_max_regions - 1; } - -uint HeapRegionManagerForHeteroHeap::start_index_of_nvdimm() const { return 0; } - -uint HeapRegionManagerForHeteroHeap::end_index_of_nvdimm() const { return _max_regions - 1; } --- old/src/hotspot/share/gc/g1/heapRegionManagerForHeteroHeap.hpp 2018-11-19 17:24:56.527322600 -0800 +++ /dev/null 2018-11-19 17:24:57.000000000 -0800 @@ -1,132 +0,0 @@ -/* - * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_G1_HEAPREGIONMANAGERFORHETEROHEAP_HPP -#define SHARE_VM_GC_G1_HEAPREGIONMANAGERFORHETEROHEAP_HPP - -// This class manages heap regions on heterogenous memory comprising of dram and nv-dimm. -// Regions in dram (dram_set) are used for young objects and archive regions (CDS). -// Regions in nv-dimm (nvdimm_set) are used for old objects and humongous objects. -// At any point there are some regions committed on dram and some on nv-dimm with the following guarantees: -// 1. The total number of regions committed in dram and nv-dimm equals the current size of heap. -// 2. Consequently, total number of regions committed is less than or equal to Xmx. -// 3. To maintain the guarantee stated by 1., whenever one set grows (new regions committed), the other set shrinks (regions un-committed). -// 3a. If more dram regions are needed (young generation expansion), corresponding number of regions in nv-dimm are un-committed. -// 3b. When old generation or humongous set grows, and new regions need to be committed to nv-dimm, corresponding number of regions -// are un-committed in dram. -class HeapRegionManagerForHeteroHeap : public HeapRegionManager { - - uint _max_regions; - uint _max_dram_regions; - uint _max_nvdimm_regions; - uint _total_commited_before_full_gc; - uint _start_index_of_nvdimm; - - uint start_index_of_nvdimm() const; - uint start_index_of_dram() const; - uint end_index_of_nvdimm() const; - uint end_index_of_dram() const; - - uint total_regions_committed() const; - uint num_committed_dram() const; - uint num_committed_nvdimm() const; - - // Similar to find_unavailable_from_idx() function from base class, difference is this function searches in range [start, end]. - uint find_unavailable_in_range(uint start_idx, uint end_idx, uint* res_idx) const; - - // Expand into dram. Maintains the invariant that total number of committed regions is less than current heap size. - uint expand_dram(uint num_regions, WorkGang* pretouch_workers); - - // Expand into nv-dimm. - uint expand_nvdimm(uint num_regions, WorkGang* pretouch_workers); - - // Expand by finding unavailable regions in [start, end] range. - uint expand_in_range(uint start, uint end, uint num_regions, WorkGang* pretouch_workers); - - // Shrink dram set of regions. - uint shrink_dram(uint num_regions, bool update_free_list = true); - - // Shrink nv-dimm set of regions. - uint shrink_nvdimm(uint num_regions, bool update_free_list = true); - - // Shrink regions from [start, end] range. - uint shrink_in_range(uint start, uint end, uint num_regions, bool update_free_list = true); - - // Similar to find_empty_from_idx_reverse() in base class. Only here it searches in a range. - uint find_empty_in_range_reverse(uint start_idx, uint end_idx, uint* res_idx); - - // Similar to find_contiguous() in base class, with [start, end] range - uint find_contiguous(size_t start, size_t end, size_t num, bool empty_only); - - uint free_list_dram_length() const; - uint free_list_nvdimm_length() const; - - // is region with given index in nv-dimm? - bool is_in_nvdimm(uint index) const; - bool is_in_dram(uint index) const; - -public: - - // Empty constructor, we'll initialize it with the initialize() method. - HeapRegionManagerForHeteroHeap(uint num_regions) : _max_regions(num_regions), _max_dram_regions(0), - _max_nvdimm_regions(0), _start_index_of_nvdimm(0) - {} - - // Override. - HeapRegion* get_dummy_region(); - - // Resize dram_set to 'expected_num_regions'. - void resize_dram_regions(uint expected_num_regions, WorkGang* pretouch_workers); - - // Should be called before starting full collection. - void prepare_for_full_collection_start(); - void prepare_for_full_collection_end(); - - virtual HeapRegion* allocate_free_region(bool is_old); - - // Return maximum number of regions that heap can expand to. - uint max_expandable_length() const; - - // Override. Expand in nv-dimm. - uint expand_by(uint num_regions, WorkGang* pretouch_workers); - - // Override. - uint expand_at(uint start, uint num_regions, WorkGang* pretouch_workers); - - // Override. This function is called for humongous allocation, so we need to find empty regions in nv-dimm. - uint find_contiguous_only_empty(size_t num); - - // Override. This function is called for humongous allocation, so we need to find empty or unavailable regions in nv-dimm. - uint find_contiguous_empty_or_unavailable(size_t num); - - // Overrides base class implementation to find highest free region in dram. - uint find_highest_free(bool* expanded); - - // Override. This fuction is called to shrink the heap, we shrink in dram first then in nv-dimm. - uint shrink_by(uint num_regions_to_remove); - - void verify(); -}; - -#endif