--- old/src/hotspot/share/gc/shared/memAllocator.cpp 2019-07-08 17:19:16.270723129 +0000 +++ new/src/hotspot/share/gc/shared/memAllocator.cpp 2019-07-08 17:19:15.998722848 +0000 @@ -375,12 +375,13 @@ return obj; } -void MemAllocator::mem_clear(HeapWord* mem) const { +HeapWord* MemAllocator::mem_clear(HeapWord* mem) const { assert(mem != NULL, "cannot initialize NULL object"); const size_t hs = oopDesc::header_size(); assert(_word_size >= hs, "unexpected object size"); oopDesc::set_klass_gap(mem, 0); Copy::fill_to_aligned_words(mem + hs, _word_size - hs); + return mem; } oop MemAllocator::finish(HeapWord* mem) const { @@ -399,7 +400,7 @@ } oop ObjAllocator::initialize(HeapWord* mem) const { - mem_clear(mem); + mem = mem_clear(mem); return finish(mem); } @@ -418,7 +419,7 @@ // concurrent GC. assert(_length >= 0, "length should be non-negative"); if (_do_zero) { - mem_clear(mem); + mem = mem_clear(mem); } arrayOopDesc::set_length(mem, _length); return finish(mem); @@ -429,7 +430,7 @@ // non-NULL _klass field indicates that the object is parsable by // concurrent GC. assert(_word_size > 0, "oop_size must be positive."); - mem_clear(mem); + mem = mem_clear(mem); java_lang_Class::set_oop_size(mem, (int)_word_size); return finish(mem); } --- old/src/hotspot/share/gc/shared/memAllocator.hpp 2019-07-08 17:19:16.630723503 +0000 +++ new/src/hotspot/share/gc/shared/memAllocator.hpp 2019-07-08 17:19:16.362723225 +0000 @@ -55,7 +55,7 @@ { } // This function clears the memory of the object - void mem_clear(HeapWord* mem) const; + virtual HeapWord* mem_clear(HeapWord* mem) const; // This finish constructing an oop by installing the mark word and the Klass* pointer // last. At the point when the Klass pointer is initialized, this is a constructed object // that must be parseable as an oop by concurrent collectors. --- old/src/hotspot/share/gc/z/zCollectedHeap.cpp 2019-07-08 17:19:16.990723876 +0000 +++ new/src/hotspot/share/gc/z/zCollectedHeap.cpp 2019-07-08 17:19:16.722723598 +0000 @@ -23,6 +23,7 @@ #include "precompiled.hpp" #include "gc/shared/gcHeapSummary.hpp" +#include "gc/shared/memAllocator.hpp" #include "gc/shared/suspendibleThreadSet.hpp" #include "gc/z/zCollectedHeap.hpp" #include "gc/z/zGlobals.hpp" @@ -32,8 +33,74 @@ #include "gc/z/zStat.hpp" #include "gc/z/zUtils.inline.hpp" #include "memory/universe.hpp" +#include "runtime/interfaceSupport.inline.hpp" #include "runtime/mutexLocker.hpp" +class PinAllocating { +private: + const HeapWord* _mem; +public: + PinAllocating(HeapWord* mem) : + _mem(mem) { + ZHeap::heap()->pin_allocating((uintptr_t)_mem); + } + + ~PinAllocating() { + ZHeap::heap()->unpin_allocating((uintptr_t)_mem); + } +}; + +class ZObjArrayAllocator: public ObjArrayAllocator { +private: + const bool _is_large; + const size_t _chunk_size; + + void mem_clear_large(HeapWord* mem) const { + log_info(gc)("ZMemClearer " PTR_FORMAT " " SIZE_FORMAT " M", p2i(mem), _word_size * BytesPerWord / M); + assert(mem != NULL, "cannot initialize NULL object"); + const size_t hs = oopDesc::header_size(); + assert(_word_size >= hs, "unexpected object size"); + oopDesc::set_klass_gap(mem, 0); + + PinAllocating pin_allocating(mem); + + size_t offset = hs; + while (offset + _chunk_size < _word_size) { + Copy::fill_to_aligned_words(mem + offset, _chunk_size); + offset += _chunk_size; + + { + ThreadBlockInVM tbivm(JavaThread::current()); + } + + // Fix colors + mem = (HeapWord*)ZAddress::good((uintptr_t)mem); + } + + if (offset < _word_size) { + Copy::fill_to_aligned_words(mem + offset, _word_size - offset); + } + } + +protected: + virtual HeapWord* mem_clear(HeapWord* mem) const { + if (_is_large) { + mem_clear_large(mem); + } else { + mem = ObjArrayAllocator::mem_clear(mem); + } + return (HeapWord*)ZAddress::good((uintptr_t)mem); + } + +public: + ZObjArrayAllocator(Klass* klass, size_t word_size, int length, bool do_zero, + Thread* thread = Thread::current()) : + ObjArrayAllocator(klass, word_size, length, do_zero, thread), + _is_large(static_cast(length) > ZObjectSizeLimitMedium), + _chunk_size(ZObjectSizeLimitMedium / BytesPerWord) + {} +}; + ZCollectedHeap* ZCollectedHeap::heap() { CollectedHeap* heap = Universe::heap(); assert(heap != NULL, "Uninitialized access to ZCollectedHeap::heap()"); @@ -127,6 +194,11 @@ return (HeapWord*)addr; } +oop ZCollectedHeap::array_allocate(Klass* klass, int size, int length, bool do_zero, TRAPS) { + ZObjArrayAllocator allocator(klass, size, length, do_zero, THREAD); + return allocator.allocate(); +} + HeapWord* ZCollectedHeap::mem_allocate(size_t size, bool* gc_overhead_limit_was_exceeded) { const size_t size_in_bytes = ZUtils::words_to_bytes(align_object_size(size)); return (HeapWord*)_heap.alloc_object(size_in_bytes); --- old/src/hotspot/share/gc/z/zCollectedHeap.hpp 2019-07-08 17:19:17.350724250 +0000 +++ new/src/hotspot/share/gc/z/zCollectedHeap.hpp 2019-07-08 17:19:17.082723972 +0000 @@ -53,6 +53,8 @@ size_t requested_size, size_t* actual_size); + oop array_allocate(Klass* klass, int size, int length, bool do_zero, TRAPS); + public: static ZCollectedHeap* heap(); --- old/src/hotspot/share/gc/z/zHeap.hpp 2019-07-08 17:19:17.706724619 +0000 +++ new/src/hotspot/share/gc/z/zHeap.hpp 2019-07-08 17:19:17.438724341 +0000 @@ -140,6 +140,8 @@ void undo_alloc_object_for_relocation(uintptr_t addr, size_t size); bool is_alloc_stalled() const; void check_out_of_memory(); + void pin_allocating(uintptr_t page_start); + void unpin_allocating(uintptr_t page_start); // Marking bool is_object_live(uintptr_t addr) const; --- old/src/hotspot/share/gc/z/zHeap.inline.hpp 2019-07-08 17:19:18.062724988 +0000 +++ new/src/hotspot/share/gc/z/zHeap.inline.hpp 2019-07-08 17:19:17.794724710 +0000 @@ -134,6 +134,16 @@ _page_allocator.check_out_of_memory(); } +inline void ZHeap::pin_allocating(uintptr_t page_start) { + ZPage* page = _page_table.get(page_start); + page->pin_allocating(); +} + +inline void ZHeap::unpin_allocating(uintptr_t page_start) { + ZPage* page = _page_table.get(page_start); + page->unpin_allocating(); +} + inline bool ZHeap::is_oop(oop object) const { // Verify that we have a good address. Note that ZAddress::is_good() // would not be a strong enough verification, since it only verifies --- old/src/hotspot/share/gc/z/zPage.hpp 2019-07-08 17:19:18.418725358 +0000 +++ new/src/hotspot/share/gc/z/zPage.hpp 2019-07-08 17:19:18.150725080 +0000 @@ -74,6 +74,9 @@ uint8_t numa_id(); bool is_allocating() const; + void pin_allocating(); + void unpin_allocating(); + bool is_relocatable() const; bool is_mapped() const; --- old/src/hotspot/share/gc/z/zPage.inline.hpp 2019-07-08 17:19:18.778725731 +0000 +++ new/src/hotspot/share/gc/z/zPage.inline.hpp 2019-07-08 17:19:18.506725449 +0000 @@ -146,7 +146,15 @@ } inline bool ZPage::is_allocating() const { - return _seqnum == ZGlobalSeqNum; + return _seqnum >= ZGlobalSeqNum; +} + +inline void ZPage::pin_allocating() { + _seqnum = (uint32_t)-1; +} + +inline void ZPage::unpin_allocating() { + _seqnum = ZGlobalSeqNum; } inline bool ZPage::is_relocatable() const {