--- old/src/hotspot/share/gc/z/zCollectedHeap.cpp 2018-09-18 21:51:57.789689150 +0200 +++ new/src/hotspot/share/gc/z/zCollectedHeap.cpp 2018-09-18 21:51:57.531678024 +0200 @@ -117,6 +117,10 @@ return is_in(p); } +void ZCollectedHeap::fill_with_dummy_object(HeapWord* start, HeapWord* end, bool zap) { + // Does nothing, not a parsable heap +} + HeapWord* ZCollectedHeap::allocate_new_tlab(size_t min_size, size_t requested_size, size_t* actual_size) { const size_t size_in_bytes = ZUtils::words_to_bytes(align_object_size(requested_size)); const uintptr_t addr = _heap.alloc_tlab(size_in_bytes); --- old/src/hotspot/share/gc/z/zCollectedHeap.hpp 2018-09-18 21:51:58.188706356 +0200 +++ new/src/hotspot/share/gc/z/zCollectedHeap.hpp 2018-09-18 21:51:57.881693117 +0200 @@ -56,9 +56,6 @@ public: static ZCollectedHeap* heap(); - using CollectedHeap::ensure_parsability; - using CollectedHeap::resize_all_tlabs; - ZCollectedHeap(ZCollectorPolicy* policy); virtual Name kind() const; virtual const char* name() const; @@ -78,6 +75,8 @@ virtual bool is_in(const void* p) const; virtual bool is_in_closed_subset(const void* p) const; + virtual void fill_with_dummy_object(HeapWord* start, HeapWord* end, bool zap); + virtual HeapWord* mem_allocate(size_t size, bool* gc_overhead_limit_was_exceeded); virtual MetaWord* satisfy_failed_metadata_allocation(ClassLoaderData* loader_data, size_t size, --- old/src/hotspot/share/gc/z/zHeap.cpp 2018-09-18 21:51:58.589723648 +0200 +++ new/src/hotspot/share/gc/z/zHeap.cpp 2018-09-18 21:51:58.285710539 +0200 @@ -273,13 +273,13 @@ // Update statistics ZStatSample(ZSamplerHeapUsedBeforeMark, used()); - // Retire TLABs - _object_allocator.retire_tlabs(); - // Flip address view ZAddressMasks::flip_to_marked(); flip_views(); + // Retire allocating pages + _object_allocator.retire_pages(); + // Reset allocated/reclaimed/used statistics _page_allocator.reset_statistics(); @@ -475,9 +475,6 @@ ZAddressMasks::flip_to_remapped(); flip_views(); - // Remap TLABs - _object_allocator.remap_tlabs(); - // Enter relocate phase ZGlobalPhase = ZPhaseRelocate; --- old/src/hotspot/share/gc/z/zInitialize.cpp 2018-09-18 21:51:58.999741329 +0200 +++ new/src/hotspot/share/gc/z/zInitialize.cpp 2018-09-18 21:51:58.694728176 +0200 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016, 2017, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2016, 2018, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -30,6 +30,7 @@ #include "gc/z/zLargePages.hpp" #include "gc/z/zNUMA.hpp" #include "gc/z/zStat.hpp" +#include "gc/z/zStatTLAB.hpp" #include "gc/z/zTracer.hpp" #include "logging/log.hpp" #include "runtime/vm_version.hpp" @@ -45,6 +46,7 @@ ZNUMA::initialize(); ZCPU::initialize(); ZStatValue::initialize(); + ZStatTLAB::initialize(); ZTracer::initialize(); ZLargePages::initialize(); ZBarrierSet::set_barrier_set(barrier_set); --- old/src/hotspot/share/gc/z/zMark.cpp 2018-09-18 21:51:59.398758535 +0200 +++ new/src/hotspot/share/gc/z/zMark.cpp 2018-09-18 21:51:59.093745382 +0200 @@ -32,6 +32,7 @@ #include "gc/z/zPageTable.inline.hpp" #include "gc/z/zRootsIterator.hpp" #include "gc/z/zStat.hpp" +#include "gc/z/zStatTLAB.hpp" #include "gc/z/zTask.hpp" #include "gc/z/zThread.hpp" #include "gc/z/zUtils.inline.hpp" @@ -119,11 +120,25 @@ class ZMarkRootsIteratorClosure : public ZRootsIteratorClosure { public: + ZMarkRootsIteratorClosure() { + ZStatTLAB::reset(); + } + + ~ZMarkRootsIteratorClosure() { + ZStatTLAB::publish(); + } + virtual void do_thread(Thread* thread) { ZRootsIteratorClosure::do_thread(thread); // Update thread local address bad mask ZThreadLocalData::set_address_bad_mask(thread, ZAddressBadMask); + + // Retire TLAB + if (UseTLAB && thread->is_Java_thread()) { + thread->tlab().retire(ZStatTLAB::get()); + thread->tlab().resize(); + } } virtual void do_oop(oop* p) { --- old/src/hotspot/share/gc/z/zObjectAllocator.cpp 2018-09-18 21:51:59.806776129 +0200 +++ new/src/hotspot/share/gc/z/zObjectAllocator.cpp 2018-09-18 21:51:59.502763020 +0200 @@ -22,7 +22,6 @@ */ #include "precompiled.hpp" -#include "gc/shared/threadLocalAllocBuffer.inline.hpp" #include "gc/z/zCollectedHeap.hpp" #include "gc/z/zGlobals.hpp" #include "gc/z/zHeap.inline.hpp" @@ -41,8 +40,6 @@ static const ZStatCounter ZCounterUndoObjectAllocationSucceeded("Memory", "Undo Object Allocation Succeeded", ZStatUnitOpsPerSecond); static const ZStatCounter ZCounterUndoObjectAllocationFailed("Memory", "Undo Object Allocation Failed", ZStatUnitOpsPerSecond); -static const ZStatSubPhase ZSubPhasePauseRetireTLABS("Pause Retire TLABS"); -static const ZStatSubPhase ZSubPhasePauseRemapTLABS("Pause Remap TLABS"); ZObjectAllocator::ZObjectAllocator(uint nworkers) : _nworkers(nworkers), @@ -293,17 +290,9 @@ return 0; } -void ZObjectAllocator::retire_tlabs() { - ZStatTimer timer(ZSubPhasePauseRetireTLABS); +void ZObjectAllocator::retire_pages() { assert(SafepointSynchronize::is_at_safepoint(), "Should be at safepoint"); - // Retire TLABs - if (UseTLAB) { - ZCollectedHeap* heap = ZCollectedHeap::heap(); - heap->ensure_parsability(true /* retire_tlabs */); - heap->resize_all_tlabs(); - } - // Reset used _used.set_all(0); @@ -312,18 +301,3 @@ _shared_small_page.set_all(NULL); _worker_small_page.set_all(NULL); } - -static void remap_tlab_address(HeapWord** p) { - *p = (HeapWord*)ZAddress::good_or_null((uintptr_t)*p); -} - -void ZObjectAllocator::remap_tlabs() { - ZStatTimer timer(ZSubPhasePauseRemapTLABS); - assert(SafepointSynchronize::is_at_safepoint(), "Should be at safepoint"); - - if (UseTLAB) { - for (JavaThreadIteratorWithHandle iter; JavaThread* thread = iter.next(); ) { - thread->tlab().addresses_do(remap_tlab_address); - } - } -} --- old/src/hotspot/share/gc/z/zObjectAllocator.hpp 2018-09-18 21:52:00.216793809 +0200 +++ new/src/hotspot/share/gc/z/zObjectAllocator.hpp 2018-09-18 21:51:59.912780700 +0200 @@ -72,8 +72,7 @@ size_t used() const; size_t remaining() const; - void retire_tlabs(); - void remap_tlabs(); + void retire_pages(); }; #endif // SHARE_GC_Z_ZOBJECTALLOCATOR_HPP --- old/src/hotspot/share/gc/z/zRelocate.cpp 2018-09-18 21:52:00.616811059 +0200 +++ new/src/hotspot/share/gc/z/zRelocate.cpp 2018-09-18 21:52:00.313797992 +0200 @@ -22,6 +22,8 @@ */ #include "precompiled.hpp" +#include "gc/z/zAddress.inline.hpp" +#include "gc/z/zBarrier.inline.hpp" #include "gc/z/zHeap.hpp" #include "gc/z/zOopClosures.inline.hpp" #include "gc/z/zPage.hpp" @@ -35,12 +37,22 @@ _workers(workers) {} class ZRelocateRootsIteratorClosure : public ZRootsIteratorClosure { +private: + static void remap_address(HeapWord** p) { + *p = (HeapWord*)ZAddress::good_or_null((uintptr_t)*p); + } + public: virtual void do_thread(Thread* thread) { ZRootsIteratorClosure::do_thread(thread); // Update thread local address bad mask ZThreadLocalData::set_address_bad_mask(thread, ZAddressBadMask); + + // Remap TLAB + if (UseTLAB && thread->is_Java_thread()) { + thread->tlab().addresses_do(remap_address); + } } virtual void do_oop(oop* p) { --- old/src/hotspot/share/gc/z/zValue.hpp 2018-09-18 21:52:01.016828308 +0200 +++ new/src/hotspot/share/gc/z/zValue.hpp 2018-09-18 21:52:00.710815112 +0200 @@ -131,7 +131,7 @@ class ZValueIterator; template -class ZValue { +class ZValue : public CHeapObj { private: const uintptr_t _addr; --- /dev/null 2018-07-06 10:05:37.831845830 +0200 +++ new/src/hotspot/share/gc/z/zStatTLAB.cpp 2018-09-18 21:52:01.107832232 +0200 @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#include "gc/z/zStatTLAB.hpp" + +ZPerWorker* ZStatTLAB::_stats = NULL; + +void ZStatTLAB::initialize() { + if (UseTLAB) { + assert(_stats == NULL, "Already initialized"); + _stats = new ZPerWorker(); + reset(); + } +} + +void ZStatTLAB::reset() { + if (UseTLAB) { + ZPerWorkerIterator iter(_stats); + for (ThreadLocalAllocStats* stats; iter.next(&stats);) { + stats->reset(); + } + } +} + +ThreadLocalAllocStats* ZStatTLAB::get() { + if (UseTLAB) { + return _stats->addr(); + } + + return NULL; +} + +void ZStatTLAB::publish() { + if (UseTLAB) { + ThreadLocalAllocStats total; + + ZPerWorkerIterator iter(_stats); + for (ThreadLocalAllocStats* stats; iter.next(&stats);) { + total.update(*stats); + } + + total.publish(); + } +} --- /dev/null 2018-07-06 10:05:37.831845830 +0200 +++ new/src/hotspot/share/gc/z/zStatTLAB.hpp 2018-09-18 21:52:01.530850473 +0200 @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_ZSTATTLAB_HPP +#define SHARE_GC_Z_ZSTATTLAB_HPP + +#include "gc/shared/threadLocalAllocBuffer.hpp" +#include "gc/z/zValue.hpp" +#include "memory/allocation.hpp" + +class ZStatTLAB : public AllStatic { +private: + static ZPerWorker* _stats; + +public: + static void initialize(); + static void reset(); + static ThreadLocalAllocStats* get(); + static void publish(); +}; + +#endif // SHARE_GC_Z_ZSTATTLAB_HPP