--- old/src/hotspot/share/gc/g1/g1CollectedHeap.cpp 2018-05-13 17:59:36.337327042 +0200 +++ new/src/hotspot/share/gc/g1/g1CollectedHeap.cpp 2018-05-13 17:59:36.165330032 +0200 @@ -62,6 +62,7 @@ #include "gc/g1/heapRegionSet.inline.hpp" #include "gc/g1/vm_operations_g1.hpp" #include "gc/shared/adaptiveSizePolicy.hpp" +#include "gc/shared/collectedHeap.inline.hpp" #include "gc/shared/gcHeapSummary.hpp" #include "gc/shared/gcId.hpp" #include "gc/shared/gcLocker.hpp" @@ -392,10 +393,14 @@ } HeapWord* -G1CollectedHeap::mem_allocate(size_t word_size, +G1CollectedHeap::mem_allocate(size_t word_size, Klass* klass, Thread* thread, bool* gc_overhead_limit_was_exceeded) { assert_heap_not_locked_and_not_at_safepoint(); + HeapWord* obj = allocate_from_tlab(klass, thread, word_size); + if (obj != NULL) { + return obj; + } if (is_humongous(word_size)) { return attempt_allocation_humongous(word_size); } --- old/src/hotspot/share/gc/g1/g1CollectedHeap.hpp 2018-05-13 17:59:36.847318177 +0200 +++ new/src/hotspot/share/gc/g1/g1CollectedHeap.hpp 2018-05-13 17:59:36.700320732 +0200 @@ -422,7 +422,7 @@ virtual HeapWord* allocate_new_tlab(size_t word_size); - virtual HeapWord* mem_allocate(size_t word_size, + virtual HeapWord* mem_allocate(size_t word_size, Klass* klass, Thread* thread, bool* gc_overhead_limit_was_exceeded); // First-level mutator allocation attempt: try to allocate out of --- old/src/hotspot/share/gc/parallel/parallelScavengeHeap.cpp 2018-05-13 17:59:37.366309155 +0200 +++ new/src/hotspot/share/gc/parallel/parallelScavengeHeap.cpp 2018-05-13 17:59:37.186312284 +0200 @@ -37,6 +37,7 @@ #include "gc/parallel/psPromotionManager.hpp" #include "gc/parallel/psScavenge.hpp" #include "gc/parallel/vmPSOperations.hpp" +#include "gc/shared/collectedHeap.inline.hpp" #include "gc/shared/gcHeapSummary.hpp" #include "gc/shared/gcLocker.hpp" #include "gc/shared/gcWhen.hpp" @@ -230,18 +231,23 @@ // during failed allocation attempts. If the java heap becomes exhausted, // we rely on the size_policy object to force a bail out. HeapWord* ParallelScavengeHeap::mem_allocate( - size_t size, + size_t size, Klass* klass, Thread* thread, bool* gc_overhead_limit_was_exceeded) { assert(!SafepointSynchronize::is_at_safepoint(), "should not be at safepoint"); assert(Thread::current() != (Thread*)VMThread::vm_thread(), "should not be in vm thread"); assert(!Heap_lock->owned_by_self(), "this thread should not own the Heap_lock"); + HeapWord* result = allocate_from_tlab(klass, thread, size); + if (result != NULL) { + return result; + } + // In general gc_overhead_limit_was_exceeded should be false so // set it so here and reset it to true only if the gc time // limit is being exceeded as checked below. *gc_overhead_limit_was_exceeded = false; - HeapWord* result = young_gen()->allocate(size); + result = young_gen()->allocate(size); uint loop_count = 0; uint gc_count = 0; --- old/src/hotspot/share/gc/parallel/parallelScavengeHeap.hpp 2018-05-13 17:59:37.851300724 +0200 +++ new/src/hotspot/share/gc/parallel/parallelScavengeHeap.hpp 2018-05-13 17:59:37.707303227 +0200 @@ -178,7 +178,8 @@ // an excessive amount of time is being spent doing collections // and caused a NULL to be returned. If a NULL is not returned, // "gc_time_limit_was_exceeded" has an undefined meaning. - HeapWord* mem_allocate(size_t size, bool* gc_overhead_limit_was_exceeded); + HeapWord* mem_allocate(size_t size, Klass* klass, Thread* thread, + bool* gc_overhead_limit_was_exceeded); // Allocation attempt(s) during a safepoint. It should never be called // to allocate a new TLAB as this allocation might be satisfied out --- old/src/hotspot/share/gc/shared/collectedHeap.cpp 2018-05-13 17:59:38.358291910 +0200 +++ new/src/hotspot/share/gc/shared/collectedHeap.cpp 2018-05-13 17:59:38.185294918 +0200 @@ -390,7 +390,7 @@ #endif // ASSERT } thread->tlab().fill(obj, obj + size, new_tlab_size); - return Universe::heap()->tlab_post_allocation_setup(obj); + return obj; } size_t CollectedHeap::max_tlab_size() const { @@ -467,7 +467,7 @@ { DEBUG_ONLY(fill_args_check(start, words);) HandleMark hm; // Free handles before leaving. - fill_with_object_impl(start, words, zap); + Universe::heap()->fill_with_object_impl(start, words, zap); } void CollectedHeap::fill_with_objects(HeapWord* start, size_t words, bool zap) @@ -487,7 +487,7 @@ words -= cur; } - fill_with_object_impl(start, words, zap); + Universe::heap()->fill_with_object_impl(start, words, zap); } HeapWord* CollectedHeap::allocate_new_tlab(size_t size) { @@ -628,10 +628,6 @@ ShouldNotReachHere(); } -HeapWord* CollectedHeap::tlab_post_allocation_setup(HeapWord* obj) { - return obj; -} - uint CollectedHeap::oop_extra_words() { // Default implementation doesn't need extra space for oops. return 0; --- old/src/hotspot/share/gc/shared/collectedHeap.hpp 2018-05-13 17:59:38.850283358 +0200 +++ new/src/hotspot/share/gc/shared/collectedHeap.hpp 2018-05-13 17:59:38.705285879 +0200 @@ -174,7 +174,7 @@ static inline void fill_with_array(HeapWord* start, size_t words, bool zap = true); // Fill with a single object (either an int array or a java.lang.Object). - static inline void fill_with_object_impl(HeapWord* start, size_t words, bool zap = true); + virtual void fill_with_object_impl(HeapWord* start, size_t words, bool zap = true); virtual void trace_heap(GCWhen::Type when, const GCTracer* tracer); @@ -201,8 +201,6 @@ virtual Name kind() const = 0; - virtual HeapWord* tlab_post_allocation_setup(HeapWord* obj); - virtual const char* name() const = 0; /** @@ -317,7 +315,7 @@ // The obj and array allocate methods are covers for these methods. // mem_allocate() should never be // called to allocate TLABs, only individual objects. - virtual HeapWord* mem_allocate(size_t size, + virtual HeapWord* mem_allocate(size_t size, Klass* klass, Thread* thread, bool* gc_overhead_limit_was_exceeded) = 0; // Utilities for turning raw memory into filler objects. --- old/src/hotspot/share/gc/shared/collectedHeap.inline.hpp 2018-05-13 17:59:39.361274475 +0200 +++ new/src/hotspot/share/gc/shared/collectedHeap.inline.hpp 2018-05-13 17:59:39.188277482 +0200 @@ -137,18 +137,9 @@ return NULL; // caller does a CHECK_0 too } - HeapWord* result = NULL; - if (UseTLAB) { - result = allocate_from_tlab(klass, THREAD, size); - if (result != NULL) { - assert(!HAS_PENDING_EXCEPTION, - "Unexpected exception, will result in uninitialized storage"); - return result; - } - } bool gc_overhead_limit_was_exceeded = false; - result = Universe::heap()->mem_allocate(size, - &gc_overhead_limit_was_exceeded); + HeapWord* result = Universe::heap()->mem_allocate(size, klass, THREAD, + &gc_overhead_limit_was_exceeded); if (result != NULL) { NOT_PRODUCT(Universe::heap()-> check_for_non_bad_heap_word_value(result, size)); @@ -194,16 +185,22 @@ } HeapWord* CollectedHeap::allocate_from_tlab(Klass* klass, Thread* thread, size_t size) { - assert(UseTLAB, "should use UseTLAB"); - - size += Universe::heap()->oop_extra_words(); - HeapWord* obj = thread->tlab().allocate(size); - if (obj != NULL) { - obj = Universe::heap()->tlab_post_allocation_setup(obj); + if (UseTLAB) { + HeapWord* obj = thread->tlab().allocate(size); + if (obj != NULL) { + assert(!thread->has_pending_exception(), + "Unexpected exception, will result in uninitialized storage"); + return obj; + } + // Otherwise... + obj = allocate_from_tlab_slow(klass, thread, size); + if (obj != NULL) { + assert(!thread->has_pending_exception(), + "Unexpected exception, will result in uninitialized storage"); + } return obj; } - // Otherwise... - return allocate_from_tlab_slow(klass, thread, size); + return NULL; } void CollectedHeap::init_obj(HeapWord* obj, size_t size) { --- old/src/hotspot/share/gc/shared/genCollectedHeap.cpp 2018-05-13 17:59:39.850265975 +0200 +++ new/src/hotspot/share/gc/shared/genCollectedHeap.cpp 2018-05-13 17:59:39.704268513 +0200 @@ -439,8 +439,14 @@ return res; } -HeapWord* GenCollectedHeap::mem_allocate(size_t size, +HeapWord* GenCollectedHeap::mem_allocate(size_t size, Klass* klass, Thread* thread, bool* gc_overhead_limit_was_exceeded) { + + HeapWord* obj = allocate_from_tlab(klass, thread, size); + if (obj != NULL) { + return obj; + } + return mem_allocate_work(size, false /* is_tlab */, gc_overhead_limit_was_exceeded); --- old/src/hotspot/share/gc/shared/genCollectedHeap.hpp 2018-05-13 17:59:40.360257109 +0200 +++ new/src/hotspot/share/gc/shared/genCollectedHeap.hpp 2018-05-13 17:59:40.188260099 +0200 @@ -213,7 +213,8 @@ size_t max_capacity() const; - HeapWord* mem_allocate(size_t size, bool* gc_overhead_limit_was_exceeded); + HeapWord* mem_allocate(size_t size, Klass* klass, Thread* thread, + bool* gc_overhead_limit_was_exceeded); // We may support a shared contiguous allocation area, if the youngest // generation does. --- old/src/hotspot/share/gc/shared/plab.cpp 2018-05-13 17:59:40.847248644 +0200 +++ new/src/hotspot/share/gc/shared/plab.cpp 2018-05-13 17:59:40.699251216 +0200 @@ -83,17 +83,15 @@ size_t PLAB::retire_internal() { size_t result = 0; - if (_top + Universe::heap()->oop_extra_words() < _hard_end) { - HeapWord* obj = Universe::heap()->tlab_post_allocation_setup(_top); - CollectedHeap::fill_with_object(obj, _hard_end); + if (_top < _hard_end) { + CollectedHeap::fill_with_object(_top, _hard_end); result += invalidate(); } return result; } void PLAB::add_undo_waste(HeapWord* obj, size_t word_sz) { - HeapWord* head_obj = Universe::heap()->tlab_post_allocation_setup(obj); - CollectedHeap::fill_with_object(head_obj, word_sz - (head_obj - obj)); + CollectedHeap::fill_with_object(obj, word_sz); _undo_wasted += word_sz; } --- old/src/hotspot/share/gc/shared/threadLocalAllocBuffer.cpp 2018-05-13 17:59:41.344240004 +0200 +++ new/src/hotspot/share/gc/shared/threadLocalAllocBuffer.cpp 2018-05-13 17:59:41.174242959 +0200 @@ -114,8 +114,7 @@ myThread()->incr_allocated_bytes(used_bytes()); } - HeapWord* obj = Universe::heap()->tlab_post_allocation_setup(top()); - CollectedHeap::fill_with_object(obj, hard_end(), retire && zap); + CollectedHeap::fill_with_object(top(), hard_end(), retire && zap); if (retire || ZeroTLAB) { // "Reset" the TLAB set_start(NULL); --- old/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp 2018-05-13 17:59:41.826231625 +0200 +++ new/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp 2018-05-13 17:59:41.677234216 +0200 @@ -24,6 +24,7 @@ #include "precompiled.hpp" #include "memory/allocation.hpp" +#include "gc/shared/collectedHeap.inline.hpp" #include "gc/shared/gcTimer.hpp" #include "gc/shared/gcTraceTime.inline.hpp" #include "gc/shared/parallelCleaning.hpp" @@ -799,18 +800,30 @@ return _free_set->allocate(word_size, type, in_new_region); } -HeapWord* ShenandoahHeap::mem_allocate(size_t size, - bool* gc_overhead_limit_was_exceeded) { - HeapWord* filler = allocate_memory(size + BrooksPointer::word_size(), _alloc_shared); - HeapWord* result = filler + BrooksPointer::word_size(); - if (filler != NULL) { - BrooksPointer::initialize(oop(result)); +HeapWord* ShenandoahHeap::mem_allocate(size_t size, Klass* klass, Thread* thread, + bool* gc_overhead_limit_was_exceeded) { - assert(! in_collection_set(result), "never allocate in targetted region"); - return result; - } else { - return NULL; + size += BrooksPointer::word_size(); + HeapWord* obj = allocate_from_tlab(klass, thread, size); + if (obj == NULL) { + obj = allocate_memory(size, _alloc_shared); + } + + if (obj != NULL) { + obj = obj + BrooksPointer::word_size(); + BrooksPointer::initialize(oop(obj)); + assert(! in_collection_set(obj), "never allocate in targetted region"); + } + return obj; +} + +void ShenandoahHeap::fill_with_object_impl(HeapWord* start, size_t words, bool zap) { + if (words > 0) { + start += BrooksPointer::word_size(); + words -= BrooksPointer::word_size(); } + CollectedHeap::fill_with_object_impl(start, words, zap); + BrooksPointer::initialize(oop(start)); } class ShenandoahEvacuateUpdateRootsClosure: public ExtendedOopClosure { @@ -1855,13 +1868,6 @@ set_gc_state_mask(EVACUATION, in_progress); } -HeapWord* ShenandoahHeap::tlab_post_allocation_setup(HeapWord* obj) { - // Initialize Brooks pointer for the next object - HeapWord* result = obj + BrooksPointer::word_size(); - BrooksPointer::initialize(oop(result)); - return result; -} - uint ShenandoahHeap::oop_extra_words() { return BrooksPointer::word_size(); } --- old/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp 2018-05-13 17:59:42.345222604 +0200 +++ new/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp 2018-05-13 17:59:42.178225507 +0200 @@ -351,7 +351,8 @@ size_t initial_capacity() const /* override */; bool is_in(const void* p) const /* override */; bool is_scavengable(oop obj) /* override */; - HeapWord* mem_allocate(size_t size, bool* what) /* override */; + HeapWord* mem_allocate(size_t size, Klass* klass, Thread* thread, bool* what) /* override */; + void fill_with_object_impl(HeapWord* start, size_t words, bool zap = true) /* override */; bool can_elide_tlab_store_barriers() const /* override */; oop new_store_pre_barrier(JavaThread* thread, oop new_obj) /* override */; bool can_elide_initializing_store_barrier(oop new_obj) /* override */; @@ -377,7 +378,6 @@ void safe_object_iterate(ObjectClosure* cl) /* override */; size_t unsafe_max_tlab_alloc(Thread *thread) const /* override */; size_t max_tlab_size() const /* override */; - HeapWord* tlab_post_allocation_setup(HeapWord* obj) /* override */; uint oop_extra_words() /* override */; size_t tlab_used(Thread* ignored) const /* override */; void stop() /* override */; --- old/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp 2018-05-13 17:59:42.829214190 +0200 +++ new/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp 2018-05-13 17:59:42.684216711 +0200 @@ -349,7 +349,7 @@ if (alloc_from_gclab) { ShenandoahThreadLocalData::gclab(thread)->undo_allocation(filler, size_with_fwdptr); } else { - fill_with_object(copy, size_no_fwdptr); + fill_with_object(filler, size_with_fwdptr); } log_develop_trace(gc, compaction)("Copy object: " PTR_FORMAT " -> " PTR_FORMAT " failed, use other: " PTR_FORMAT, p2i(p), p2i(copy), p2i(result)); --- old/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp 2018-05-13 17:59:43.331205464 +0200 +++ new/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp 2018-05-13 17:59:43.159208454 +0200 @@ -479,10 +479,8 @@ void ShenandoahHeapRegion::fill_region() { if (free() > (BrooksPointer::word_size() + CollectedHeap::min_fill_size())) { - HeapWord* filler = allocate(BrooksPointer::word_size(), ShenandoahHeap::_alloc_shared); HeapWord* obj = allocate(end() - top(), ShenandoahHeap::_alloc_shared); _heap->fill_with_object(obj, end() - obj); - BrooksPointer::initialize(oop(obj)); } }