diff --git a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp --- a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp +++ b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp @@ -52,7 +52,7 @@ bool checkcast = (decorators & ARRAYCOPY_CHECKCAST) != 0; bool disjoint = (decorators & ARRAYCOPY_DISJOINT) != 0; bool obj_int = type == T_OBJECT LP64_ONLY(&& UseCompressedOops); - bool dest_uninitialized = (decorators & AS_DEST_NOT_INITIALIZED) != 0; + bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0; if (type == T_OBJECT || type == T_ARRAY) { #ifdef _LP64 @@ -528,12 +528,12 @@ Address dst, Register val, Register tmp1, Register tmp2) { bool in_heap = (decorators & IN_HEAP) != 0; - bool in_concurrent_root = (decorators & IN_CONCURRENT_ROOT) != 0; + bool as_normal = (decorators & AS_NORMAL) != 0; if (in_heap) { write_barrier(masm, dst.base()); } if (type == T_OBJECT || type == T_ARRAY) { - bool needs_pre_barrier = in_heap || in_concurrent_root; + bool needs_pre_barrier = as_normal; bool needs_post_barrier = val != noreg && in_heap && UseShenandoahMatrix; Register tmp3 = LP64_ONLY(r8) NOT_LP64(rsi); @@ -618,8 +618,57 @@ } } +void ShenandoahBarrierSetAssembler::tlab_allocate(MacroAssembler* masm, + Register thread, Register obj, + Register var_size_in_bytes, + int con_size_in_bytes, + Register t1, Register t2, + Label& slow_case) { + assert_different_registers(obj, t1, t2); + assert_different_registers(obj, var_size_in_bytes, t1); + Register end = t2; + if (!thread->is_valid()) { +#ifdef _LP64 + thread = r15_thread; +#else + assert(t1->is_valid(), "need temp reg"); + thread = t1; + __ get_thread(thread); +#endif + } + + __ verify_tlab(); + + __ movptr(obj, Address(thread, JavaThread::tlab_top_offset())); + if (var_size_in_bytes == noreg) { + __ lea(end, Address(obj, con_size_in_bytes + BrooksPointer::byte_size())); + } else { + __ addptr(var_size_in_bytes, BrooksPointer::byte_size()); + __ lea(end, Address(obj, var_size_in_bytes, Address::times_1)); + } + __ cmpptr(end, Address(thread, JavaThread::tlab_end_offset())); + __ jcc(Assembler::above, slow_case); + + // update the tlab top pointer + __ movptr(Address(thread, JavaThread::tlab_top_offset()), end); + + // Initialize brooks pointer +#ifdef _LP64 + __ incrementq(obj, BrooksPointer::byte_size()); +#else + __ incrementl(obj, BrooksPointer::byte_size()); +#endif + __ movptr(Address(obj, BrooksPointer::byte_offset()), obj); + + // recover var_size_in_bytes if necessary + if (var_size_in_bytes == end) { + __ subptr(var_size_in_bytes, obj); + } + __ verify_tlab(); +} + void ShenandoahBarrierSetAssembler::resolve_for_read(MacroAssembler* masm, DecoratorSet decorators, Register obj) { - bool oop_not_null = (decorators & OOP_NOT_NULL) != 0; + bool oop_not_null = (decorators & IS_NOT_NULL) != 0; if (oop_not_null) { read_barrier_not_null(masm, obj); } else { diff --git a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.hpp b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.hpp --- a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.hpp +++ b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.hpp @@ -102,6 +102,13 @@ virtual void obj_equals(MacroAssembler* masm, Register src1, Register src2); virtual void obj_equals(MacroAssembler* masm, Register src1, Address src2); + virtual void tlab_allocate(MacroAssembler* masm, + Register thread, Register obj, + Register var_size_in_bytes, + int con_size_in_bytes, + Register t1, Register t2, + Label& slow_case); + virtual void resolve_for_read(MacroAssembler* masm, DecoratorSet decorators, Register obj); virtual void resolve_for_write(MacroAssembler* masm, DecoratorSet decorators, Register obj); diff --git a/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.cpp b/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.cpp --- a/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.cpp +++ b/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.cpp @@ -46,11 +46,6 @@ LIRGenerator* gen = access.gen(); CodeEmitInfo* info = access.access_emit_info(); DecoratorSet decorators = access.decorators(); - bool in_heap = (decorators & IN_HEAP) != 0; - bool in_conc_root = (decorators & IN_CONCURRENT_ROOT) != 0; - if (!in_heap && !in_conc_root) { - return; - } // First we test whether marking is in progress. BasicType flag_type; diff --git a/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp b/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp --- a/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp +++ b/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp @@ -640,10 +640,8 @@ const TypePtr* adr_type = access.addr().type(); Node* adr = access.addr().node(); - bool on_array = (decorators & IN_HEAP_ARRAY) != 0; bool anonymous = (decorators & ON_UNKNOWN_OOP_REF) != 0; bool on_heap = (decorators & IN_HEAP) != 0; - bool use_precise = on_array || anonymous; if (!access.is_oop() || (!on_heap && !anonymous)) { return BarrierSetC2::store_at_resolved(access, val); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp --- a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp @@ -31,6 +31,7 @@ #include "gc/shenandoah/shenandoahHeap.inline.hpp" #include "gc/shenandoah/shenandoahHeuristics.hpp" #include "gc/shenandoah/shenandoahTraversalGC.hpp" +#include "memory/iterator.inline.hpp" #include "runtime/interfaceSupport.inline.hpp" #ifdef COMPILER1 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp" @@ -45,7 +46,7 @@ ShenandoahSATBMarkQueueSet ShenandoahBarrierSet::_satb_mark_queue_set; template -class ShenandoahUpdateRefsForOopClosure: public ExtendedOopClosure { +class ShenandoahUpdateRefsForOopClosure: public BasicOopIterateClosure { private: ShenandoahHeap* _heap; ShenandoahBarrierSet* _bs; @@ -69,8 +70,9 @@ ShenandoahUpdateRefsForOopClosure() : _heap(ShenandoahHeap::heap()), _bs(ShenandoahBarrierSet::barrier_set()) { assert(UseShenandoahGC && ShenandoahCloneBarrier, "should be enabled"); } - void do_oop(oop* p) { do_oop_work(p); } - void do_oop(narrowOop* p) { do_oop_work(p); } + + virtual void do_oop(oop* p) { do_oop_work(p); } + virtual void do_oop(narrowOop* p) { do_oop_work(p); } }; ShenandoahBarrierSet::ShenandoahBarrierSet(ShenandoahHeap* heap) : diff --git a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp --- a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp @@ -240,7 +240,7 @@ _heap->connection_matrix()->set_connected(cur_dst, obj); } - RawAccess::oop_store(cur_dst, obj); + RawAccess::oop_store(cur_dst, obj); } else { // Store null. RawAccess<>::oop_store(cur_dst, o); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahCodeRoots.cpp b/src/hotspot/share/gc/shenandoah/shenandoahCodeRoots.cpp --- a/src/hotspot/share/gc/shenandoah/shenandoahCodeRoots.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahCodeRoots.cpp @@ -66,7 +66,7 @@ oop obj2 = ShenandoahBarrierSet::barrier_set()->write_barrier(obj1); if (! oopDesc::unsafe_equals(obj1, obj2)) { assert (!ShenandoahHeap::heap()->in_collection_set(obj2), "sanity"); - RawAccess::oop_store(p, obj2); + RawAccess::oop_store(p, obj2); } } } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.cpp b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.cpp --- a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.cpp @@ -37,7 +37,7 @@ #include "gc/shenandoah/shenandoahMarkCompact.hpp" #include "gc/shenandoah/shenandoahHeap.inline.hpp" #include "gc/shenandoah/shenandoahRootProcessor.hpp" -#include "gc/shenandoah/shenandoah_specialized_oop_closures.hpp" +#include "gc/shenandoah/shenandoahOopClosures.hpp" #include "gc/shenandoah/shenandoahTaskqueue.hpp" #include "gc/shenandoah/shenandoahUtils.hpp" #include "gc/shared/weakProcessor.hpp" @@ -56,7 +56,7 @@ ShenandoahHeap* _heap; template - inline void do_oop_nv(T* p) { + inline void do_oop_work(T* p) { ShenandoahConcurrentMark::mark_through_ref(p, _heap, _queue); } @@ -64,12 +64,12 @@ ShenandoahInitMarkRootsClosure(ShenandoahObjToScanQueue* q) : _queue(q), _heap(ShenandoahHeap::heap()) {}; - void do_oop(narrowOop* p) { do_oop_nv(p); } - void do_oop(oop* p) { do_oop_nv(p); } + void do_oop(narrowOop* p) { do_oop_work(p); } + void do_oop(oop* p) { do_oop_work(p); } }; ShenandoahMarkRefsSuperClosure::ShenandoahMarkRefsSuperClosure(ShenandoahObjToScanQueue* q, ReferenceProcessor* rp) : - MetadataAwareOopClosure(rp), + MetadataVisitingOopIterateClosure(rp), _queue(q), _dedup_queue(NULL), _heap(ShenandoahHeap::heap()) @@ -77,7 +77,7 @@ ShenandoahMarkRefsSuperClosure::ShenandoahMarkRefsSuperClosure(ShenandoahObjToScanQueue* q, ShenandoahStrDedupQueue* dq, ReferenceProcessor* rp) : - MetadataAwareOopClosure(rp), + MetadataVisitingOopIterateClosure(rp), _queue(q), _dedup_queue(dq), _heap(ShenandoahHeap::heap()) @@ -585,7 +585,7 @@ ShenandoahHeap* _heap; template - inline void do_oop_nv(T* p) { + inline void do_oop_work(T* p) { ShenandoahConcurrentMark::mark_through_ref(p, _heap, _queue); } @@ -593,8 +593,8 @@ ShenandoahCMKeepAliveClosure(ShenandoahObjToScanQueue* q) : _queue(q), _heap(ShenandoahHeap::heap()) {} - void do_oop(narrowOop* p) { do_oop_nv(p); } - void do_oop(oop* p) { do_oop_nv(p); } + void do_oop(narrowOop* p) { do_oop_work(p); } + void do_oop(oop* p) { do_oop_work(p); } }; class ShenandoahCMKeepAliveUpdateClosure : public OopClosure { @@ -603,7 +603,7 @@ ShenandoahHeap* _heap; template - inline void do_oop_nv(T* p) { + inline void do_oop_work(T* p) { ShenandoahConcurrentMark::mark_through_ref(p, _heap, _queue); } @@ -611,8 +611,8 @@ ShenandoahCMKeepAliveUpdateClosure(ShenandoahObjToScanQueue* q) : _queue(q), _heap(ShenandoahHeap::heap()) {} - void do_oop(narrowOop* p) { do_oop_nv(p); } - void do_oop(oop* p) { do_oop_nv(p); } + void do_oop(narrowOop* p) { do_oop_work(p); } + void do_oop(oop* p) { do_oop_work(p); } }; class ShenandoahRefProcTaskProxy : public AbstractGangTask { @@ -801,7 +801,7 @@ ShenandoahHeap* _heap; template - inline void do_oop_nv(T* p) { + inline void do_oop_work(T* p) { ShenandoahConcurrentMark::mark_through_ref(p, _heap, _queue); } @@ -809,8 +809,8 @@ ShenandoahPrecleanKeepAliveUpdateClosure(ShenandoahObjToScanQueue* q) : _queue(q), _heap(ShenandoahHeap::heap()) {} - void do_oop(narrowOop* p) { do_oop_nv(p); } - void do_oop(oop* p) { do_oop_nv(p); } + void do_oop(narrowOop* p) { do_oop_work(p); } + void do_oop(oop* p) { do_oop_work(p); } }; void ShenandoahConcurrentMark::preclean_weak_refs() { @@ -1041,6 +1041,3 @@ jushort* ShenandoahConcurrentMark::get_liveness(uint worker_id) { return _liveness_local[worker_id]; } - -// Generate Shenandoah specialized oop_oop_iterate functions. -SPECIALIZED_OOP_OOP_ITERATE_CLOSURES_SHENANDOAH(ALL_KLASS_OOP_OOP_ITERATE_DEFN) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -26,6 +26,7 @@ #include "gc/shared/gcTimer.hpp" #include "gc/shared/gcTraceTime.inline.hpp" +#include "gc/shared/memAllocator.hpp" #include "gc/shared/parallelCleaning.hpp" #include "gc/shared/plab.hpp" @@ -73,7 +74,7 @@ #ifdef ASSERT template -void ShenandoahAssertToSpaceClosure::do_oop_nv(T* p) { +void ShenandoahAssertToSpaceClosure::do_oop_work(T* p) { T o = RawAccess<>::oop_load(p); if (! CompressedOops::is_null(o)) { oop obj = CompressedOops::decode_not_null(o); @@ -81,8 +82,8 @@ } } -void ShenandoahAssertToSpaceClosure::do_oop(narrowOop* p) { do_oop_nv(p); } -void ShenandoahAssertToSpaceClosure::do_oop(oop* p) { do_oop_nv(p); } +void ShenandoahAssertToSpaceClosure::do_oop(narrowOop* p) { do_oop_work(p); } +void ShenandoahAssertToSpaceClosure::do_oop(oop* p) { do_oop_work(p); } #endif const char* ShenandoahHeap::name() const { @@ -885,16 +886,79 @@ return _free_set->allocate(word_size, type, in_new_region); } -HeapWord* ShenandoahHeap::obj_allocate_raw(Klass* klass, size_t size, - bool* gc_overhead_limit_was_exceeded, TRAPS) { - size += BrooksPointer::word_size(); - HeapWord* result = CollectedHeap::obj_allocate_raw(klass, size, gc_overhead_limit_was_exceeded, THREAD); - if (result != NULL) { - result += BrooksPointer::word_size(); - BrooksPointer::initialize(oop(result)); - assert(! in_collection_set(result), "never allocate in targetted region"); +class ShenandoahObjAllocator : public ObjAllocator { +public: + ShenandoahObjAllocator(Klass* klass, size_t word_size, Thread* thread) : + ObjAllocator(klass, word_size, thread) {} + + virtual HeapWord* mem_allocate(Allocation& allocation) { + // Allocate object. + _word_size += BrooksPointer::word_size(); + HeapWord* result = ObjAllocator::mem_allocate(allocation); + _word_size -= BrooksPointer::word_size(); + // Initialize brooks-pointer + if (result != NULL) { + result += BrooksPointer::word_size(); + BrooksPointer::initialize(oop(result)); + assert(! ShenandoahHeap::heap()->in_collection_set(result), "never allocate in targetted region"); + } + return result; } - return result; +}; + +oop ShenandoahHeap::obj_allocate(Klass* klass, int size, TRAPS) { + ShenandoahObjAllocator allocator(klass, size, THREAD); + return allocator.allocate(); +} + +class ShenandoahObjArrayAllocator : public ObjArrayAllocator { +public: + ShenandoahObjArrayAllocator(Klass* klass, size_t word_size, int length, bool do_zero, + Thread* thread) : + ObjArrayAllocator(klass, word_size, length, do_zero, thread) {} + + virtual HeapWord* mem_allocate(Allocation& allocation) { + // Allocate object. + _word_size += BrooksPointer::word_size(); + HeapWord* result = ObjArrayAllocator::mem_allocate(allocation); + _word_size -= BrooksPointer::word_size(); + if (result != NULL) { + result += BrooksPointer::word_size(); + BrooksPointer::initialize(oop(result)); + assert(! ShenandoahHeap::heap()->in_collection_set(result), "never allocate in targetted region"); + } + return result; + } + +}; + +oop ShenandoahHeap::array_allocate(Klass* klass, int size, int length, bool do_zero, TRAPS) { + ShenandoahObjArrayAllocator allocator(klass, size, length, do_zero, THREAD); + return allocator.allocate(); +} + +class ShenandoahClassAllocator : public ClassAllocator { +public: + ShenandoahClassAllocator(Klass* klass, size_t word_size, Thread* thread) : + ClassAllocator(klass, word_size, thread) {} + + virtual HeapWord* mem_allocate(Allocation& allocation) { + _word_size += BrooksPointer::word_size(); + HeapWord* result = ClassAllocator::mem_allocate(allocation); + _word_size -= BrooksPointer::word_size(); + if (result != NULL) { + result += BrooksPointer::word_size(); + BrooksPointer::initialize(oop(result)); + assert(! ShenandoahHeap::heap()->in_collection_set(result), "never allocate in targetted region"); + } + return result; + } + +}; + +oop ShenandoahHeap::class_allocate(Klass* klass, int size, TRAPS) { + ShenandoahClassAllocator allocator(klass, size, THREAD); + return allocator.allocate(); } HeapWord* ShenandoahHeap::mem_allocate(size_t size, @@ -907,7 +971,7 @@ CollectedHeap::fill_with_object(obj, end); } -class ShenandoahEvacuateUpdateRootsClosure: public ExtendedOopClosure { +class ShenandoahEvacuateUpdateRootsClosure: public BasicOopIterateClosure { private: ShenandoahHeap* _heap; Thread* _thread; @@ -930,7 +994,7 @@ if (oopDesc::unsafe_equals(resolved, obj)) { resolved = _heap->evacuate_object(obj, _thread); } - RawAccess::oop_store(p, resolved); + RawAccess::oop_store(p, resolved); } } } @@ -944,7 +1008,7 @@ } }; -class ShenandoahEvacuateRootsClosure: public ExtendedOopClosure { +class ShenandoahEvacuateRootsClosure: public BasicOopIterateClosure { private: ShenandoahHeap* _heap; Thread* _thread; @@ -1439,7 +1503,7 @@ return _free_set->capacity(); } -class ObjectIterateScanRootClosure : public ExtendedOopClosure { +class ObjectIterateScanRootClosure : public BasicOopIterateClosure { private: MarkBitMap* _bitmap; Stack* _oop_stack; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp @@ -104,7 +104,7 @@ class ShenandoahAssertToSpaceClosure : public OopClosure { private: template - void do_oop_nv(T* p); + void do_oop_work(T* p); public: void do_oop(narrowOop* p); void do_oop(oop* p); @@ -359,8 +359,9 @@ size_t initial_capacity() const /* override */; bool is_in(const void* p) const /* override */; bool is_scavengable(oop obj) /* override */; - virtual HeapWord* obj_allocate_raw(Klass* klass, size_t size, - bool* gc_overhead_limit_was_exceeded, TRAPS) /*override*/; + virtual oop obj_allocate(Klass* klass, int size, TRAPS); + virtual oop array_allocate(Klass* klass, int size, int length, bool do_zero, TRAPS); + virtual oop class_allocate(Klass* klass, int size, TRAPS); HeapWord* mem_allocate(size_t size, bool* what) /* override */; virtual void fill_with_dummy_object(HeapWord* start, HeapWord* end, bool zap); bool can_elide_tlab_store_barriers() const /* override */; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp @@ -111,7 +111,7 @@ if (in_collection_set(obj)) { shenandoah_assert_forwarded_except(p, obj, is_full_gc_in_progress() || cancelled_gc()); obj = ShenandoahBarrierSet::resolve_forwarded_not_null(obj); - RawAccess::oop_store(p, obj); + RawAccess::oop_store(p, obj); } #ifdef ASSERT else { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp @@ -31,6 +31,7 @@ #include "gc/shenandoah/shenandoahHeapRegion.hpp" #include "gc/shenandoah/shenandoahTraversalGC.hpp" #include "gc/shared/space.inline.hpp" +#include "memory/iterator.inline.hpp" #include "memory/universe.hpp" #include "oops/oop.inline.hpp" #include "runtime/java.hpp" @@ -449,7 +450,7 @@ st->cr(); } -void ShenandoahHeapRegion::oop_iterate(ExtendedOopClosure* blk) { +void ShenandoahHeapRegion::oop_iterate(OopIterateClosure* blk) { if (!is_active()) return; if (is_humongous()) { oop_iterate_humongous(blk); @@ -458,7 +459,7 @@ } } -void ShenandoahHeapRegion::oop_iterate_objects(ExtendedOopClosure* blk) { +void ShenandoahHeapRegion::oop_iterate_objects(OopIterateClosure* blk) { assert(! is_humongous(), "no humongous region here"); HeapWord* obj_addr = bottom() + BrooksPointer::word_size(); HeapWord* t = top(); @@ -469,7 +470,7 @@ } } -void ShenandoahHeapRegion::oop_iterate_humongous(ExtendedOopClosure* blk) { +void ShenandoahHeapRegion::oop_iterate_humongous(OopIterateClosure* blk) { assert(is_humongous(), "only humongous region here"); // Find head. ShenandoahHeapRegion* r = humongous_start_region(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp @@ -358,7 +358,7 @@ void recycle(); - void oop_iterate(ExtendedOopClosure* cl); + void oop_iterate(OopIterateClosure* cl); HeapWord* block_start_const(const void* p) const; @@ -417,8 +417,8 @@ void do_commit(); void do_uncommit(); - void oop_iterate_objects(ExtendedOopClosure* cl); - void oop_iterate_humongous(ExtendedOopClosure* cl); + void oop_iterate_objects(OopIterateClosure* cl); + void oop_iterate_humongous(OopIterateClosure* cl); inline void internal_increase_live_data(size_t s); }; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahMarkCompact.cpp b/src/hotspot/share/gc/shenandoah/shenandoahMarkCompact.cpp --- a/src/hotspot/share/gc/shenandoah/shenandoahMarkCompact.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahMarkCompact.cpp @@ -502,7 +502,7 @@ } } -class ShenandoahAdjustPointersClosure : public MetadataAwareOopClosure { +class ShenandoahAdjustPointersClosure : public MetadataVisitingOopIterateClosure { private: ShenandoahHeap* const _heap; size_t _new_obj_offset; @@ -514,7 +514,7 @@ oop obj = CompressedOops::decode_not_null(o); assert(_heap->is_marked_complete(obj), "must be marked"); oop forw = oop(BrooksPointer::get_raw(obj)); - RawAccess::oop_store(p, forw); + RawAccess::oop_store(p, forw); if (UseShenandoahMatrix) { if (_heap->is_in_reserved(p)) { assert(_heap->is_in_reserved(forw), "must be in heap"); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahOopClosures.hpp b/src/hotspot/share/gc/shenandoah/shenandoahOopClosures.hpp --- a/src/hotspot/share/gc/shenandoah/shenandoahOopClosures.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahOopClosures.hpp @@ -36,152 +36,164 @@ CONCURRENT // Reference updating using CAS }; -class ShenandoahMarkRefsSuperClosure : public MetadataAwareOopClosure { +class ShenandoahMarkRefsSuperClosure : public MetadataVisitingOopIterateClosure { private: ShenandoahObjToScanQueue* _queue; ShenandoahStrDedupQueue* _dedup_queue; ShenandoahHeap* _heap; + +protected: + template + void work(T *p); + public: ShenandoahMarkRefsSuperClosure(ShenandoahObjToScanQueue* q, ReferenceProcessor* rp); ShenandoahMarkRefsSuperClosure(ShenandoahObjToScanQueue* q, ShenandoahStrDedupQueue* dq, ReferenceProcessor* rp); - template - void work(T *p); - inline void set_base_object(oop obj) { /* Not needed */ } }; class ShenandoahMarkUpdateRefsClosure : public ShenandoahMarkRefsSuperClosure { +private: + template + inline void do_oop_work(T* p) { work(p); } + public: ShenandoahMarkUpdateRefsClosure(ShenandoahObjToScanQueue* q, ReferenceProcessor* rp) : ShenandoahMarkRefsSuperClosure(q, rp) {}; - template - inline void do_oop_nv(T* p) { work(p); } - virtual void do_oop(narrowOop* p) { do_oop_nv(p); } - virtual void do_oop(oop* p) { do_oop_nv(p); } - inline bool do_metadata_nv() { return false; } + virtual void do_oop(narrowOop* p) { do_oop_work(p); } + virtual void do_oop(oop* p) { do_oop_work(p); } virtual bool do_metadata() { return false; } }; class ShenandoahMarkUpdateRefsDedupClosure : public ShenandoahMarkRefsSuperClosure { +private: + template + inline void do_oop_work(T* p) { work(p); } + public: ShenandoahMarkUpdateRefsDedupClosure(ShenandoahObjToScanQueue* q, ShenandoahStrDedupQueue* dq, ReferenceProcessor* rp) : ShenandoahMarkRefsSuperClosure(q, dq, rp) {}; - template - inline void do_oop_nv(T* p) { work(p); } - virtual void do_oop(narrowOop* p) { do_oop_nv(p); } - virtual void do_oop(oop* p) { do_oop_nv(p); } - inline bool do_metadata_nv() { return false; } + virtual void do_oop(narrowOop* p) { do_oop_work(p); } + virtual void do_oop(oop* p) { do_oop_work(p); } virtual bool do_metadata() { return false; } }; class ShenandoahMarkUpdateRefsMetadataClosure : public ShenandoahMarkRefsSuperClosure { +private: + template + inline void do_oop_work(T* p) { work(p); } + public: ShenandoahMarkUpdateRefsMetadataClosure(ShenandoahObjToScanQueue* q, ReferenceProcessor* rp) : ShenandoahMarkRefsSuperClosure(q, rp) {}; - template - inline void do_oop_nv(T* p) { work(p); } - virtual void do_oop(narrowOop* p) { do_oop_nv(p); } - virtual void do_oop(oop* p) { do_oop_nv(p); } - inline bool do_metadata_nv() { return true; } + virtual void do_oop(narrowOop* p) { do_oop_work(p); } + virtual void do_oop(oop* p) { do_oop_work(p); } virtual bool do_metadata() { return true; } }; class ShenandoahMarkUpdateRefsMetadataDedupClosure : public ShenandoahMarkRefsSuperClosure { +private: + template + inline void do_oop_work(T* p) { work(p); } + public: ShenandoahMarkUpdateRefsMetadataDedupClosure(ShenandoahObjToScanQueue* q, ShenandoahStrDedupQueue* dq, ReferenceProcessor* rp) : ShenandoahMarkRefsSuperClosure(q, dq, rp) {}; - template - inline void do_oop_nv(T* p) { work(p); } - virtual void do_oop(narrowOop* p) { do_oop_nv(p); } - virtual void do_oop(oop* p) { do_oop_nv(p); } - inline bool do_metadata_nv() { return true; } + virtual void do_oop(narrowOop* p) { do_oop_work(p); } + virtual void do_oop(oop* p) { do_oop_work(p); } virtual bool do_metadata() { return true; } }; class ShenandoahMarkRefsClosure : public ShenandoahMarkRefsSuperClosure { +private: + template + inline void do_oop_work(T* p) { work(p); } + public: ShenandoahMarkRefsClosure(ShenandoahObjToScanQueue* q, ReferenceProcessor* rp) : ShenandoahMarkRefsSuperClosure(q, rp) {}; - template - inline void do_oop_nv(T* p) { work(p); } - virtual void do_oop(narrowOop* p) { do_oop_nv(p); } - virtual void do_oop(oop* p) { do_oop_nv(p); } - inline bool do_metadata_nv() { return false; } + virtual void do_oop(narrowOop* p) { do_oop_work(p); } + virtual void do_oop(oop* p) { do_oop_work(p); } virtual bool do_metadata() { return false; } }; class ShenandoahMarkRefsDedupClosure : public ShenandoahMarkRefsSuperClosure { +private: + template + inline void do_oop_work(T* p) { work(p); } + public: ShenandoahMarkRefsDedupClosure(ShenandoahObjToScanQueue* q, ShenandoahStrDedupQueue* dq, ReferenceProcessor* rp) : ShenandoahMarkRefsSuperClosure(q, dq, rp) {}; - template - inline void do_oop_nv(T* p) { work(p); } - virtual void do_oop(narrowOop* p) { do_oop_nv(p); } - virtual void do_oop(oop* p) { do_oop_nv(p); } - inline bool do_metadata_nv() { return false; } + virtual void do_oop(narrowOop* p) { do_oop_work(p); } + virtual void do_oop(oop* p) { do_oop_work(p); } virtual bool do_metadata() { return false; } }; class ShenandoahMarkResolveRefsClosure : public ShenandoahMarkRefsSuperClosure { +private: + template + inline void do_oop_work(T* p) { work(p); } + public: ShenandoahMarkResolveRefsClosure(ShenandoahObjToScanQueue* q, ReferenceProcessor* rp) : ShenandoahMarkRefsSuperClosure(q, rp) {}; - template - inline void do_oop_nv(T* p) { work(p); } - virtual void do_oop(narrowOop* p) { do_oop_nv(p); } - virtual void do_oop(oop* p) { do_oop_nv(p); } - inline bool do_metadata_nv() { return false; } + virtual void do_oop(narrowOop* p) { do_oop_work(p); } + virtual void do_oop(oop* p) { do_oop_work(p); } virtual bool do_metadata() { return false; } }; class ShenandoahMarkResolveRefsDedupClosure : public ShenandoahMarkRefsSuperClosure { +private: + template + inline void do_oop_work(T* p) { work(p); } + public: ShenandoahMarkResolveRefsDedupClosure(ShenandoahObjToScanQueue* q, ShenandoahStrDedupQueue* dq, ReferenceProcessor* rp) : ShenandoahMarkRefsSuperClosure(q, dq, rp) {}; - template - inline void do_oop_nv(T* p) { work(p); } - virtual void do_oop(narrowOop* p) { do_oop_nv(p); } - virtual void do_oop(oop* p) { do_oop_nv(p); } - inline bool do_metadata_nv() { return false; } + virtual void do_oop(narrowOop* p) { do_oop_work(p); } + virtual void do_oop(oop* p) { do_oop_work(p); } virtual bool do_metadata() { return false; } }; class ShenandoahMarkRefsMetadataClosure : public ShenandoahMarkRefsSuperClosure { +private: + template + inline void do_oop_work(T* p) { work(p); } + public: ShenandoahMarkRefsMetadataClosure(ShenandoahObjToScanQueue* q, ReferenceProcessor* rp) : ShenandoahMarkRefsSuperClosure(q, rp) {}; - template - inline void do_oop_nv(T* p) { work(p); } - virtual void do_oop(narrowOop* p) { do_oop_nv(p); } - virtual void do_oop(oop* p) { do_oop_nv(p); } - inline bool do_metadata_nv() { return true; } + virtual void do_oop(narrowOop* p) { do_oop_work(p); } + virtual void do_oop(oop* p) { do_oop_work(p); } virtual bool do_metadata() { return true; } }; class ShenandoahMarkRefsMetadataDedupClosure : public ShenandoahMarkRefsSuperClosure { +private: + template + inline void do_oop_work(T* p) { work(p); } + public: ShenandoahMarkRefsMetadataDedupClosure(ShenandoahObjToScanQueue* q, ShenandoahStrDedupQueue* dq, ReferenceProcessor* rp) : ShenandoahMarkRefsSuperClosure(q, dq, rp) {}; - template - inline void do_oop_nv(T* p) { work(p); } - virtual void do_oop(narrowOop* p) { do_oop_nv(p); } - virtual void do_oop(oop* p) { do_oop_nv(p); } - inline bool do_metadata_nv() { return true; } + virtual void do_oop(narrowOop* p) { do_oop_work(p); } + virtual void do_oop(oop* p) { do_oop_work(p); } virtual bool do_metadata() { return true; } }; -class ShenandoahUpdateHeapRefsSuperClosure : public ExtendedOopClosure { +class ShenandoahUpdateHeapRefsSuperClosure : public BasicOopIterateClosure { private: ShenandoahHeap* _heap; public: @@ -193,13 +205,15 @@ }; class ShenandoahUpdateHeapRefsClosure : public ShenandoahUpdateHeapRefsSuperClosure { +private: + template + inline void do_oop_work(T* p) { work(p); } + public: ShenandoahUpdateHeapRefsClosure() : ShenandoahUpdateHeapRefsSuperClosure() {} - template - inline void do_oop_nv(T* p) { work(p); } - virtual void do_oop(narrowOop* p) { do_oop_nv(p); } - virtual void do_oop(oop* p) { do_oop_nv(p); } + virtual void do_oop(narrowOop* p) { do_oop_work(p); } + virtual void do_oop(oop* p) { do_oop_work(p); } }; class ShenandoahUpdateHeapRefsMatrixClosure : public ShenandoahUpdateHeapRefsSuperClosure { @@ -207,12 +221,12 @@ ShenandoahUpdateHeapRefsMatrixClosure() : ShenandoahUpdateHeapRefsSuperClosure() {} template - inline void do_oop_nv(T* p) { work(p); } - virtual void do_oop(narrowOop* p) { do_oop_nv(p); } - virtual void do_oop(oop* p) { do_oop_nv(p); } + inline void do_oop_work(T* p) { work(p); } + virtual void do_oop(narrowOop* p) { do_oop_work(p); } + virtual void do_oop(oop* p) { do_oop_work(p); } }; -class ShenandoahTraversalSuperClosure : public MetadataAwareOopClosure { +class ShenandoahTraversalSuperClosure : public MetadataVisitingOopIterateClosure { private: ShenandoahTraversalGC* _traversal_gc; Thread* _thread; @@ -221,13 +235,13 @@ oop _base_obj; protected: ShenandoahTraversalSuperClosure(ShenandoahObjToScanQueue* q, ReferenceProcessor* rp) : - MetadataAwareOopClosure(rp), + MetadataVisitingOopIterateClosure(rp), _traversal_gc(ShenandoahHeap::heap()->traversal_gc()), _thread(Thread::current()), _queue(q), _dedup_queue(NULL), _base_obj(NULL) { } ShenandoahTraversalSuperClosure(ShenandoahObjToScanQueue* q, ReferenceProcessor* rp, ShenandoahStrDedupQueue* dq) : - MetadataAwareOopClosure(rp), + MetadataVisitingOopIterateClosure(rp), _traversal_gc(ShenandoahHeap::heap()->traversal_gc()), _thread(Thread::current()), _queue(q), _dedup_queue(dq) { } @@ -242,242 +256,242 @@ }; class ShenandoahTraversalClosure : public ShenandoahTraversalSuperClosure { +private: + template + inline void do_oop_work(T* p) { work(p); } + public: ShenandoahTraversalClosure(ShenandoahObjToScanQueue* q, ReferenceProcessor* rp) : ShenandoahTraversalSuperClosure(q, rp) {} - template - inline void do_oop_nv(T* p) { work(p); } + virtual void do_oop(narrowOop* p) { do_oop_work(p); } + virtual void do_oop(oop* p) { do_oop_work(p); } - virtual void do_oop(narrowOop* p) { do_oop_nv(p); } - virtual void do_oop(oop* p) { do_oop_nv(p); } - - inline bool do_metadata_nv() { return false; } virtual bool do_metadata() { return false; } }; class ShenandoahTraversalMetadataClosure : public ShenandoahTraversalSuperClosure { +private: + template + inline void do_oop_work(T* p) { work(p); } + public: ShenandoahTraversalMetadataClosure(ShenandoahObjToScanQueue* q, ReferenceProcessor* rp) : ShenandoahTraversalSuperClosure(q, rp) {} - template - inline void do_oop_nv(T* p) { work(p); } + virtual void do_oop(narrowOop* p) { do_oop_work(p); } + virtual void do_oop(oop* p) { do_oop_work(p); } - virtual void do_oop(narrowOop* p) { do_oop_nv(p); } - virtual void do_oop(oop* p) { do_oop_nv(p); } - - inline bool do_metadata_nv() { return true; } virtual bool do_metadata() { return true; } }; class ShenandoahTraversalDedupClosure : public ShenandoahTraversalSuperClosure { +private: + template + inline void do_oop_work(T* p) { work(p); } + public: ShenandoahTraversalDedupClosure(ShenandoahObjToScanQueue* q, ReferenceProcessor* rp, ShenandoahStrDedupQueue* dq) : ShenandoahTraversalSuperClosure(q, rp, dq) {} - template - inline void do_oop_nv(T* p) { work(p); } + virtual void do_oop(narrowOop* p) { do_oop_work(p); } + virtual void do_oop(oop* p) { do_oop_work(p); } - virtual void do_oop(narrowOop* p) { do_oop_nv(p); } - virtual void do_oop(oop* p) { do_oop_nv(p); } - - inline bool do_metadata_nv() { return false; } virtual bool do_metadata() { return false; } }; class ShenandoahTraversalMetadataDedupClosure : public ShenandoahTraversalSuperClosure { +private: + template + inline void do_oop_work(T* p) { work(p); } + public: ShenandoahTraversalMetadataDedupClosure(ShenandoahObjToScanQueue* q, ReferenceProcessor* rp, ShenandoahStrDedupQueue* dq) : ShenandoahTraversalSuperClosure(q, rp, dq) {} - template - inline void do_oop_nv(T* p) { work(p); } + virtual void do_oop(narrowOop* p) { do_oop_work(p); } + virtual void do_oop(oop* p) { do_oop_work(p); } - virtual void do_oop(narrowOop* p) { do_oop_nv(p); } - virtual void do_oop(oop* p) { do_oop_nv(p); } - - inline bool do_metadata_nv() { return true; } virtual bool do_metadata() { return true; } }; class ShenandoahTraversalDegenClosure : public ShenandoahTraversalSuperClosure { +private: + template + inline void do_oop_work(T* p) { work(p); } + public: ShenandoahTraversalDegenClosure(ShenandoahObjToScanQueue* q, ReferenceProcessor* rp) : ShenandoahTraversalSuperClosure(q, rp) {} - template - inline void do_oop_nv(T* p) { work(p); } + virtual void do_oop(narrowOop* p) { do_oop_work(p); } + virtual void do_oop(oop* p) { do_oop_work(p); } - virtual void do_oop(narrowOop* p) { do_oop_nv(p); } - virtual void do_oop(oop* p) { do_oop_nv(p); } - - inline bool do_metadata_nv() { return false; } virtual bool do_metadata() { return false; } }; class ShenandoahTraversalMetadataDegenClosure : public ShenandoahTraversalSuperClosure { +private: + template + inline void do_oop_work(T* p) { work(p); } + public: ShenandoahTraversalMetadataDegenClosure(ShenandoahObjToScanQueue* q, ReferenceProcessor* rp) : ShenandoahTraversalSuperClosure(q, rp) {} - template - inline void do_oop_nv(T* p) { work(p); } + virtual void do_oop(narrowOop* p) { do_oop_work(p); } + virtual void do_oop(oop* p) { do_oop_work(p); } - virtual void do_oop(narrowOop* p) { do_oop_nv(p); } - virtual void do_oop(oop* p) { do_oop_nv(p); } - - inline bool do_metadata_nv() { return true; } virtual bool do_metadata() { return true; } }; class ShenandoahTraversalDedupDegenClosure : public ShenandoahTraversalSuperClosure { +private: + template + inline void do_oop_work(T* p) { work(p); } + public: ShenandoahTraversalDedupDegenClosure(ShenandoahObjToScanQueue* q, ReferenceProcessor* rp, ShenandoahStrDedupQueue* dq) : ShenandoahTraversalSuperClosure(q, rp, dq) {} - template - inline void do_oop_nv(T* p) { work(p); } + virtual void do_oop(narrowOop* p) { do_oop_work(p); } + virtual void do_oop(oop* p) { do_oop_work(p); } - virtual void do_oop(narrowOop* p) { do_oop_nv(p); } - virtual void do_oop(oop* p) { do_oop_nv(p); } - - inline bool do_metadata_nv() { return false; } virtual bool do_metadata() { return false; } }; class ShenandoahTraversalMetadataDedupDegenClosure : public ShenandoahTraversalSuperClosure { +private: + template + inline void do_oop_work(T* p) { work(p); } + public: ShenandoahTraversalMetadataDedupDegenClosure(ShenandoahObjToScanQueue* q, ReferenceProcessor* rp, ShenandoahStrDedupQueue* dq) : ShenandoahTraversalSuperClosure(q, rp, dq) {} - template - inline void do_oop_nv(T* p) { work(p); } + virtual void do_oop(narrowOop* p) { do_oop_work(p); } + virtual void do_oop(oop* p) { do_oop_work(p); } - virtual void do_oop(narrowOop* p) { do_oop_nv(p); } - virtual void do_oop(oop* p) { do_oop_nv(p); } - - inline bool do_metadata_nv() { return true; } virtual bool do_metadata() { return true; } }; class ShenandoahTraversalMatrixClosure : public ShenandoahTraversalSuperClosure { +private: + template + inline void do_oop_work(T* p) { work(p); } + public: ShenandoahTraversalMatrixClosure(ShenandoahObjToScanQueue* q, ReferenceProcessor* rp) : ShenandoahTraversalSuperClosure(q, rp) {} - template - inline void do_oop_nv(T* p) { work(p); } + virtual void do_oop(narrowOop* p) { do_oop_work(p); } + virtual void do_oop(oop* p) { do_oop_work(p); } - virtual void do_oop(narrowOop* p) { do_oop_nv(p); } - virtual void do_oop(oop* p) { do_oop_nv(p); } - - inline bool do_metadata_nv() { return false; } virtual bool do_metadata() { return false; } }; class ShenandoahTraversalMetadataMatrixClosure : public ShenandoahTraversalSuperClosure { +private: + template + inline void do_oop_work(T* p) { work(p); } + public: ShenandoahTraversalMetadataMatrixClosure(ShenandoahObjToScanQueue* q, ReferenceProcessor* rp) : ShenandoahTraversalSuperClosure(q, rp) {} - template - inline void do_oop_nv(T* p) { work(p); } + virtual void do_oop(narrowOop* p) { do_oop_work(p); } + virtual void do_oop(oop* p) { do_oop_work(p); } - virtual void do_oop(narrowOop* p) { do_oop_nv(p); } - virtual void do_oop(oop* p) { do_oop_nv(p); } - - inline bool do_metadata_nv() { return true; } virtual bool do_metadata() { return true; } }; class ShenandoahTraversalDedupMatrixClosure : public ShenandoahTraversalSuperClosure { +private: + template + inline void do_oop_work(T* p) { work(p); } + public: ShenandoahTraversalDedupMatrixClosure(ShenandoahObjToScanQueue* q, ReferenceProcessor* rp, ShenandoahStrDedupQueue* dq) : ShenandoahTraversalSuperClosure(q, rp, dq) {} - template - inline void do_oop_nv(T* p) { work(p); } + virtual void do_oop(narrowOop* p) { do_oop_work(p); } + virtual void do_oop(oop* p) { do_oop_work(p); } - virtual void do_oop(narrowOop* p) { do_oop_nv(p); } - virtual void do_oop(oop* p) { do_oop_nv(p); } - - inline bool do_metadata_nv() { return false; } virtual bool do_metadata() { return false; } }; class ShenandoahTraversalMetadataDedupMatrixClosure : public ShenandoahTraversalSuperClosure { +private: + template + inline void do_oop_work(T* p) { work(p); } + public: ShenandoahTraversalMetadataDedupMatrixClosure(ShenandoahObjToScanQueue* q, ReferenceProcessor* rp, ShenandoahStrDedupQueue* dq) : ShenandoahTraversalSuperClosure(q, rp, dq) {} - template - inline void do_oop_nv(T* p) { work(p); } + virtual void do_oop(narrowOop* p) { do_oop_work(p); } + virtual void do_oop(oop* p) { do_oop_work(p); } - virtual void do_oop(narrowOop* p) { do_oop_nv(p); } - virtual void do_oop(oop* p) { do_oop_nv(p); } - - inline bool do_metadata_nv() { return true; } virtual bool do_metadata() { return true; } }; class ShenandoahTraversalDegenMatrixClosure : public ShenandoahTraversalSuperClosure { +private: + template + inline void do_oop_work(T* p) { work(p); } + public: ShenandoahTraversalDegenMatrixClosure(ShenandoahObjToScanQueue* q, ReferenceProcessor* rp) : ShenandoahTraversalSuperClosure(q, rp) {} - template - inline void do_oop_nv(T* p) { work(p); } + virtual void do_oop(narrowOop* p) { do_oop_work(p); } + virtual void do_oop(oop* p) { do_oop_work(p); } - virtual void do_oop(narrowOop* p) { do_oop_nv(p); } - virtual void do_oop(oop* p) { do_oop_nv(p); } - - inline bool do_metadata_nv() { return false; } virtual bool do_metadata() { return false; } }; class ShenandoahTraversalMetadataDegenMatrixClosure : public ShenandoahTraversalSuperClosure { +private: + template + inline void do_oop_work(T* p) { work(p); } + public: ShenandoahTraversalMetadataDegenMatrixClosure(ShenandoahObjToScanQueue* q, ReferenceProcessor* rp) : ShenandoahTraversalSuperClosure(q, rp) {} - template - inline void do_oop_nv(T* p) { work(p); } + virtual void do_oop(narrowOop* p) { do_oop_work(p); } + virtual void do_oop(oop* p) { do_oop_work(p); } - virtual void do_oop(narrowOop* p) { do_oop_nv(p); } - virtual void do_oop(oop* p) { do_oop_nv(p); } - - inline bool do_metadata_nv() { return true; } virtual bool do_metadata() { return true; } }; class ShenandoahTraversalDedupDegenMatrixClosure : public ShenandoahTraversalSuperClosure { +private: + template + inline void do_oop_work(T* p) { work(p); } + public: ShenandoahTraversalDedupDegenMatrixClosure(ShenandoahObjToScanQueue* q, ReferenceProcessor* rp, ShenandoahStrDedupQueue* dq) : ShenandoahTraversalSuperClosure(q, rp, dq) {} - template - inline void do_oop_nv(T* p) { work(p); } + virtual void do_oop(narrowOop* p) { do_oop_work(p); } + virtual void do_oop(oop* p) { do_oop_work(p); } - virtual void do_oop(narrowOop* p) { do_oop_nv(p); } - virtual void do_oop(oop* p) { do_oop_nv(p); } - - inline bool do_metadata_nv() { return false; } virtual bool do_metadata() { return false; } }; class ShenandoahTraversalMetadataDedupDegenMatrixClosure : public ShenandoahTraversalSuperClosure { +private: + template + inline void do_oop_work(T* p) { work(p); } + public: ShenandoahTraversalMetadataDedupDegenMatrixClosure(ShenandoahObjToScanQueue* q, ReferenceProcessor* rp, ShenandoahStrDedupQueue* dq) : ShenandoahTraversalSuperClosure(q, rp, dq) {} - template - inline void do_oop_nv(T* p) { work(p); } + virtual void do_oop(narrowOop* p) { do_oop_work(p); } + virtual void do_oop(oop* p) { do_oop_work(p); } - virtual void do_oop(narrowOop* p) { do_oop_nv(p); } - virtual void do_oop(oop* p) { do_oop_nv(p); } - - inline bool do_metadata_nv() { return true; } virtual bool do_metadata() { return true; } }; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahPrinter.cpp b/src/hotspot/share/gc/shenandoah/shenandoahPrinter.cpp --- a/src/hotspot/share/gc/shenandoah/shenandoahPrinter.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahPrinter.cpp @@ -25,11 +25,12 @@ #ifdef ASSERT #include "memory/allocation.hpp" +#include "memory/iterator.inline.hpp" #include "gc/shenandoah/shenandoahHeap.hpp" #include "gc/shenandoah/shenandoahHeap.inline.hpp" #include "gc/shenandoah/shenandoahPrinter.hpp" -class ShenandoahPrintAllRefsOopClosure: public ExtendedOopClosure { +class ShenandoahPrintAllRefsOopClosure: public BasicOopIterateClosure { private: int _index; const char* _prefix; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahStrDedupQueue.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahStrDedupQueue.inline.hpp --- a/src/hotspot/share/gc/shenandoah/shenandoahStrDedupQueue.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahStrDedupQueue.inline.hpp @@ -46,7 +46,7 @@ oop obj = CompressedOops::decode_not_null(o); assert(_heap->is_in(obj), "Must be in the heap"); if (!_heap->is_marked_next(obj)) { - RawAccess::oop_store(p, oop()); + RawAccess::oop_store(p, oop()); } } } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahTraversalGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahTraversalGC.cpp --- a/src/hotspot/share/gc/shenandoah/shenandoahTraversalGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahTraversalGC.cpp @@ -761,7 +761,7 @@ oop obj = CompressedOops::decode_not_null(o); oop forw = ShenandoahBarrierSet::resolve_forwarded_not_null(obj); if (!oopDesc::unsafe_equals(obj, forw)) { - RawAccess::oop_store(p, forw); + RawAccess::oop_store(p, forw); } } } @@ -841,7 +841,7 @@ Thread* _thread; ShenandoahTraversalGC* _traversal_gc; template - inline void do_oop_nv(T* p) { + inline void do_oop_work(T* p) { _traversal_gc->process_oop(p, _thread, _queue, NULL); } @@ -850,8 +850,8 @@ _queue(q), _thread(Thread::current()), _traversal_gc(ShenandoahHeap::heap()->traversal_gc()) {} - void do_oop(narrowOop* p) { do_oop_nv(p); } - void do_oop(oop* p) { do_oop_nv(p); } + void do_oop(narrowOop* p) { do_oop_work(p); } + void do_oop(oop* p) { do_oop_work(p); } }; class ShenandoahTraversalKeepAliveUpdateDegenClosure : public OopClosure { @@ -860,7 +860,7 @@ Thread* _thread; ShenandoahTraversalGC* _traversal_gc; template - inline void do_oop_nv(T* p) { + inline void do_oop_work(T* p) { _traversal_gc->process_oop(p, _thread, _queue, NULL); } @@ -869,8 +869,8 @@ _queue(q), _thread(Thread::current()), _traversal_gc(ShenandoahHeap::heap()->traversal_gc()) {} - void do_oop(narrowOop* p) { do_oop_nv(p); } - void do_oop(oop* p) { do_oop_nv(p); } + void do_oop(narrowOop* p) { do_oop_work(p); } + void do_oop(oop* p) { do_oop_work(p); } }; class ShenandoahTraversalKeepAliveUpdateMatrixClosure : public OopClosure { @@ -879,7 +879,7 @@ Thread* _thread; ShenandoahTraversalGC* _traversal_gc; template - inline void do_oop_nv(T* p) { + inline void do_oop_work(T* p) { // TODO: Need to somehow pass base_obj here? _traversal_gc->process_oop(p, _thread, _queue, NULL); } @@ -889,8 +889,8 @@ _queue(q), _thread(Thread::current()), _traversal_gc(ShenandoahHeap::heap()->traversal_gc()) {} - void do_oop(narrowOop* p) { do_oop_nv(p); } - void do_oop(oop* p) { do_oop_nv(p); } + void do_oop(narrowOop* p) { do_oop_work(p); } + void do_oop(oop* p) { do_oop_work(p); } }; class ShenandoahTraversalKeepAliveUpdateDegenMatrixClosure : public OopClosure { @@ -899,7 +899,7 @@ Thread* _thread; ShenandoahTraversalGC* _traversal_gc; template - inline void do_oop_nv(T* p) { + inline void do_oop_work(T* p) { // TODO: Need to somehow pass base_obj here? _traversal_gc->process_oop(p, _thread, _queue, NULL); } @@ -909,8 +909,8 @@ _queue(q), _thread(Thread::current()), _traversal_gc(ShenandoahHeap::heap()->traversal_gc()) {} - void do_oop(narrowOop* p) { do_oop_nv(p); } - void do_oop(oop* p) { do_oop_nv(p); } + void do_oop(narrowOop* p) { do_oop_work(p); } + void do_oop(oop* p) { do_oop_work(p); } }; void ShenandoahTraversalGC::preclean_weak_refs() { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahTraversalGC.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahTraversalGC.inline.hpp --- a/src/hotspot/share/gc/shenandoah/shenandoahTraversalGC.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahTraversalGC.inline.hpp @@ -45,7 +45,7 @@ oop forw = ShenandoahBarrierSet::resolve_forwarded_not_null(obj); if (!oopDesc::unsafe_equals(obj, forw)) { // Update reference. - RawAccess::oop_store(p, forw); + RawAccess::oop_store(p, forw); } obj = forw; } else if (_heap->in_collection_set(obj)) { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahUtils.cpp b/src/hotspot/share/gc/shenandoah/shenandoahUtils.cpp --- a/src/hotspot/share/gc/shenandoah/shenandoahUtils.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahUtils.cpp @@ -39,6 +39,7 @@ sh->shenandoahPolicy()->record_cycle_start(); sh->heuristics()->record_cycle_start(); _trace_cycle.initialize(sh->cycle_memory_manager(), sh->gc_cause(), + /* allMemoryPoolsAffected */ true, /* recordGCBeginTime = */ true, /* recordPreGCUsage = */ true, /* recordPeakUsage = */ true, @@ -59,6 +60,7 @@ ShenandoahHeap* sh = ShenandoahHeap::heap(); _trace_pause.initialize(sh->stw_memory_manager(), sh->gc_cause(), + /* allMemoryPoolsAffected */ true, /* recordGCBeginTime = */ true, /* recordPreGCUsage = */ false, /* recordPeakUsage = */ false, diff --git a/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp b/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp --- a/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp @@ -33,13 +33,14 @@ #include "gc/shenandoah/shenandoahTaskqueue.hpp" #include "gc/shenandoah/shenandoahVerifier.hpp" #include "memory/allocation.hpp" +#include "memory/iterator.inline.hpp" // Avoid name collision on verify_oop (defined in macroAssembler_arm.hpp) #ifdef verify_oop #undef verify_oop #endif -class ShenandoahVerifyOopClosure : public ExtendedOopClosure { +class ShenandoahVerifyOopClosure : public BasicOopIterateClosure { private: const char* _phase; ShenandoahVerifier::VerifyOptions _options; @@ -57,7 +58,7 @@ _phase(phase), _options(options) {}; private: - void verify(ShenandoahAsserts::SafeLevel level, oop obj, bool test, const char* label) { + void check(ShenandoahAsserts::SafeLevel level, oop obj, bool test, const char* label) { if (!test) { ShenandoahAsserts::print_failure(level, obj, _interior_loc, _loc, _phase, label, __FILE__, __LINE__); } @@ -92,9 +93,9 @@ // that failure report would not try to touch something that was not yet verified to be // safe to process. - verify(ShenandoahAsserts::_safe_unknown, obj, _heap->is_in(obj), + check(ShenandoahAsserts::_safe_unknown, obj, _heap->is_in(obj), "oop must be in heap"); - verify(ShenandoahAsserts::_safe_unknown, obj, check_obj_alignment(obj), + check(ShenandoahAsserts::_safe_unknown, obj, check_obj_alignment(obj), "oop must be aligned"); ShenandoahHeapRegion *obj_reg = _heap->heap_region_containing(obj); @@ -103,30 +104,30 @@ // Verify that obj is not in dead space: { // Do this before touching obj->size() - verify(ShenandoahAsserts::_safe_unknown, obj, obj_klass != NULL, + check(ShenandoahAsserts::_safe_unknown, obj, obj_klass != NULL, "Object klass pointer should not be NULL"); - verify(ShenandoahAsserts::_safe_unknown, obj, Metaspace::contains(obj_klass), + check(ShenandoahAsserts::_safe_unknown, obj, Metaspace::contains(obj_klass), "Object klass pointer must go to metaspace"); HeapWord *obj_addr = (HeapWord *) obj; - verify(ShenandoahAsserts::_safe_unknown, obj, obj_addr < obj_reg->top(), + check(ShenandoahAsserts::_safe_unknown, obj, obj_addr < obj_reg->top(), "Object start should be within the region"); if (!obj_reg->is_humongous()) { - verify(ShenandoahAsserts::_safe_unknown, obj, (obj_addr + obj->size()) <= obj_reg->top(), + check(ShenandoahAsserts::_safe_unknown, obj, (obj_addr + obj->size()) <= obj_reg->top(), "Object end should be within the region"); } else { size_t humongous_start = obj_reg->region_number(); size_t humongous_end = humongous_start + (obj->size() >> ShenandoahHeapRegion::region_size_words_shift()); for (size_t idx = humongous_start + 1; idx < humongous_end; idx++) { - verify(ShenandoahAsserts::_safe_unknown, obj, _heap->get_region(idx)->is_humongous_continuation(), + check(ShenandoahAsserts::_safe_unknown, obj, _heap->get_region(idx)->is_humongous_continuation(), "Humongous object is in continuation that fits it"); } } // ------------ obj is safe at this point -------------- - verify(ShenandoahAsserts::_safe_oop, obj, obj_reg->is_active(), + check(ShenandoahAsserts::_safe_oop, obj, obj_reg->is_active(), "Object should be in active region"); switch (_options._verify_liveness) { @@ -137,7 +138,7 @@ Atomic::add(obj->size() + BrooksPointer::word_size(), &_ld[obj_reg->region_number()]); // fallthrough for fast failure for un-live regions: case ShenandoahVerifier::_verify_liveness_conservative: - verify(ShenandoahAsserts::_safe_oop, obj, obj_reg->has_live(), + check(ShenandoahAsserts::_safe_oop, obj, obj_reg->has_live(), "Object must belong to region with live data"); break; default: @@ -150,36 +151,36 @@ ShenandoahHeapRegion* fwd_reg = NULL; if (!oopDesc::unsafe_equals(obj, fwd)) { - verify(ShenandoahAsserts::_safe_oop, obj, _heap->is_in(fwd), + check(ShenandoahAsserts::_safe_oop, obj, _heap->is_in(fwd), "Forwardee must be in heap"); - verify(ShenandoahAsserts::_safe_oop, obj, !CompressedOops::is_null(fwd), + check(ShenandoahAsserts::_safe_oop, obj, !CompressedOops::is_null(fwd), "Forwardee is set"); - verify(ShenandoahAsserts::_safe_oop, obj, check_obj_alignment(fwd), + check(ShenandoahAsserts::_safe_oop, obj, check_obj_alignment(fwd), "Forwardee must be aligned"); // Do this before touching fwd->size() Klass* fwd_klass = fwd->klass_or_null(); - verify(ShenandoahAsserts::_safe_oop, obj, fwd_klass != NULL, + check(ShenandoahAsserts::_safe_oop, obj, fwd_klass != NULL, "Forwardee klass pointer should not be NULL"); - verify(ShenandoahAsserts::_safe_oop, obj, Metaspace::contains(fwd_klass), + check(ShenandoahAsserts::_safe_oop, obj, Metaspace::contains(fwd_klass), "Forwardee klass pointer must go to metaspace"); - verify(ShenandoahAsserts::_safe_oop, obj, obj_klass == fwd_klass, + check(ShenandoahAsserts::_safe_oop, obj, obj_klass == fwd_klass, "Forwardee klass pointer must go to metaspace"); fwd_reg = _heap->heap_region_containing(fwd); // Verify that forwardee is not in the dead space: - verify(ShenandoahAsserts::_safe_oop, obj, !fwd_reg->is_humongous(), + check(ShenandoahAsserts::_safe_oop, obj, !fwd_reg->is_humongous(), "Should have no humongous forwardees"); HeapWord *fwd_addr = (HeapWord *) fwd; - verify(ShenandoahAsserts::_safe_oop, obj, fwd_addr < fwd_reg->top(), + check(ShenandoahAsserts::_safe_oop, obj, fwd_addr < fwd_reg->top(), "Forwardee start should be within the region"); - verify(ShenandoahAsserts::_safe_oop, obj, (fwd_addr + fwd->size()) <= fwd_reg->top(), + check(ShenandoahAsserts::_safe_oop, obj, (fwd_addr + fwd->size()) <= fwd_reg->top(), "Forwardee end should be within the region"); oop fwd2 = (oop) BrooksPointer::get_raw_unchecked(fwd); - verify(ShenandoahAsserts::_safe_oop, obj, oopDesc::unsafe_equals(fwd, fwd2), + check(ShenandoahAsserts::_safe_oop, obj, oopDesc::unsafe_equals(fwd, fwd2), "Double forwarding"); } else { fwd_reg = obj_reg; @@ -192,11 +193,11 @@ // skip break; case ShenandoahVerifier::_verify_marked_next: - verify(ShenandoahAsserts::_safe_all, obj, _heap->is_marked_next(obj), + check(ShenandoahAsserts::_safe_all, obj, _heap->is_marked_next(obj), "Must be marked in next bitmap"); break; case ShenandoahVerifier::_verify_marked_complete: - verify(ShenandoahAsserts::_safe_all, obj, _heap->is_marked_complete(obj), + check(ShenandoahAsserts::_safe_all, obj, _heap->is_marked_complete(obj), "Must be marked in complete bitmap"); break; default: @@ -208,13 +209,13 @@ // skip break; case ShenandoahVerifier::_verify_forwarded_none: { - verify(ShenandoahAsserts::_safe_all, obj, oopDesc::unsafe_equals(obj, fwd), + check(ShenandoahAsserts::_safe_all, obj, oopDesc::unsafe_equals(obj, fwd), "Should not be forwarded"); break; } case ShenandoahVerifier::_verify_forwarded_allow: { if (!oopDesc::unsafe_equals(obj, fwd)) { - verify(ShenandoahAsserts::_safe_all, obj, obj_reg != fwd_reg, + check(ShenandoahAsserts::_safe_all, obj, obj_reg != fwd_reg, "Forwardee should be in another region"); } break; @@ -228,12 +229,12 @@ // skip break; case ShenandoahVerifier::_verify_cset_none: - verify(ShenandoahAsserts::_safe_all, obj, !_heap->in_collection_set(obj), + check(ShenandoahAsserts::_safe_all, obj, !_heap->in_collection_set(obj), "Should not have references to collection set"); break; case ShenandoahVerifier::_verify_cset_forwarded: if (_heap->in_collection_set(obj)) { - verify(ShenandoahAsserts::_safe_all, obj, !oopDesc::unsafe_equals(obj, fwd), + check(ShenandoahAsserts::_safe_all, obj, !oopDesc::unsafe_equals(obj, fwd), "Object in collection set, should have forwardee"); } break; @@ -251,7 +252,7 @@ size_t from_idx = _heap->heap_region_index_containing(interior); size_t to_idx = _heap->heap_region_index_containing(obj); _interior_loc = interior; - verify(ShenandoahAsserts::_safe_all, obj, _heap->connection_matrix()->is_connected(from_idx, to_idx), + check(ShenandoahAsserts::_safe_all, obj, _heap->connection_matrix()->is_connected(from_idx, to_idx), "Must be connected"); _interior_loc = NULL; break; @@ -299,8 +300,8 @@ _loc = NULL; } - void do_oop(oop* p) { do_oop_work(p); } - void do_oop(narrowOop* p) { do_oop_work(p); } + virtual void do_oop(oop* p) { do_oop_work(p); } + virtual void do_oop(narrowOop* p) { do_oop_work(p); } }; class ShenandoahCalculateRegionStatsClosure : public ShenandoahHeapRegionClosure { diff --git a/src/hotspot/share/gc/shenandoah/shenandoah_specialized_oop_closures.hpp b/src/hotspot/share/gc/shenandoah/shenandoah_specialized_oop_closures.hpp deleted file mode 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoah_specialized_oop_closures.hpp +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright (c) 2015, Red Hat, Inc. and/or its affiliates. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_SHENANDOAH_SHENANDOAH_SPECIALIZED_OOP_CLOSURES_HPP -#define SHARE_VM_GC_SHENANDOAH_SHENANDOAH_SPECIALIZED_OOP_CLOSURES_HPP - -class ShenandoahMarkUpdateRefsClosure; -class ShenandoahMarkUpdateRefsDedupClosure; -class ShenandoahMarkUpdateRefsMetadataClosure; -class ShenandoahMarkUpdateRefsMetadataDedupClosure; -class ShenandoahMarkRefsClosure; -class ShenandoahMarkRefsDedupClosure; -class ShenandoahMarkRefsMetadataClosure; -class ShenandoahMarkRefsMetadataDedupClosure; -class ShenandoahUpdateHeapRefsClosure; -class ShenandoahUpdateHeapRefsMatrixClosure; -class ShenandoahTraversalClosure; -class ShenandoahTraversalMetadataClosure; -class ShenandoahTraversalDedupClosure; -class ShenandoahTraversalMetadataDedupClosure; - -#define SPECIALIZED_OOP_OOP_ITERATE_CLOSURES_SHENANDOAH(f) \ - f(ShenandoahMarkUpdateRefsClosure,_nv) \ - f(ShenandoahMarkUpdateRefsMetadataClosure,_nv) \ - f(ShenandoahMarkRefsClosure,_nv) \ - f(ShenandoahMarkRefsMetadataClosure,_nv) \ - f(ShenandoahUpdateHeapRefsClosure,_nv) \ - f(ShenandoahUpdateHeapRefsMatrixClosure,_nv) \ - f(ShenandoahTraversalClosure,_nv) \ - f(ShenandoahTraversalMetadataClosure,_nv) \ - f(ShenandoahMarkUpdateRefsDedupClosure,_nv) \ - f(ShenandoahMarkUpdateRefsMetadataDedupClosure,_nv) \ - f(ShenandoahMarkRefsDedupClosure,_nv) \ - f(ShenandoahMarkRefsMetadataDedupClosure,_nv) \ - f(ShenandoahTraversalDedupClosure,_nv) \ - f(ShenandoahTraversalMetadataDedupClosure,_nv) - -#endif // SHARE_VM_GC_SHENANDOAH_SHENANDOAH_SPECIALIZED_OOP_CLOSURES_HPP