# HG changeset patch # Parent 2795496dbaf3401b91090a40adc4ccf50494bbf9 diff --git a/make/hotspot/lib/JvmOverrideFiles.gmk b/make/hotspot/lib/JvmOverrideFiles.gmk --- a/make/hotspot/lib/JvmOverrideFiles.gmk +++ b/make/hotspot/lib/JvmOverrideFiles.gmk @@ -34,6 +34,7 @@ BUILD_LIBJVM_jvmciCompilerToVM.cpp_CXXFLAGS := -fno-var-tracking-assignments # Need extra inlining to collapse all the templated closures into the hot loop BUILD_LIBJVM_shenandoahConcurrentMark.cpp_CXXFLAGS := --param inline-unit-growth=1000 + BUILD_LIBJVM_shenandoahTraversalGC.cpp_CXXFLAGS := --param inline-unit-growth=1000 endif ifeq ($(OPENJDK_TARGET_OS), linux) diff --git a/src/hotspot/cpu/x86/c1_Runtime1_x86.cpp b/src/hotspot/cpu/x86/c1_Runtime1_x86.cpp --- a/src/hotspot/cpu/x86/c1_Runtime1_x86.cpp +++ b/src/hotspot/cpu/x86/c1_Runtime1_x86.cpp @@ -1615,7 +1615,7 @@ break; } - if (bs->kind() == BarrierSet::ShenandoahBarrierSet && !ShenandoahSATBBarrier && !ShenandoahConditionalSATBBarrier) { + if (bs->kind() == BarrierSet::ShenandoahBarrierSet && !ShenandoahSATBBarrier && !ShenandoahConditionalSATBBarrier && !ShenandoahStoreValEnqueueBarrier) { break; } diff --git a/src/hotspot/cpu/x86/shenandoahBarrierSet_x86.cpp b/src/hotspot/cpu/x86/shenandoahBarrierSet_x86.cpp --- a/src/hotspot/cpu/x86/shenandoahBarrierSet_x86.cpp +++ b/src/hotspot/cpu/x86/shenandoahBarrierSet_x86.cpp @@ -150,7 +150,7 @@ #endif } -void ShenandoahBarrierSet::interpreter_storeval_barrier(MacroAssembler* masm, Register dst) { +void ShenandoahBarrierSet::interpreter_storeval_barrier(MacroAssembler* masm, Register dst, Register tmp, Register thread) { if (ShenandoahStoreValWriteBarrier) { Label is_null; __ testptr(dst, dst); @@ -158,6 +158,24 @@ interpreter_write_barrier_impl(masm, dst); __ bind(is_null); } + + if (ShenandoahStoreValEnqueueBarrier) { + __ push(rbx); + __ push(rcx); + __ push(rdx); + __ push(c_rarg1); + __ subptr(rsp, 2 * Interpreter::stackElementSize); + __ movdbl(Address(rsp, 0), xmm0); + + __ g1_write_barrier_pre(noreg, dst, r15_thread, tmp, true, false); + + __ movdbl(xmm0, Address(rsp, 0)); + __ addptr(rsp, 2 * Interpreter::stackElementSize); + __ pop(c_rarg1); + __ pop(rdx); + __ pop(rcx); + __ pop(rbx); + } if (ShenandoahStoreValReadBarrier) { interpreter_read_barrier_impl(masm, dst); } diff --git a/src/hotspot/cpu/x86/templateInterpreterGenerator_x86.cpp b/src/hotspot/cpu/x86/templateInterpreterGenerator_x86.cpp --- a/src/hotspot/cpu/x86/templateInterpreterGenerator_x86.cpp +++ b/src/hotspot/cpu/x86/templateInterpreterGenerator_x86.cpp @@ -727,7 +727,7 @@ const int referent_offset = java_lang_ref_Reference::referent_offset; guarantee(referent_offset > 0, "referent offset not initialized"); - if (UseG1GC || UseShenandoahGC) { + if (UseG1GC || (UseShenandoahGC && ShenandoahKeepAliveBarrier)) { Label slow_path; // rbx: method diff --git a/src/hotspot/cpu/x86/templateTable_x86.cpp b/src/hotspot/cpu/x86/templateTable_x86.cpp --- a/src/hotspot/cpu/x86/templateTable_x86.cpp +++ b/src/hotspot/cpu/x86/templateTable_x86.cpp @@ -194,7 +194,7 @@ __ store_heap_oop_null(Address(rdx, 0)); } else { // For Shenandoah, make sure we only store refs into to-space. - oopDesc::bs()->interpreter_storeval_barrier(_masm, val); + oopDesc::bs()->interpreter_storeval_barrier(_masm, val, rtmp, rthread); // G1 barrier needs uncompressed oop for region cross check. Register new_val = val; diff --git a/src/hotspot/share/c1/c1_LIRGenerator.cpp b/src/hotspot/share/c1/c1_LIRGenerator.cpp --- a/src/hotspot/share/c1/c1_LIRGenerator.cpp +++ b/src/hotspot/share/c1/c1_LIRGenerator.cpp @@ -2100,10 +2100,13 @@ __ move(obj, result); obj = result; } - return shenandoah_write_barrier_impl(obj, info, need_null_check); + obj = shenandoah_write_barrier_impl(obj, info, need_null_check); + } + if (ShenandoahStoreValEnqueueBarrier) { + G1SATBCardTableModRef_pre_barrier(LIR_OprFact::illegalOpr, obj, false, false, NULL); } if (ShenandoahStoreValReadBarrier) { - return shenandoah_read_barrier_impl(obj, info, need_null_check); + obj = shenandoah_read_barrier_impl(obj, info, need_null_check); } } return obj; diff --git a/src/hotspot/share/gc/shared/barrierSet.hpp b/src/hotspot/share/gc/shared/barrierSet.hpp --- a/src/hotspot/share/gc/shared/barrierSet.hpp +++ b/src/hotspot/share/gc/shared/barrierSet.hpp @@ -185,7 +185,7 @@ virtual void interpreter_write_barrier(MacroAssembler* masm, Register dst) { // Default implementation does nothing. } - virtual void interpreter_storeval_barrier(MacroAssembler* masm, Register dst) { + virtual void interpreter_storeval_barrier(MacroAssembler* masm, Register dst, Register tmp, Register thread) { // Default implementation does nothing. } virtual void asm_acmp_barrier(MacroAssembler* masm, Register op1, Register op2) { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp --- a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp @@ -29,7 +29,7 @@ #include "gc/shenandoah/shenandoahHeap.inline.hpp" #include "runtime/interfaceSupport.hpp" -template +template class ShenandoahUpdateRefsForOopClosure: public ExtendedOopClosure { private: ShenandoahHeap* _heap; @@ -39,8 +39,10 @@ if (STOREVAL_WRITE_BARRIER) { bool evac; o = _heap->evac_update_oop_ref(p, evac); - if (evac) { - G1SATBCardTableModRefBS::enqueue(o); + if (ALWAYS_ENQUEUE && !oopDesc::is_null(o)) { + ShenandoahBarrierSet::enqueue(o); + } else if (evac) { + ShenandoahBarrierSet::enqueue(o); } } else { o = _heap->maybe_update_oop_ref(p); @@ -167,7 +169,7 @@ } bool ShenandoahBarrierSet::need_update_refs_barrier() { - if (UseShenandoahMatrix) { + if (_heap->is_concurrent_partial_in_progress() || _heap->is_concurrent_traversal_in_progress()) { return true; } if (_heap->shenandoahPolicy()->update_refs()) { @@ -181,10 +183,10 @@ ShouldNotReachHere(); } -template +template void ShenandoahBarrierSet::write_ref_array_loop(HeapWord* start, size_t count) { assert(UseShenandoahGC && ShenandoahCloneBarrier, "should be enabled"); - ShenandoahUpdateRefsForOopClosure cl; + ShenandoahUpdateRefsForOopClosure cl; T* dst = (T*) start; for (size_t i = 0; i < count; i++) { cl.do_oop(dst++); @@ -199,24 +201,32 @@ if (UseCompressedOops) { if (UseShenandoahMatrix) { if (_heap->is_concurrent_partial_in_progress()) { - write_ref_array_loop(start, count); + write_ref_array_loop(start, count); } else { - write_ref_array_loop(start, count); + write_ref_array_loop(start, count); } } else { assert(! _heap->is_concurrent_partial_in_progress(), "partial GC needs matrix"); - write_ref_array_loop(start, count); + if (!_heap->is_concurrent_traversal_in_progress()) { + write_ref_array_loop(start, count); + } else { + write_ref_array_loop(start, count); + } } } else { if (UseShenandoahMatrix) { if (_heap->is_concurrent_partial_in_progress()) { - write_ref_array_loop(start, count); + write_ref_array_loop(start, count); } else { - write_ref_array_loop(start, count); + write_ref_array_loop(start, count); } } else { assert(! _heap->is_concurrent_partial_in_progress(), "partial GC needs matrix"); - write_ref_array_loop(start, count); + if (_heap->is_concurrent_traversal_in_progress()) { + write_ref_array_loop(start, count); + } else { + write_ref_array_loop(start, count); + } } } } @@ -240,7 +250,7 @@ for (int i = 0; i < count; i++, elem_ptr++) { T heap_oop = oopDesc::load_heap_oop(elem_ptr); if (!oopDesc::is_null(heap_oop)) { - G1SATBCardTableModRefBS::enqueue(oopDesc::decode_heap_oop_not_null(heap_oop)); + enqueue(oopDesc::decode_heap_oop_not_null(heap_oop)); } } } @@ -279,7 +289,7 @@ if (_heap->is_concurrent_mark_in_progress()) { T heap_oop = oopDesc::load_heap_oop(field); if (!oopDesc::is_null(heap_oop)) { - G1SATBCardTableModRefBS::enqueue(oopDesc::decode_heap_oop(heap_oop)); + enqueue(oopDesc::decode_heap_oop(heap_oop)); } } if (UseShenandoahMatrix && ! oopDesc::is_null(new_val)) { @@ -331,16 +341,21 @@ assert(oopDesc::is_oop(obj), "must be an oop"); if (UseShenandoahMatrix) { if (_heap->is_concurrent_partial_in_progress()) { - ShenandoahUpdateRefsForOopClosure cl; + ShenandoahUpdateRefsForOopClosure cl; obj->oop_iterate(&cl); } else { - ShenandoahUpdateRefsForOopClosure cl; + ShenandoahUpdateRefsForOopClosure cl; obj->oop_iterate(&cl); } } else { assert(! _heap->is_concurrent_partial_in_progress(), "partial GC needs matrix"); - ShenandoahUpdateRefsForOopClosure cl; - obj->oop_iterate(&cl); + if (_heap->is_concurrent_traversal_in_progress()) { + ShenandoahUpdateRefsForOopClosure cl; + obj->oop_iterate(&cl); + } else { + ShenandoahUpdateRefsForOopClosure cl; + obj->oop_iterate(&cl); + } } } @@ -406,7 +421,7 @@ bool evac; oop copy = _heap->evacuate_object(obj, Thread::current(), evac, true /* from write barrier */); if (evac && _heap->is_concurrent_partial_in_progress()) { - G1SATBCardTableModRefBS::enqueue(copy); + enqueue(copy); } return copy; } else { @@ -427,10 +442,13 @@ oop ShenandoahBarrierSet::storeval_barrier(oop obj) { if (ShenandoahStoreValWriteBarrier) { - return write_barrier_impl(obj); + obj = write_barrier(obj); + } + if (ShenandoahStoreValEnqueueBarrier && !oopDesc::is_null(obj)) { + enqueue(obj); } if (ShenandoahStoreValReadBarrier) { - return resolve_oop_static(obj); + obj = resolve_oop_static(obj); } return obj; } @@ -438,13 +456,19 @@ void ShenandoahBarrierSet::keep_alive_barrier(oop obj) { if (ShenandoahKeepAliveBarrier) { if (_heap->is_concurrent_mark_in_progress()) { - G1SATBCardTableModRefBS::enqueue(obj); + enqueue(obj); } else if (_heap->is_concurrent_partial_in_progress()) { write_barrier_impl(obj); } } } +void ShenandoahBarrierSet::enqueue(oop obj) { + assert(oopDesc::unsafe_equals(obj, ShenandoahBarrierSet::resolve_oop_static_not_null(obj)), "only enqueue to-space oops"); + assert(!ShenandoahHeap::heap()->in_collection_set(obj), "no cset objects please"); + G1SATBCardTableModRefBS::enqueue(obj); +} + #ifdef ASSERT void ShenandoahBarrierSet::verify_safe_oop(oop p) { ShenandoahHeap* heap = ShenandoahHeap::heap(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.hpp b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.hpp --- a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.hpp @@ -111,10 +111,12 @@ virtual void verify_safe_oop(oop p); #endif + static void enqueue(oop obj); + private: bool need_update_refs_barrier(); - template + template void write_ref_array_loop(HeapWord* start, size_t count); oop write_barrier_impl(oop obj); @@ -124,7 +126,7 @@ void interpreter_read_barrier(MacroAssembler* masm, Register dst); void interpreter_read_barrier_not_null(MacroAssembler* masm, Register dst); void interpreter_write_barrier(MacroAssembler* masm, Register dst); - void interpreter_storeval_barrier(MacroAssembler* masm, Register dst); + void interpreter_storeval_barrier(MacroAssembler* masm, Register dst, Register tmp, Register thread); void asm_acmp_barrier(MacroAssembler* masm, Register op1, Register op2); private: @@ -132,6 +134,7 @@ void interpreter_read_barrier_not_null_impl(MacroAssembler* masm, Register dst); void interpreter_write_barrier_impl(MacroAssembler* masm, Register dst); + #endif }; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahCollectorPolicy.cpp b/src/hotspot/share/gc/shenandoah/shenandoahCollectorPolicy.cpp --- a/src/hotspot/share/gc/shenandoah/shenandoahCollectorPolicy.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahCollectorPolicy.cpp @@ -180,6 +180,14 @@ return false; } + virtual bool should_start_traversal_gc() { + return false; + } + + virtual bool can_do_traversal_gc() { + return false; + } + virtual bool handover_cancelled_marking() { return _cancelled_cm_cycles_in_a_row <= ShenandoahFullGCThreshold; } @@ -1316,6 +1324,81 @@ }; +class ShenandoahTraversalHeuristics : public ShenandoahHeuristics { +public: + ShenandoahTraversalHeuristics() : ShenandoahHeuristics() { + FLAG_SET_DEFAULT(UseShenandoahMatrix, false); + FLAG_SET_DEFAULT(ShenandoahSATBBarrier, false); + FLAG_SET_DEFAULT(ShenandoahConditionalSATBBarrier, false); + FLAG_SET_DEFAULT(ShenandoahStoreValReadBarrier, false); + FLAG_SET_DEFAULT(ShenandoahStoreValWriteBarrier, true); + FLAG_SET_DEFAULT(ShenandoahStoreValEnqueueBarrier, true); + FLAG_SET_DEFAULT(ShenandoahKeepAliveBarrier, false); + FLAG_SET_DEFAULT(ShenandoahAsmWB, true); + FLAG_SET_DEFAULT(ShenandoahBarriersForConst, true); + FLAG_SET_DEFAULT(ShenandoahWBWithMemBar, false); + FLAG_SET_DEFAULT(ShenandoahWriteBarrierRB, false); + } + + virtual bool should_start_concurrent_mark(size_t used, size_t capacity) const { + return false; + } + + virtual bool is_experimental() { + return true; + } + + virtual bool is_diagnostic() { + return false; + } + + virtual bool can_do_traversal_gc() { + return true; + } + + virtual const char* name() { + return "traversal"; + } + + virtual void choose_collection_set(ShenandoahCollectionSet* collection_set) { + ShenandoahHeap* heap = ShenandoahHeap::heap(); + ShenandoahHeapRegionSet* regions = ShenandoahHeap::heap()->regions(); + size_t active = regions->active_regions(); + // tty->print_cr("CSET regions:"); + for (size_t i = 0; i < active; i++) { + ShenandoahHeapRegion* r = regions->get(i); + assert(!r->is_root(), "must not be root region"); + assert(!collection_set->is_in(r), "must not yet be in cset"); + heap->set_next_top_at_mark_start(r->bottom(), r->end()); // No implicitely live stuff. + heap->set_complete_top_at_mark_start(r->bottom(), r->top()); // For debugging purposes + size_t garbage_percent = r->garbage() * 100 / ShenandoahHeapRegion::region_size_bytes(); + if ((r->is_regular() && r->used() > 0) && garbage_percent > ShenandoahGarbageThreshold) { + // r->print_on(tty); + collection_set->add_region(r); + } + r->clear_live_data(); + } + collection_set->update_region_status(); + } + + virtual bool should_start_traversal_gc() { + + ShenandoahHeap* heap = ShenandoahHeap::heap(); + + if (heap->need_update_refs()) return false; + + size_t capacity = heap->capacity(); + size_t used = heap->used(); + return 100 - (used * 100 / capacity) < ShenandoahFreeThreshold; + } + + virtual void choose_collection_set_from_regiondata(ShenandoahCollectionSet* set, + RegionData* data, size_t data_size, + size_t trash, size_t free) { + ShouldNotReachHere(); + } +}; + ShenandoahCollectorPolicy::ShenandoahCollectorPolicy() : _cycle_counter(0), @@ -1354,7 +1437,9 @@ } else if (strcmp(ShenandoahGCHeuristics, "LRU") == 0) { _heuristics = new ShenandoahAdaptiveHeuristics(); _minor_heuristics = new ShenandoahLRUPartialHeuristics(); - } else { + } else if (strcmp(ShenandoahGCHeuristics, "traversal") == 0) { + _heuristics = new ShenandoahTraversalHeuristics(); + } else { vm_exit_during_initialization("Unknown -XX:ShenandoahGCHeuristics option"); } @@ -1561,6 +1646,14 @@ } } +bool ShenandoahCollectorPolicy::should_start_traversal_gc() { + return _heuristics->should_start_traversal_gc(); +} + +bool ShenandoahCollectorPolicy::can_do_traversal_gc() { + return _heuristics->can_do_traversal_gc(); +} + void ShenandoahCollectorPolicy::record_cycle_start() { _cycle_counter++; _heuristics->record_cycle_start(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahCollectorPolicy.hpp b/src/hotspot/share/gc/shenandoah/shenandoahCollectorPolicy.hpp --- a/src/hotspot/share/gc/shenandoah/shenandoahCollectorPolicy.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahCollectorPolicy.hpp @@ -97,6 +97,8 @@ bool should_start_concurrent_mark(size_t used, size_t capacity); bool should_start_partial_gc(); bool can_do_partial_gc(); + bool should_start_traversal_gc(); + bool can_do_traversal_gc(); // Returns true when there should be a separate concurrent reference // updating phase after evacuation. diff --git a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.cpp b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.cpp --- a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.cpp @@ -330,6 +330,7 @@ break; case ShenandoahPhaseTimings::full_gc_roots: case ShenandoahPhaseTimings::final_partial_gc_work: + case ShenandoahPhaseTimings::final_traversal_gc_work: update_code_cache = true; break; default: diff --git a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentThread.cpp b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentThread.cpp --- a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentThread.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentThread.cpp @@ -30,6 +30,7 @@ #include "gc/shenandoah/shenandoahHeap.inline.hpp" #include "gc/shenandoah/shenandoahMonitoringSupport.hpp" #include "gc/shenandoah/shenandoahPartialGC.hpp" +#include "gc/shenandoah/shenandoahTraversalGC.hpp" #include "gc/shenandoah/shenandoahUtils.hpp" #include "gc/shenandoah/shenandoahWorkerPolicy.hpp" #include "gc/shenandoah/vm_operations_shenandoah.hpp" @@ -70,9 +71,10 @@ while (!in_graceful_shutdown() && !should_terminate()) { bool partial_gc_requested = heap->shenandoahPolicy()->should_start_partial_gc(); + bool traversal_gc_requested = heap->shenandoahPolicy()->should_start_traversal_gc(); bool conc_gc_requested = is_conc_gc_requested() || heap->shenandoahPolicy()->should_start_concurrent_mark(heap->used(), heap->capacity()); bool full_gc_requested = is_full_gc(); - bool gc_requested = partial_gc_requested || conc_gc_requested || full_gc_requested; + bool gc_requested = partial_gc_requested || traversal_gc_requested || conc_gc_requested || full_gc_requested; if (gc_requested) { // If GC was requested, we are sampling the counters even without actual triggers @@ -82,8 +84,10 @@ if (full_gc_requested) { service_fullgc_cycle(); - } else if (partial_gc_requested) { + } else if (partial_gc_requested/* || is_conc_gc_requested()*/) { service_partial_cycle(); + } else if (traversal_gc_requested) { + service_traversal_cycle(); } else if (conc_gc_requested) { service_normal_cycle(); } @@ -152,6 +156,58 @@ heap->entry_cleanup(); } +void ShenandoahConcurrentThread::service_traversal_cycle() { + GCIdMark gc_id_mark; + + ShenandoahGCSession session; + + ShenandoahHeap* heap = ShenandoahHeap::heap(); + ShenandoahTraversalGC* traversal_gc = heap->traversal_gc(); + TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters()); + + { + ShenandoahGCPhase total_phase(ShenandoahPhaseTimings::total_pause_gross); + ShenandoahGCPhase traversal_phase(ShenandoahPhaseTimings::init_traversal_gc_gross); + VM_ShenandoahInitTraversalGC init_traversal_gc; + VMThread::execute(&init_traversal_gc); + } + + if (check_cancellation()) return; + + { + GCTraceTime(Info, gc) time("Concurrent traversal", heap->gc_timer(), GCCause::_no_gc, true); + TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters()); + traversal_gc->concurrent_traversal_collection(); + } + + if (check_cancellation()) return; + + { + ShenandoahGCPhase total_phase(ShenandoahPhaseTimings::total_pause_gross); + ShenandoahGCPhase traversal_phase(ShenandoahPhaseTimings::final_traversal_gc_gross); + VM_ShenandoahFinalTraversalGC final_traversal_gc; + VMThread::execute(&final_traversal_gc); + } + + if (check_cancellation()) return; + + { + GCTraceTime(Info, gc) time("Concurrent cleanup", heap->gc_timer(), GCCause::_no_gc, true); + ShenandoahGCPhase phase(ShenandoahPhaseTimings::conc_cleanup); + ShenandoahGCPhase phase_recycle(ShenandoahPhaseTimings::conc_cleanup_recycle); + heap->recycle_trash(); + } + { + ShenandoahGCPhase phase_reset(ShenandoahPhaseTimings::conc_cleanup_reset_bitmaps); + WorkGang *workers = heap->workers(); + ShenandoahPushWorkerScope scope(workers, ConcGCThreads); + heap->reset_next_mark_bitmap(); + } + + // TODO: Call this properly with Shenandoah*CycleMark + heap->set_used_at_last_gc(); +} + void ShenandoahConcurrentThread::service_normal_cycle() { // Normal cycle goes via all concurrent phases. If allocation failure (af) happens during // any of the concurrent phases, it optionally degenerates to nearest STW operation, and diff --git a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentThread.hpp b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentThread.hpp --- a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentThread.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentThread.hpp @@ -69,6 +69,7 @@ void service_normal_cycle(); void service_fullgc_cycle(); void service_partial_cycle(); + void service_traversal_cycle(); public: // Constructor diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp --- a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp @@ -97,7 +97,9 @@ if (result != NULL) { // Allocation successful, bump live data stats: - r->increase_live_data_words(word_size); + if (!ShenandoahHeap::heap()->is_concurrent_traversal_in_progress()) { + r->increase_live_data_words(word_size); + } increase_used(word_size * HeapWordSize); ShenandoahHeap::heap()->increase_used(word_size * HeapWordSize); } else { @@ -195,7 +197,9 @@ used_words = ShenandoahHeapRegion::region_size_words(); } - r->increase_live_data_words(used_words); + if (!sh->is_concurrent_traversal_in_progress()) { + r->increase_live_data_words(used_words); + } r->set_top(r->bottom() + used_words); r->reset_alloc_stats_to_shared(); sh->increase_used(used_words * HeapWordSize); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -315,6 +315,10 @@ new ShenandoahPartialGC(this, _num_regions) : NULL; + _traversal_gc = _shenandoah_policy->can_do_traversal_gc() ? + new ShenandoahTraversalGC(this, _num_regions) : + NULL; + _monitoring_support = new ShenandoahMonitoringSupport(this); _phase_timings = new ShenandoahPhaseTimings(); @@ -489,6 +493,7 @@ if (is_evacuation_in_progress()) st->print("evacuating, "); if (is_update_refs_in_progress()) st->print("updating refs, "); if (is_concurrent_partial_in_progress()) st->print("partial, "); + if (is_concurrent_traversal_in_progress()) st->print("traversal, "); if (is_full_gc_in_progress()) st->print("full gc, "); if (is_full_gc_move_in_progress()) st->print("full gc move, "); @@ -1662,6 +1667,13 @@ set_evacuation_in_progress_at_safepoint(in_progress); } +void ShenandoahHeap::set_concurrent_traversal_in_progress(bool in_progress) { + _concurrent_traversal_in_progress.set_cond(in_progress); + JavaThread::satb_mark_queue_set().set_active_all_threads(in_progress, !in_progress); + set_evacuation_in_progress_at_safepoint(in_progress); + set_need_update_refs(in_progress); +} + void ShenandoahHeap::set_evacuation_in_progress_concurrently(bool in_progress) { // Note: it is important to first release the _evacuation_in_progress flag here, // so that Java threads can get out of oom_during_evacuation() and reach a safepoint, @@ -1718,9 +1730,12 @@ bool ShenandoahForwardedIsAliveClosure::do_object_b(oop obj) { assert(_heap != NULL, "sanity"); + assert(oopDesc::unsafe_equals(obj, ShenandoahBarrierSet::resolve_oop_static_not_null(obj)) || + (!_heap->is_marked_next(obj)), "from-space obj must not be marked"); + obj = ShenandoahBarrierSet::resolve_oop_static_not_null(obj); #ifdef ASSERT - if (_heap->is_concurrent_mark_in_progress()) { + if (_heap->is_concurrent_mark_in_progress() || _heap->is_concurrent_traversal_in_progress()) { assert(oopDesc::unsafe_equals(obj, ShenandoahBarrierSet::resolve_oop_static_not_null(obj)), "only query to-space"); } #endif @@ -1740,9 +1755,11 @@ } BoolObjectClosure* ShenandoahHeap::is_alive_closure() { - return need_update_refs() ? - (BoolObjectClosure*) &_forwarded_is_alive : - (BoolObjectClosure*) &_is_alive; + if (need_update_refs()) { + return (BoolObjectClosure*) &_forwarded_is_alive; + } else { + return (BoolObjectClosure*) &_is_alive; + } } void ShenandoahHeap::ref_processing_init() { @@ -1880,7 +1897,7 @@ ShenandoahGCPhase phase(phase_unload); purged_class = SystemDictionary::do_unloading(is_alive, full_gc ? this->full_gc()->gc_timer() : gc_timer(), - true); + false /* defer cleaning */); } { @@ -2100,6 +2117,10 @@ return _partial_gc; } +ShenandoahTraversalGC* ShenandoahHeap::traversal_gc() { + return _traversal_gc; +} + ShenandoahVerifier* ShenandoahHeap::verifier() { guarantee(ShenandoahVerify, "Should be enabled"); assert (_verifier != NULL, "sanity"); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp @@ -43,6 +43,7 @@ class ShenandoahConcurrentMark; class ShenandoahMarkCompact; class ShenandoahPartialGC; +class ShenandoahTraversalGC; class ShenandoahVerifier; class ShenandoahConcurrentThread; class ShenandoahMonitoringSupport; @@ -152,6 +153,7 @@ ShenandoahConcurrentMark* _scm; ShenandoahMarkCompact* _full_gc; ShenandoahPartialGC* _partial_gc; + ShenandoahTraversalGC* _traversal_gc; ShenandoahVerifier* _verifier; ShenandoahConcurrentThread* _concurrent_gc_thread; @@ -193,6 +195,7 @@ ShenandoahSharedFlag _evacuation_in_progress; ShenandoahSharedFlag _update_refs_in_progress; ShenandoahSharedFlag _concurrent_partial_in_progress; + ShenandoahSharedFlag _concurrent_traversal_in_progress; ShenandoahSharedFlag _full_gc_in_progress; ShenandoahSharedFlag _full_gc_move_in_progress; @@ -325,6 +328,7 @@ void set_full_gc_in_progress(bool in_progress); void set_full_gc_move_in_progress(bool in_progress); void set_concurrent_partial_in_progress(bool in_progress); + void set_concurrent_traversal_in_progress(bool in_progress); void set_need_update_refs(bool update_refs); inline bool is_concurrent_mark_in_progress() const; @@ -333,6 +337,7 @@ inline bool is_full_gc_in_progress() const; inline bool is_full_gc_move_in_progress() const; inline bool is_concurrent_partial_in_progress() const; + inline bool is_concurrent_traversal_in_progress() const; inline bool need_update_refs() const; inline bool region_in_collection_set(size_t region_index) const; @@ -415,6 +420,7 @@ ShenandoahConcurrentMark* concurrentMark() { return _scm; } ShenandoahMarkCompact* full_gc() { return _full_gc; } ShenandoahPartialGC* partial_gc(); + ShenandoahTraversalGC* traversal_gc(); ShenandoahVerifier* verifier(); ReferenceProcessor* ref_processor() { return _ref_processor;} diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp @@ -414,6 +414,10 @@ return _concurrent_partial_in_progress.is_set(); } +inline bool ShenandoahHeap::is_concurrent_traversal_in_progress() const { + return _concurrent_traversal_in_progress.is_set(); +} + inline bool ShenandoahHeap::is_evacuation_in_progress() const { return _evacuation_in_progress.is_set(); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahMarkCompact.cpp b/src/hotspot/share/gc/shenandoah/shenandoahMarkCompact.cpp --- a/src/hotspot/share/gc/shenandoah/shenandoahMarkCompact.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahMarkCompact.cpp @@ -37,6 +37,7 @@ #include "gc/shenandoah/shenandoahHeap.inline.hpp" #include "gc/shenandoah/shenandoahPartialGC.hpp" #include "gc/shenandoah/shenandoahRootProcessor.hpp" +#include "gc/shenandoah/shenandoahTraversalGC.hpp" #include "gc/shenandoah/shenandoahUtils.hpp" #include "gc/shenandoah/shenandoahVerifier.hpp" #include "gc/shenandoah/shenandoahWorkerPolicy.hpp" @@ -151,6 +152,12 @@ heap->set_concurrent_partial_in_progress(false); } + // b3. Cancel concurrent traversal GC, if in progress + if (heap->is_concurrent_traversal_in_progress()) { + heap->traversal_gc()->reset(); + heap->set_concurrent_traversal_in_progress(false); + } + // c. Reset the bitmaps for new marking heap->reset_next_mark_bitmap(); assert(heap->is_next_bitmap_clear(), "sanity"); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahOopClosures.hpp b/src/hotspot/share/gc/shenandoah/shenandoahOopClosures.hpp --- a/src/hotspot/share/gc/shenandoah/shenandoahOopClosures.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahOopClosures.hpp @@ -225,7 +225,55 @@ void do_oop(oop* p) { do_oop_nv(p); } void do_oop(narrowOop* p) { do_oop_nv(p); } + + virtual ReferenceIterationMode reference_iteration_mode() { return DO_FIELDS; } + }; +class ShenandoahTraversalSuperClosure : public MetadataAwareOopClosure { +private: + ShenandoahTraversalGC* _traversal_gc; + Thread* _thread; + ShenandoahObjToScanQueue* _queue; + +public: + ShenandoahTraversalSuperClosure(ShenandoahObjToScanQueue* q, ReferenceProcessor* rp) : + MetadataAwareOopClosure(rp), + _traversal_gc(ShenandoahHeap::heap()->traversal_gc()), + _thread(Thread::current()), _queue(q) {} + + template + void work(T* p); +}; + +class ShenandoahTraversalClosure : public ShenandoahTraversalSuperClosure { +public: + ShenandoahTraversalClosure(ShenandoahObjToScanQueue* q, ReferenceProcessor* rp) : + ShenandoahTraversalSuperClosure(q, rp) {} + + template + inline void do_oop_nv(T* p) { work(p); } + + virtual void do_oop(narrowOop* p) { do_oop_nv(p); } + virtual void do_oop(oop* p) { do_oop_nv(p); } + + inline bool do_metadata_nv() { return false; } + virtual bool do_metadata() { return false; } +}; + +class ShenandoahTraversalMetadataClosure : public ShenandoahTraversalSuperClosure { +public: + ShenandoahTraversalMetadataClosure(ShenandoahObjToScanQueue* q, ReferenceProcessor* rp) : + ShenandoahTraversalSuperClosure(q, rp) {} + + template + inline void do_oop_nv(T* p) { work(p); } + + virtual void do_oop(narrowOop* p) { do_oop_nv(p); } + virtual void do_oop(oop* p) { do_oop_nv(p); } + + inline bool do_metadata_nv() { return true; } + virtual bool do_metadata() { return true; } +}; #endif // SHARE_VM_GC_SHENANDOAH_SHENANDOAHOOPCLOSURES_HPP diff --git a/src/hotspot/share/gc/shenandoah/shenandoahOopClosures.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahOopClosures.inline.hpp --- a/src/hotspot/share/gc/shenandoah/shenandoahOopClosures.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahOopClosures.inline.hpp @@ -27,6 +27,7 @@ #include "gc/shenandoah/shenandoahHeap.inline.hpp" #include "gc/shenandoah/shenandoahConcurrentMark.inline.hpp" #include "gc/shenandoah/shenandoahPartialGC.inline.hpp" +#include "gc/shenandoah/shenandoahTraversalGC.inline.hpp" template inline void ShenandoahMarkRefsSuperClosure::work(T *p) { @@ -48,4 +49,9 @@ _partial_gc->process_oop(p, _thread, _queue); } +template +inline void ShenandoahTraversalSuperClosure::work(T* p) { + _traversal_gc->process_oop(p, _thread, _queue); +} + #endif // SHARE_VM_GC_SHENANDOAH_SHENANDOAHOOPCLOSURES_INLINE_HPP diff --git a/src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.cpp b/src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.cpp --- a/src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.cpp @@ -73,6 +73,8 @@ phase == update_roots || phase == init_partial_gc_work || phase == final_partial_gc_work || + phase == init_traversal_gc_work || + phase == final_traversal_gc_work || phase == final_update_refs_roots || phase == full_gc_roots || phase == _num_phases, @@ -270,6 +272,42 @@ _phase_names[partial_gc_cleanup] = " Cleanup"; + _phase_names[init_traversal_gc_gross] = "Pause Init Traversal (G)"; + _phase_names[init_traversal_gc] = "Pause Init Traversal (N)"; + _phase_names[traversal_gc_prepare] = " Prepare"; + _phase_names[init_traversal_gc_work] = " Work"; + _phase_names[init_traversal_gc_thread_roots] = " TI: Thread Roots"; + _phase_names[init_traversal_gc_code_roots] = " TI: Code Cache Roots"; + _phase_names[init_traversal_gc_string_table_roots] = " TI: String Table Roots"; + _phase_names[init_traversal_gc_universe_roots] = " TI: Universe Roots"; + _phase_names[init_traversal_gc_jni_roots] = " TI: JNI Roots"; + _phase_names[init_traversal_gc_jni_weak_roots] = " TI: JNI Weak Roots"; + _phase_names[init_traversal_gc_synchronizer_roots] = " TI: Synchronizer Roots"; + _phase_names[init_traversal_gc_flat_profiler_roots] = " TI: Flat Profiler Roots"; + _phase_names[init_traversal_gc_management_roots] = " TI: Management Roots"; + _phase_names[init_traversal_gc_system_dict_roots] = " TI: System Dict Roots"; + _phase_names[init_traversal_gc_cldg_roots] = " TI: CLDG Roots"; + _phase_names[init_traversal_gc_jvmti_roots] = " TI: JVMTI Roots"; + _phase_names[init_traversal_gc_string_dedup_roots] = " TI: String Dedup Roots"; + _phase_names[final_traversal_gc_gross] = "Pause Final Traversal (G)"; + _phase_names[final_traversal_gc] = "Pause Final Traversal (N)"; + _phase_names[final_traversal_gc_work] = " Work"; + _phase_names[final_traversal_gc_thread_roots] = " TF: Thread Roots"; + _phase_names[final_traversal_gc_code_roots] = " TF: Code Cache Roots"; + _phase_names[final_traversal_gc_string_table_roots] = " TF: String Table Roots"; + _phase_names[final_traversal_gc_universe_roots] = " TF: Universe Roots"; + _phase_names[final_traversal_gc_jni_roots] = " TF: JNI Roots"; + _phase_names[final_traversal_gc_jni_weak_roots] = " TF: JNI Weak Roots"; + _phase_names[final_traversal_gc_synchronizer_roots] = " TF: Synchronizer Roots"; + _phase_names[final_traversal_gc_flat_profiler_roots] = " TF: Flat Profiler Roots"; + _phase_names[final_traversal_gc_management_roots] = " TF: Management Roots"; + _phase_names[final_traversal_gc_system_dict_roots] = " TF: System Dict Roots"; + _phase_names[final_traversal_gc_cldg_roots] = " TF: CLDG Roots"; + _phase_names[final_traversal_gc_jvmti_roots] = " TF: JVMTI Roots"; + _phase_names[final_traversal_gc_string_dedup_roots] = " TF: String Dedup Roots"; + + _phase_names[traversal_gc_cleanup] = " Cleanup"; + _phase_names[pause_other] = "Pause Other"; _phase_names[conc_mark] = "Concurrent Marking"; @@ -280,6 +318,7 @@ _phase_names[conc_cleanup_reset_bitmaps] = " Reset Bitmaps"; _phase_names[conc_other] = "Concurrent Other"; _phase_names[conc_partial] = "Concurrent Partial"; + _phase_names[conc_traversal] = "Concurrent Traversal"; _phase_names[init_update_refs_gross] = "Pause Init Update Refs (G)"; _phase_names[init_update_refs] = "Pause Init Update Refs (N)"; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.hpp b/src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.hpp --- a/src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.hpp @@ -180,6 +180,47 @@ partial_gc_cleanup, + init_traversal_gc_gross, + init_traversal_gc, + traversal_gc_prepare, + + // Per-thread timer block, should have "roots" counters in consistent order + init_traversal_gc_work, + init_traversal_gc_thread_roots, + init_traversal_gc_code_roots, + init_traversal_gc_string_table_roots, + init_traversal_gc_universe_roots, + init_traversal_gc_jni_roots, + init_traversal_gc_jni_weak_roots, + init_traversal_gc_synchronizer_roots, + init_traversal_gc_flat_profiler_roots, + init_traversal_gc_management_roots, + init_traversal_gc_system_dict_roots, + init_traversal_gc_cldg_roots, + init_traversal_gc_jvmti_roots, + init_traversal_gc_string_dedup_roots, + + final_traversal_gc_gross, + final_traversal_gc, + + // Per-thread timer block, should have "roots" counters in consistent order + final_traversal_gc_work, + final_traversal_gc_thread_roots, + final_traversal_gc_code_roots, + final_traversal_gc_string_table_roots, + final_traversal_gc_universe_roots, + final_traversal_gc_jni_roots, + final_traversal_gc_jni_weak_roots, + final_traversal_gc_synchronizer_roots, + final_traversal_gc_flat_profiler_roots, + final_traversal_gc_management_roots, + final_traversal_gc_system_dict_roots, + final_traversal_gc_cldg_roots, + final_traversal_gc_jvmti_roots, + final_traversal_gc_string_dedup_roots, + + traversal_gc_cleanup, + full_gc_gross, full_gc, full_gc_heapdumps, @@ -235,6 +276,7 @@ conc_cleanup_recycle, conc_cleanup_reset_bitmaps, conc_partial, + conc_traversal, // Unclassified pause_other, diff --git a/src/hotspot/share/gc/shenandoah/shenandoahTraversalGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahTraversalGC.cpp new file mode 100644 --- /dev/null +++ b/src/hotspot/share/gc/shenandoah/shenandoahTraversalGC.cpp @@ -0,0 +1,886 @@ +/* + * Copyright (c) 2017, Red Hat, Inc. and/or its affiliates. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" + +#include "gc/shared/gcTraceTime.inline.hpp" +#include "gc/shared/markBitMap.inline.hpp" +#include "gc/shared/workgroup.hpp" +#include "gc/shared/taskqueue.inline.hpp" +#include "gc/shenandoah/shenandoahBarrierSet.hpp" +#include "gc/shenandoah/shenandoahCollectionSet.hpp" +#include "gc/shenandoah/shenandoahCollectorPolicy.hpp" +#include "gc/shenandoah/shenandoahFreeSet.hpp" +#include "gc/shenandoah/shenandoahPhaseTimings.hpp" +#include "gc/shenandoah/shenandoahHeapRegionSet.hpp" +#include "gc/shenandoah/shenandoahHeap.hpp" +#include "gc/shenandoah/shenandoahHeap.inline.hpp" +#include "gc/shenandoah/shenandoahOopClosures.inline.hpp" +#include "gc/shenandoah/shenandoahTraversalGC.hpp" +#include "gc/shenandoah/shenandoahRootProcessor.hpp" +#include "gc/shenandoah/shenandoahTaskqueue.hpp" +#include "gc/shenandoah/shenandoahUtils.hpp" +#include "gc/shenandoah/shenandoahVerifier.hpp" +#include "gc/shenandoah/shenandoahWorkGroup.hpp" +#include "gc/shenandoah/shenandoahWorkerPolicy.hpp" + +#include "memory/iterator.hpp" + +class ShenandoahTraversalSATBBufferClosure : public SATBBufferClosure { +private: + ShenandoahObjToScanQueue* _queue; + ShenandoahTraversalGC* _traversal_gc; + Thread* _thread; + ShenandoahHeap* _heap; + MarkBitMap* _bitmap; +public: + ShenandoahTraversalSATBBufferClosure(ShenandoahObjToScanQueue* q) : + _queue(q), _traversal_gc(ShenandoahHeap::heap()->traversal_gc()), _thread(Thread::current()), + _heap(ShenandoahHeap::heap()), _bitmap(ShenandoahHeap::heap()->next_mark_bit_map()) + { } + + void do_buffer(void** buffer, size_t size) { + for (size_t i = 0; i < size; ++i) { + oop* p = (oop*) &buffer[i]; + oop obj = oopDesc::load_heap_oop(p); + assert(!oopDesc::is_null(obj), "no NULL refs in oop queue"); + assert(oopDesc::unsafe_equals(obj, ShenandoahBarrierSet::resolve_oop_static_not_null(obj)), "only to-space objs"); + if ((!_bitmap->isMarked((HeapWord*) obj)) && _bitmap->parMark((HeapWord*) obj)) { + _queue->push(ShenandoahMarkTask(obj)); + } + } + } +}; + +class ShenandoahTraversalSATBThreadsClosure : public ThreadClosure { + ShenandoahTraversalSATBBufferClosure* _satb_cl; + int _thread_parity; + + public: + ShenandoahTraversalSATBThreadsClosure(ShenandoahTraversalSATBBufferClosure* satb_cl) : + _satb_cl(satb_cl), + _thread_parity(Threads::thread_claim_parity()) {} + + void do_thread(Thread* thread) { + if (thread->is_Java_thread()) { + if (thread->claim_oops_do(true, _thread_parity)) { + JavaThread* jt = (JavaThread*)thread; + jt->satb_mark_queue().apply_closure_and_empty(_satb_cl); + } + } else if (thread->is_VM_thread()) { + if (thread->claim_oops_do(true, _thread_parity)) { + JavaThread::satb_mark_queue_set().shared_satb_queue()->apply_closure_and_empty(_satb_cl); + } + } + } +}; + +class ShenandoahInitTraversalCollectionTask : public AbstractGangTask { +private: + ShenandoahRootProcessor* _rp; + ShenandoahHeap* _heap; +public: + ShenandoahInitTraversalCollectionTask(ShenandoahRootProcessor* rp) : + AbstractGangTask("Shenandoah Init Traversal Collection"), + _rp(rp), + _heap(ShenandoahHeap::heap()) {} + + void work(uint worker_id) { + ShenandoahObjToScanQueueSet* queues = _heap->traversal_gc()->task_queues(); + ShenandoahObjToScanQueue* q = queues->queue(worker_id); + + // Initialize live data. + jushort* ld = _heap->traversal_gc()->get_liveness(worker_id); + Copy::fill_to_bytes(ld, _heap->num_regions() * sizeof(jushort)); + + bool process_refs = _heap->shenandoahPolicy()->process_references(); + bool unload_classes = _heap->shenandoahPolicy()->unload_classes(); + ReferenceProcessor* rp = NULL; + if (process_refs) { + rp = _heap->ref_processor(); + } + + // Step 1: Process ordinary GC roots. + { + ShenandoahTraversalClosure roots_cl(q, rp); + CLDToOopClosure cld_cl(&roots_cl); + MarkingCodeBlobClosure code_cl(&roots_cl, CodeBlobToOopClosure::FixRelocations); + if (unload_classes) { + _rp->process_strong_roots(&roots_cl, process_refs ? NULL : &roots_cl, &cld_cl, &code_cl, worker_id); + } else { + _rp->process_all_roots(&roots_cl, process_refs ? NULL : &roots_cl, &cld_cl, &code_cl, worker_id); + } + } + } +}; + +class ShenandoahConcurrentTraversalCollectionTask : public AbstractGangTask { +private: + ParallelTaskTerminator* _terminator; + ShenandoahHeap* _heap; +public: + ShenandoahConcurrentTraversalCollectionTask(ParallelTaskTerminator* terminator) : + AbstractGangTask("Shenandoah Concurrent Traversal Collection"), + _terminator(terminator), + _heap(ShenandoahHeap::heap()) {} + + void work(uint worker_id) { + ShenandoahTraversalGC* traversal_gc = _heap->traversal_gc(); + ShenandoahObjToScanQueueSet* queues = traversal_gc->task_queues(); + ShenandoahObjToScanQueue* q = queues->queue(worker_id); + + // Drain all outstanding work in queues. + traversal_gc->main_loop(worker_id, _terminator, true); + } +}; + +class ShenandoahFinalTraversalCollectionTask : public AbstractGangTask { +private: + ShenandoahRootProcessor* _rp; + ParallelTaskTerminator* _terminator; + ShenandoahHeap* _heap; +public: + ShenandoahFinalTraversalCollectionTask(ShenandoahRootProcessor* rp, ParallelTaskTerminator* terminator) : + AbstractGangTask("Shenandoah Final Traversal Collection"), + _rp(rp), + _terminator(terminator), + _heap(ShenandoahHeap::heap()) {} + + void work(uint worker_id) { + ShenandoahTraversalGC* traversal_gc = _heap->traversal_gc(); + + ShenandoahObjToScanQueueSet* queues = traversal_gc->task_queues(); + ShenandoahObjToScanQueue* q = queues->queue(worker_id); + + bool process_refs = _heap->shenandoahPolicy()->process_references(); + bool unload_classes = _heap->shenandoahPolicy()->unload_classes(); + ReferenceProcessor* rp = NULL; + if (process_refs) { + rp = _heap->ref_processor(); + } + + // Step 1: Process ordinary GC roots. + { + ShenandoahTraversalClosure roots_cl(q, rp); + CLDToOopClosure cld_cl(&roots_cl); + MarkingCodeBlobClosure code_cl(&roots_cl, CodeBlobToOopClosure::FixRelocations); + if (unload_classes) { + _rp->process_strong_roots(&roots_cl, process_refs ? NULL : &roots_cl, &cld_cl, &code_cl, worker_id); + } else { + _rp->process_all_roots(&roots_cl, process_refs ? NULL : &roots_cl, &cld_cl, &code_cl, worker_id); + } + } + + // Step 2: Drain outstanding SATB queues. + { + ShenandoahTraversalSATBBufferClosure satb_cl(q); + // Process remaining finished SATB buffers. + SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set(); + while (satb_mq_set.apply_closure_to_completed_buffer(&satb_cl)); + // Process remaining threads SATB buffers. + ShenandoahTraversalSATBThreadsClosure tc(&satb_cl); + Threads::threads_do(&tc); + } + + // Step 3: Finally drain all outstanding work in queues. + traversal_gc->main_loop(worker_id, _terminator, false); + + // Flush remaining liveness data. + traversal_gc->flush_liveness(worker_id); + + } +}; + +void ShenandoahTraversalGC::flush_liveness(uint worker_id) { + jushort* ld = get_liveness(worker_id); + for (uint i = 0; i < _heap->regions()->active_regions(); i++) { + ShenandoahHeapRegion* r = _heap->regions()->get(i); + jushort live = ld[i]; + if (live > 0) { + r->increase_live_data_words(live); + ld[i] = 0; + } + } +} + +// TODO: It should be possible to hook this up on the regular thread scanning +// that we do in final pass anyway. +class ShenandoahFinalDrainOopQueueTask : public AbstractGangTask { +public: + ShenandoahFinalDrainOopQueueTask() : + AbstractGangTask("Shenandoah Final Drain Oop Queue") { + Threads::change_thread_claim_parity(); + } + + ~ShenandoahFinalDrainOopQueueTask() { + Threads::assert_all_threads_claimed(); + } + + void work(uint worker_id) { + ShenandoahTraversalGC* traversal_gc = ShenandoahHeap::heap()->traversal_gc(); + ShenandoahObjToScanQueueSet* queues = traversal_gc->task_queues(); + ShenandoahObjToScanQueue* q = queues->queue(worker_id); + + // Step 2: Drain outstanding SATB queues. + { + ShenandoahTraversalSATBBufferClosure satb_cl(q); + // Process remaining finished SATB buffers. + SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set(); + while (satb_mq_set.apply_closure_to_completed_buffer(&satb_cl)); + // Process remaining threads SATB buffers. + ShenandoahTraversalSATBThreadsClosure tc(&satb_cl); + Threads::threads_do(&tc); + } + } +}; + +ShenandoahTraversalGC::ShenandoahTraversalGC(ShenandoahHeap* heap, size_t num_regions) : + _heap(heap), + _task_queues(new ShenandoahObjToScanQueueSet(heap->max_workers())) { + + uint num_queues = heap->max_workers(); + for (uint i = 0; i < num_queues; ++i) { + ShenandoahObjToScanQueue* task_queue = new ShenandoahObjToScanQueue(); + task_queue->initialize(); + _task_queues->register_queue(i, task_queue); + } + + uint workers = heap->max_workers(); + _liveness_local = NEW_C_HEAP_ARRAY(jushort*, workers, mtGC); + for (uint worker = 0; worker < workers; worker++) { + _liveness_local[worker] = NEW_C_HEAP_ARRAY(jushort, num_regions, mtGC); + } + +} + +ShenandoahTraversalGC::~ShenandoahTraversalGC() { +} + +void ShenandoahTraversalGC::prepare() { + _heap->collection_set()->clear(); + assert(_heap->collection_set()->count() == 0, "collection set not clear"); + + _heap->make_tlabs_parsable(true); + + assert(_heap->is_next_bitmap_clear(), "need clean mark bitmap"); + _bitmap = _heap->next_mark_bit_map(); + + ShenandoahHeapRegionSet* regions = _heap->regions(); + ShenandoahCollectionSet* collection_set = _heap->collection_set(); + size_t num_regions = _heap->num_regions(); + + // Find collection set + _heap->shenandoahPolicy()->choose_collection_set(collection_set, false); + + // Rebuild free set + ShenandoahFreeSet* _free_regions = _heap->free_regions(); + _free_regions->clear(); + + for (uint from_idx = 0; from_idx < num_regions; from_idx++) { + ShenandoahHeapRegion* r = regions->get(from_idx); + if (r->is_alloc_allowed()) { + _free_regions->add_region(r); + } + } + + log_info(gc,ergo)("Got "SIZE_FORMAT" collection set regions", collection_set->count()); +} + +void ShenandoahTraversalGC::init_traversal_collection() { + assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "STW traversal GC"); + ShenandoahWorkerScope traversal_gc_scope(_heap->workers(), ShenandoahWorkerPolicy::calc_workers_for_stw_traversal()); + + _heap->set_alloc_seq_gc_start(); + + if (ShenandoahVerify) { + _heap->verifier()->verify_before_traversal(); + } + + { + ShenandoahGCPhase phase_prepare(ShenandoahPhaseTimings::traversal_gc_prepare); + ShenandoahHeapLocker lock(_heap->lock()); + prepare(); + } + + _heap->set_concurrent_traversal_in_progress(true); + + bool process_refs = _heap->shenandoahPolicy()->process_references(); + if (process_refs) { + ReferenceProcessor* rp = _heap->ref_processor(); + rp->enable_discovery(true /*verify_no_refs*/); + rp->setup_policy(false); + } + + { + ShenandoahGCPhase phase_work(ShenandoahPhaseTimings::init_traversal_gc_work); + assert(_task_queues->is_empty(), "queues must be empty before traversal GC"); + +#if defined(COMPILER2) || INCLUDE_JVMCI + DerivedPointerTable::clear(); +#endif + + { + uint nworkers = _heap->workers()->active_workers(); + task_queues()->reserve(nworkers); + ShenandoahRootProcessor rp(_heap, nworkers, ShenandoahPhaseTimings::init_traversal_gc_work); + + if (UseShenandoahOWST) { + ShenandoahTaskTerminator terminator(nworkers, task_queues()); + ShenandoahInitTraversalCollectionTask traversal_task(&rp); + _heap->workers()->run_task(&traversal_task); + } else { + ParallelTaskTerminator terminator(nworkers, task_queues()); + ShenandoahInitTraversalCollectionTask traversal_task(&rp); + _heap->workers()->run_task(&traversal_task); + } + } + +#if defined(COMPILER2) || INCLUDE_JVMCI + DerivedPointerTable::update_pointers(); +#endif + if (_heap->cancelled_concgc()) { + _heap->fixup_roots(); + reset(); + _heap->set_concurrent_traversal_in_progress(false); + } + } +} + +void ShenandoahTraversalGC::main_loop(uint worker_id, ParallelTaskTerminator* terminator, bool do_satb) { + if (do_satb) { + main_loop_prework(worker_id, terminator); + } else { + main_loop_prework(worker_id, terminator); + } +} + +template +void ShenandoahTraversalGC::main_loop_prework(uint w, ParallelTaskTerminator* t) { + ShenandoahObjToScanQueue* q = task_queues()->queue(w); + jushort* ld = get_liveness(w); + + ReferenceProcessor* rp = NULL; + if (_heap->shenandoahPolicy()->process_references()) { + rp = _heap->ref_processor(); + } + if (_heap->shenandoahPolicy()->unload_classes()) { + ShenandoahTraversalMetadataClosure cl(q, rp); + main_loop_work(&cl, ld, w, t); + } else { + ShenandoahTraversalClosure cl(q, rp); + main_loop_work(&cl, ld, w, t); + } +} + +template +void ShenandoahTraversalGC::main_loop_work(T* cl, jushort* live_data, uint worker_id, ParallelTaskTerminator* terminator) { + ShenandoahObjToScanQueueSet* queues = task_queues(); + ShenandoahObjToScanQueue* q = queues->queue(worker_id); + + uintx stride = ShenandoahMarkLoopStride; + + ShenandoahMarkTask task; + + // Process outstanding queues, if any. + q = queues->claim_next(); + while (q != NULL) { + if (_heap->check_cancelled_concgc_and_yield()) { + ShenandoahCancelledTerminatorTerminator tt; + while (!terminator->offer_termination(&tt)); + return; + } + + for (uint i = 0; i < stride; i++) { + if (q->pop_buffer(task) || + q->pop_local(task) || + q->pop_overflow(task)) { + do_task(q, cl, live_data, &task); + } else { + assert(q->is_empty(), "Must be empty"); + q = queues->claim_next(); + break; + } + } + } + + // Normal loop. + q = queues->queue(worker_id); + ShenandoahTraversalSATBBufferClosure satb_cl(q); + SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set(); + + int seed = 17; + + while (true) { + if (check_and_handle_cancelled_gc(terminator)) return; + + for (uint i = 0; i < stride; i++) { + if ((q->pop_buffer(task) || + q->pop_local(task) || + q->pop_overflow(task) || + (DO_SATB && satb_mq_set.apply_closure_to_completed_buffer(&satb_cl) && q->pop_buffer(task)) || + queues->steal(worker_id, &seed, task))) { + do_task(q, cl, live_data, &task); + } else { + if (terminator->offer_termination()) return; + } + } + } +} + +bool ShenandoahTraversalGC::check_and_handle_cancelled_gc(ParallelTaskTerminator* terminator) { + if (_heap->cancelled_concgc()) { + ShenandoahCancelledTerminatorTerminator tt; + while (! terminator->offer_termination(&tt)); + return true; + } + return false; +} + +void ShenandoahTraversalGC::concurrent_traversal_collection() { + ClassLoaderDataGraph::clear_claimed_marks(); + + ShenandoahWorkerScope traversal_gc_scope(_heap->workers(), ShenandoahWorkerPolicy::calc_workers_for_conc_traversal()); + + ShenandoahGCPhase phase_work(ShenandoahPhaseTimings::conc_traversal); + if (!_heap->cancelled_concgc()) { + uint nworkers = _heap->workers()->active_workers(); + task_queues()->reserve(nworkers); + if (UseShenandoahOWST) { + ShenandoahTaskTerminator terminator(nworkers, task_queues()); + ShenandoahConcurrentTraversalCollectionTask traversal_task(&terminator); + _heap->workers()->run_task(&traversal_task); + } else { + ParallelTaskTerminator terminator(nworkers, task_queues()); + ShenandoahConcurrentTraversalCollectionTask traversal_task(&terminator); + _heap->workers()->run_task(&traversal_task); + } + } + + if (!_heap->cancelled_concgc() && ShenandoahPreclean && _heap->shenandoahPolicy()->process_references()) { + preclean_weak_refs(); + } + + if (_heap->cancelled_concgc()) { + _task_queues->clear(); + } + assert(_task_queues->is_empty(), "queues must be empty after traversal GC"); +} + +class ShenandoahRemarkCLDClosure : public CLDClosure { +private: + OopClosure* _cl; +public: + ShenandoahRemarkCLDClosure(OopClosure* cl) : _cl(cl) {} + void do_cld(ClassLoaderData* cld) { + if (cld->has_modified_oops()) { + cld->oops_do(_cl, false, true); + } + } +}; + +void ShenandoahTraversalGC::final_traversal_collection() { + + ShenandoahWorkerScope traversal_gc_scope(_heap->workers(), ShenandoahWorkerPolicy::calc_workers_for_stw_traversal()); + + _heap->make_tlabs_parsable(true); + + if (!_heap->cancelled_concgc()) { +#if defined(COMPILER2) || INCLUDE_JVMCI + DerivedPointerTable::clear(); +#endif + ShenandoahGCPhase phase_work(ShenandoahPhaseTimings::final_traversal_gc_work); + uint nworkers = _heap->workers()->active_workers(); + task_queues()->reserve(nworkers); + + // Drain OOP queues. + ShenandoahFinalDrainOopQueueTask drain_oop_queue_task; + _heap->workers()->run_task(&drain_oop_queue_task); + + { + bool process_refs = _heap->shenandoahPolicy()->process_references(); + bool unload_classes = _heap->shenandoahPolicy()->unload_classes(); + ReferenceProcessor* rp = NULL; + if (process_refs) { + rp = _heap->ref_processor(); + } + + ShenandoahTraversalClosure roots_cl(task_queues()->queue(0), rp); + ShenandoahRemarkCLDClosure remark_clds(&roots_cl); + ClassLoaderDataGraph::cld_do(&remark_clds); + } + + // Finish traversal + ShenandoahRootProcessor rp(_heap, nworkers, ShenandoahPhaseTimings::final_traversal_gc_work); + if (UseShenandoahOWST) { + ShenandoahTaskTerminator terminator(nworkers, task_queues()); + ShenandoahFinalTraversalCollectionTask traversal_task(&rp, &terminator); + _heap->workers()->run_task(&traversal_task); + } else { + ParallelTaskTerminator terminator(nworkers, task_queues()); + ShenandoahFinalTraversalCollectionTask traversal_task(&rp, &terminator); + _heap->workers()->run_task(&traversal_task); + } +#if defined(COMPILER2) || INCLUDE_JVMCI + DerivedPointerTable::update_pointers(); +#endif + } + + if (_heap->shenandoahPolicy()->process_references()) { + weak_refs_work(); + } + + if (_heap->shenandoahPolicy()->unload_classes()) { + _heap->unload_classes_and_cleanup_tables(false); + _heap->concurrentMark()->update_roots(ShenandoahPhaseTimings::final_traversal_gc_work); + } + + if (!_heap->cancelled_concgc()) { + // Still good? We can now trash the cset, and make final verification + { + ShenandoahGCPhase phase_cleanup(ShenandoahPhaseTimings::traversal_gc_cleanup); + ShenandoahHeapLocker lock(_heap->lock()); + + // Trash everything + // Clear immediate garbage regions. + ShenandoahHeapRegionSet* regions = _heap->regions(); + size_t active = regions->active_regions(); + ShenandoahFreeSet* free_regions = _heap->free_regions(); + free_regions->clear(); + for (size_t i = 0; i < active; i++) { + ShenandoahHeapRegion* r = regions->get(i); + if (r->is_humongous_start() && !r->has_live()) { + HeapWord* humongous_obj = r->bottom() + BrooksPointer::word_size(); + assert(!_bitmap->isMarked(humongous_obj), "must not be marked"); + r->make_trash(); + while (regions->get(i+1)->is_humongous_continuation()) { + i++; + r = regions->get(i); + assert(r->is_humongous_continuation(), "must be humongous continuation"); + r->make_trash(); + } + } else if (!r->is_empty() && !r->has_live()) { + if (r->is_humongous()) { + r->print_on(tty); + } + assert(!r->is_humongous(), "handled above"); + assert(!r->is_trash(), "must not already be trashed"); + r->make_trash(); + } else if (r->is_alloc_allowed()) { + free_regions->add_region(r); + } + } + _heap->collection_set()->clear(); + reset(); + } + + if (ShenandoahVerify) { + _heap->verifier()->verify_after_traversal(); + } + } else { + // On cancellation path, fixup roots to make them consistent + _heap->fixup_roots(); + reset(); + } + + assert(_task_queues->is_empty(), "queues must be empty after traversal GC"); + _heap->set_concurrent_traversal_in_progress(false); +} + +void ShenandoahTraversalGC::reset() { + _task_queues->clear(); +} + +ShenandoahObjToScanQueueSet* ShenandoahTraversalGC::task_queues() { + return _task_queues; +} + +jushort* ShenandoahTraversalGC::get_liveness(uint worker_id) { + return _liveness_local[worker_id]; +} + +class ShenandoahTraversalCancelledGCYieldClosure : public YieldClosure { +private: + ShenandoahHeap* const _heap; +public: + ShenandoahTraversalCancelledGCYieldClosure() : _heap(ShenandoahHeap::heap()) {}; + virtual bool should_return() { return _heap->cancelled_concgc(); } +}; + +class ShenandoahTraversalPrecleanCompleteGCClosure : public VoidClosure { +public: + void do_void() { + ShenandoahHeap* sh = ShenandoahHeap::heap(); + ShenandoahTraversalGC* traversal_gc = sh->traversal_gc(); + assert(sh->shenandoahPolicy()->process_references(), "why else would we be here?"); + ReferenceProcessor* rp = sh->ref_processor(); + ParallelTaskTerminator terminator(1, traversal_gc->task_queues()); + ReferenceProcessorIsAliveMutator fix_alive(rp, sh->is_alive_closure()); + traversal_gc->main_loop((uint) 0, &terminator, false); + } +}; + +class ShenandoahTraversalKeepAliveUpdateClosure : public OopClosure { +private: + ShenandoahObjToScanQueue* _queue; + Thread* _thread; + ShenandoahTraversalGC* _traversal_gc; + + template + inline void do_oop_nv(T* p) { + _traversal_gc->process_oop(p, _thread, _queue); + } + +public: + ShenandoahTraversalKeepAliveUpdateClosure(ShenandoahObjToScanQueue* q) : + _queue(q), _thread(Thread::current()), + _traversal_gc(ShenandoahHeap::heap()->traversal_gc()) {} + + void do_oop(narrowOop* p) { do_oop_nv(p); } + void do_oop(oop* p) { do_oop_nv(p); } +}; + +void ShenandoahTraversalGC::preclean_weak_refs() { + // Pre-cleaning weak references before diving into STW makes sense at the + // end of concurrent mark. This will filter out the references which referents + // are alive. Note that ReferenceProcessor already filters out these on reference + // discovery, and the bulk of work is done here. This phase processes leftovers + // that missed the initial filtering, i.e. when referent was marked alive after + // reference was discovered by RP. + + assert(_heap->shenandoahPolicy()->process_references(), "sanity"); + + ShenandoahHeap* sh = ShenandoahHeap::heap(); + ReferenceProcessor* rp = sh->ref_processor(); + + // Shortcut if no references were discovered to avoid winding up threads. + if (!rp->has_discovered_references()) { + return; + } + + ReferenceProcessorMTDiscoveryMutator fix_mt_discovery(rp, false); + ReferenceProcessorIsAliveMutator fix_alive(rp, sh->is_alive_closure()); + + // Interrupt on cancelled GC + ShenandoahTraversalCancelledGCYieldClosure yield; + + assert(task_queues()->is_empty(), "Should be empty"); + + ShenandoahTraversalPrecleanCompleteGCClosure complete_gc; + ShenandoahForwardedIsAliveClosure is_alive; + ShenandoahTraversalKeepAliveUpdateClosure keep_alive(task_queues()->queue(0)); + ResourceMark rm; + rp->preclean_discovered_references(&is_alive, &keep_alive, + &complete_gc, &yield, + NULL); + assert(task_queues()->is_empty(), "Should be empty"); +} + +// Weak Reference Closures +class ShenandoahTraversalDrainMarkingStackClosure: public VoidClosure { + uint _worker_id; + ParallelTaskTerminator* _terminator; + bool _reset_terminator; + +public: + ShenandoahTraversalDrainMarkingStackClosure(uint worker_id, ParallelTaskTerminator* t, bool reset_terminator = false): + _worker_id(worker_id), + _terminator(t), + _reset_terminator(reset_terminator) { + } + + void do_void() { + assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint"); + + ShenandoahHeap* sh = ShenandoahHeap::heap(); + ShenandoahTraversalGC* traversal_gc = sh->traversal_gc(); + assert(sh->shenandoahPolicy()->process_references(), "why else would we be here?"); + ReferenceProcessor* rp = sh->ref_processor(); + ReferenceProcessorIsAliveMutator fix_alive(rp, sh->is_alive_closure()); + + traversal_gc->main_loop(_worker_id, _terminator, false); + traversal_gc->flush_liveness(_worker_id); + + if (_reset_terminator) { + _terminator->reset_for_reuse(); + } + } +}; + +void ShenandoahTraversalGC::weak_refs_work() { + assert(_heap->shenandoahPolicy()->process_references(), "sanity"); + + ShenandoahHeap* sh = ShenandoahHeap::heap(); + + ShenandoahPhaseTimings::Phase phase_root = ShenandoahPhaseTimings::weakrefs; + + ShenandoahGCPhase phase(phase_root); + + ReferenceProcessor* rp = sh->ref_processor(); + + // NOTE: We cannot shortcut on has_discovered_references() here, because + // we will miss marking JNI Weak refs then, see implementation in + // ReferenceProcessor::process_discovered_references. + weak_refs_work_doit(); + + rp->verify_no_references_recorded(); + assert(!rp->discovery_enabled(), "Post condition"); + +} + +class ShenandoahTraversalRefProcTaskProxy : public AbstractGangTask { + +private: + AbstractRefProcTaskExecutor::ProcessTask& _proc_task; + ParallelTaskTerminator* _terminator; +public: + + ShenandoahTraversalRefProcTaskProxy(AbstractRefProcTaskExecutor::ProcessTask& proc_task, + ParallelTaskTerminator* t) : + AbstractGangTask("Process reference objects in parallel"), + _proc_task(proc_task), + _terminator(t) { + } + + void work(uint worker_id) { + assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint"); + ShenandoahHeap* heap = ShenandoahHeap::heap(); + ShenandoahTraversalDrainMarkingStackClosure complete_gc(worker_id, _terminator); + + ShenandoahForwardedIsAliveClosure is_alive; + ShenandoahTraversalKeepAliveUpdateClosure keep_alive(heap->traversal_gc()->task_queues()->queue(worker_id)); + _proc_task.work(worker_id, is_alive, keep_alive, complete_gc); + } +}; + +class ShenandoahTraversalRefEnqueueTaskProxy : public AbstractGangTask { + +private: + AbstractRefProcTaskExecutor::EnqueueTask& _enqueue_task; + +public: + + ShenandoahTraversalRefEnqueueTaskProxy(AbstractRefProcTaskExecutor::EnqueueTask& enqueue_task) : + AbstractGangTask("Enqueue reference objects in parallel"), + _enqueue_task(enqueue_task) { + } + + void work(uint worker_id) { + _enqueue_task.work(worker_id); + } +}; + +class ShenandoahTraversalRefProcTaskExecutor : public AbstractRefProcTaskExecutor { + +private: + WorkGang* _workers; + +public: + + ShenandoahTraversalRefProcTaskExecutor(WorkGang* workers) : + _workers(workers) { + } + + // Executes a task using worker threads. + void execute(ProcessTask& task) { + assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint"); + + // Shortcut execution if task is empty. + // This should be replaced with the generic ReferenceProcessor shortcut, + // see JDK-8181214, JDK-8043575, JDK-6938732. + if (task.is_empty()) { + return; + } + + ShenandoahHeap* heap = ShenandoahHeap::heap(); + ShenandoahTraversalGC* traversal_gc = heap->traversal_gc(); + uint nworkers = _workers->active_workers(); + traversal_gc->task_queues()->reserve(nworkers); + if (UseShenandoahOWST) { + ShenandoahTaskTerminator terminator(nworkers, traversal_gc->task_queues()); + ShenandoahTraversalRefProcTaskProxy proc_task_proxy(task, &terminator); + _workers->run_task(&proc_task_proxy); + } else { + ParallelTaskTerminator terminator(nworkers, traversal_gc->task_queues()); + ShenandoahTraversalRefProcTaskProxy proc_task_proxy(task, &terminator); + _workers->run_task(&proc_task_proxy); + } + } + + void execute(EnqueueTask& task) { + ShenandoahTraversalRefEnqueueTaskProxy enqueue_task_proxy(task); + _workers->run_task(&enqueue_task_proxy); + } +}; + +void ShenandoahTraversalGC::weak_refs_work_doit() { + ShenandoahHeap* sh = ShenandoahHeap::heap(); + + ReferenceProcessor* rp = sh->ref_processor(); + + ShenandoahPhaseTimings::Phase phase_process = ShenandoahPhaseTimings::weakrefs_process; + ShenandoahPhaseTimings::Phase phase_enqueue = ShenandoahPhaseTimings::weakrefs_enqueue; + + ReferenceProcessorIsAliveMutator fix_alive(rp, sh->is_alive_closure()); + + WorkGang* workers = sh->workers(); + uint nworkers = workers->active_workers(); + + // Setup collector policy for softref cleaning. + bool clear_soft_refs = sh->collector_policy()->use_should_clear_all_soft_refs(true /* bogus arg*/); + log_develop_debug(gc, ref)("clearing soft refs: %s", BOOL_TO_STR(clear_soft_refs)); + rp->setup_policy(clear_soft_refs); + rp->set_active_mt_degree(nworkers); + + assert(task_queues()->is_empty(), "Should be empty"); + + // complete_gc and keep_alive closures instantiated here are only needed for + // single-threaded path in RP. They share the queue 0 for tracking work, which + // simplifies implementation. Since RP may decide to call complete_gc several + // times, we need to be able to reuse the terminator. + uint serial_worker_id = 0; + ParallelTaskTerminator terminator(1, task_queues()); + ShenandoahTraversalDrainMarkingStackClosure complete_gc(serial_worker_id, &terminator, /* reset_terminator = */ true); + + ShenandoahTraversalRefProcTaskExecutor executor(workers); + + ReferenceProcessorPhaseTimes pt(sh->gc_timer(), rp->num_q()); + + { + ShenandoahGCPhase phase(phase_process); + + ShenandoahForwardedIsAliveClosure is_alive; + ShenandoahTraversalKeepAliveUpdateClosure keep_alive(task_queues()->queue(serial_worker_id)); + rp->process_discovered_references(&is_alive, &keep_alive, + &complete_gc, &executor, + &pt); + pt.print_all_references(); + + assert(task_queues()->is_empty(), "Should be empty"); + } + + { + ShenandoahGCPhase phase(phase_enqueue); + rp->enqueue_discovered_references(&executor, &pt); + pt.print_enqueue_phase(); + } +} diff --git a/src/hotspot/share/gc/shenandoah/shenandoahTraversalGC.hpp b/src/hotspot/share/gc/shenandoah/shenandoahTraversalGC.hpp new file mode 100644 --- /dev/null +++ b/src/hotspot/share/gc/shenandoah/shenandoahTraversalGC.hpp @@ -0,0 +1,97 @@ +/* + * Copyright (c) 2017, Red Hat, Inc. and/or its affiliates. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_SHENANDOAH_SHENANDOAHTRAVERSALGC_HPP +#define SHARE_VM_GC_SHENANDOAH_SHENANDOAHTRAVERSALGC_HPP + +#include "memory/allocation.hpp" +#include "gc/shenandoah/shenandoahTaskqueue.hpp" + +class Thread; +class ShenandoahHeapRegionSet; +class ShenandoahHeap; + +class ShenandoahTraversalGC : public CHeapObj { +private: + ShenandoahHeap* _heap; + MarkBitMap* _bitmap; + ShenandoahObjToScanQueueSet* _task_queues; + + // Used for buffering per-region liveness data. + // Needed since ShenandoahHeapRegion uses atomics to update liveness. + // + // The array has max-workers elements, each of which is an array of + // jushort * max_regions. The choice of jushort is not accidental: + // there is a tradeoff between static/dynamic footprint that translates + // into cache pressure (which is already high during marking), and + // too many atomic updates. size_t/jint is too large, jbyte is too small. + jushort** _liveness_local; + +public: + ShenandoahTraversalGC(ShenandoahHeap* heap, size_t num_regions); + ~ShenandoahTraversalGC(); + + void reset(); + void prepare(); + void init_traversal_collection(); + void concurrent_traversal_collection(); + void final_traversal_collection(); + + template + inline void process_oop(T* p, Thread* thread, ShenandoahObjToScanQueue* queue); + + bool check_and_handle_cancelled_gc(ParallelTaskTerminator* terminator); + + ShenandoahObjToScanQueueSet* task_queues(); + + jushort* get_liveness(uint worker_id); + void flush_liveness(uint worker_id); + + void main_loop(uint worker_id, ParallelTaskTerminator* terminator, bool do_satb); + +private: + + template + void main_loop_prework(uint w, ParallelTaskTerminator* t); + + template + void main_loop_work(T* cl, jushort* live_data, uint worker_id, ParallelTaskTerminator* terminator); + + template + inline void do_task(ShenandoahObjToScanQueue* q, T* cl, jushort* live_data, ShenandoahMarkTask* task); + + template + inline void do_chunked_array_start(ShenandoahObjToScanQueue* q, T* cl, oop array); + + template + inline void do_chunked_array(ShenandoahObjToScanQueue* q, T* cl, oop array, int chunk, int pow); + + inline void count_liveness(jushort* live_data, oop obj); + + void preclean_weak_refs(); + void weak_refs_work(); + void weak_refs_work_doit(); + +}; + +#endif // SHARE_VM_GC_SHENANDOAH_SHENANDOAHTRAVERSALGC_HPP diff --git a/src/hotspot/share/gc/shenandoah/shenandoahTraversalGC.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahTraversalGC.inline.hpp new file mode 100644 --- /dev/null +++ b/src/hotspot/share/gc/shenandoah/shenandoahTraversalGC.inline.hpp @@ -0,0 +1,207 @@ +/* + * Copyright (c) 2017, Red Hat, Inc. and/or its affiliates. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_SHENANDOAH_SHENANDOAHTRAVERSALGC_INLINE_HPP +#define SHARE_VM_GC_SHENANDOAH_SHENANDOAHTRAVERSALGC_INLINE_HPP + +#include "gc/shared/markBitMap.inline.hpp" +#include "gc/shenandoah/shenandoahBarrierSet.inline.hpp" +#include "gc/shenandoah/shenandoahHeap.inline.hpp" +#include "gc/shenandoah/shenandoahHeapRegion.inline.hpp" +#include "gc/shenandoah/shenandoahTraversalGC.hpp" +#include "gc/shenandoah/shenandoahTaskqueue.hpp" +#include "memory/iterator.inline.hpp" +#include "oops/oop.inline.hpp" + +template +void ShenandoahTraversalGC::process_oop(T* p, Thread* thread, ShenandoahObjToScanQueue* queue) { + T o = oopDesc::load_heap_oop(p); + if (! oopDesc::is_null(o)) { + oop obj = oopDesc::decode_heap_oop_not_null(o); + if (_heap->in_collection_set(obj)) { + oop forw = ShenandoahBarrierSet::resolve_oop_static_not_null(obj); + if (oopDesc::unsafe_equals(obj, forw)) { + bool evacuated = false; + forw = _heap->evacuate_object(obj, thread, evacuated); + } + assert(! oopDesc::unsafe_equals(obj, forw) || _heap->cancelled_concgc(), "must be evacuated"); + // Update reference. + _heap->atomic_compare_exchange_oop(forw, p, obj); + obj = forw; + } + + if ((!_bitmap->isMarked((HeapWord*) obj)) && _bitmap->parMark((HeapWord*) obj)) { + bool succeeded = queue->push(ShenandoahMarkTask(obj)); + assert(succeeded, "must succeed to push to task queue"); + } + } +} + +template +void ShenandoahTraversalGC::do_task(ShenandoahObjToScanQueue* q, T* cl, jushort* live_data, ShenandoahMarkTask* task) { + oop obj = task->obj(); + + assert(obj != NULL, "expect non-null object"); + assert(oopDesc::unsafe_equals(obj, ShenandoahBarrierSet::resolve_oop_static_not_null(obj)), "expect forwarded obj in queue"); + assert(_heap->cancelled_concgc() + || oopDesc::bs()->is_safe(obj), + "we don't want to mark objects in from-space"); + assert(_heap->is_in(obj), "referenced objects must be in the heap. No?"); + assert(_heap->is_marked_next(obj), "only marked objects on task queue"); + + if (task->is_not_chunked()) { + count_liveness(live_data, obj); + if (obj->is_instance()) { + // Case 1: Normal oop, process as usual. + obj->oop_iterate(cl); + } else if (obj->is_objArray()) { + // Case 2: Object array instance and no chunk is set. Must be the first + // time we visit it, start the chunked processing. + do_chunked_array_start(q, cl, obj); + } else { + // Case 3: Primitive array. Do nothing, no oops there. We use the same + // performance tweak TypeArrayKlass::oop_oop_iterate_impl is using: + // We skip iterating over the klass pointer since we know that + // Universe::TypeArrayKlass never moves. + assert (obj->is_typeArray(), "should be type array"); + } + } else { + // Case 4: Array chunk, has sensible chunk id. Process it. + do_chunked_array(q, cl, obj, task->chunk(), task->pow()); + } +} + +inline void ShenandoahTraversalGC::count_liveness(jushort* live_data, oop obj) { + size_t region_idx = _heap->heap_region_index_containing(obj); + jushort cur = live_data[region_idx]; + int size = obj->size() + BrooksPointer::word_size(); + int max = (1 << (sizeof(jushort) * 8)) - 1; + if (size >= max) { + // too big, add to region data directly + _heap->regions()->get(region_idx)->increase_live_data_words(size); + } else { + int new_val = cur + size; + if (new_val >= max) { + // overflow, flush to region data + _heap->regions()->get(region_idx)->increase_live_data_words(new_val); + live_data[region_idx] = 0; + } else { + // still good, remember in locals + live_data[region_idx] = (jushort) new_val; + } + } +} + +template +inline void ShenandoahTraversalGC::do_chunked_array_start(ShenandoahObjToScanQueue* q, T* cl, oop obj) { + assert(obj->is_objArray(), "expect object array"); + objArrayOop array = objArrayOop(obj); + int len = array->length(); + + if (len <= (int) ObjArrayMarkingStride*2) { + // A few slices only, process directly + array->oop_iterate_range(cl, 0, len); + } else { + int bits = log2_long(len); + // Compensate for non-power-of-two arrays, cover the array in excess: + if (len != (1 << bits)) bits++; + + // Only allow full chunks on the queue. This frees do_chunked_array() from checking from/to + // boundaries against array->length(), touching the array header on every chunk. + // + // To do this, we cut the prefix in full-sized chunks, and submit them on the queue. + // If the array is not divided in chunk sizes, then there would be an irregular tail, + // which we will process separately. + + int last_idx = 0; + + int chunk = 1; + int pow = bits; + + // Handle overflow + if (pow >= 31) { + assert (pow == 31, "sanity"); + pow--; + chunk = 2; + last_idx = (1 << pow); + bool pushed = q->push(ShenandoahMarkTask(array, 1, pow)); + assert(pushed, "overflow queue should always succeed pushing"); + } + + // Split out tasks, as suggested in ObjArrayChunkedTask docs. Record the last + // successful right boundary to figure out the irregular tail. + while ((1 << pow) > (int)ObjArrayMarkingStride && + (chunk*2 < ShenandoahMarkTask::chunk_size())) { + pow--; + int left_chunk = chunk*2 - 1; + int right_chunk = chunk*2; + int left_chunk_end = left_chunk * (1 << pow); + if (left_chunk_end < len) { + bool pushed = q->push(ShenandoahMarkTask(array, left_chunk, pow)); + assert(pushed, "overflow queue should always succeed pushing"); + chunk = right_chunk; + last_idx = left_chunk_end; + } else { + chunk = left_chunk; + } + } + + // Process the irregular tail, if present + int from = last_idx; + if (from < len) { + array->oop_iterate_range(cl, from, len); + } + } +} + +template +inline void ShenandoahTraversalGC::do_chunked_array(ShenandoahObjToScanQueue* q, T* cl, oop obj, int chunk, int pow) { + assert(obj->is_objArray(), "expect object array"); + objArrayOop array = objArrayOop(obj); + + assert (ObjArrayMarkingStride > 0, "sanity"); + + // Split out tasks, as suggested in ObjArrayChunkedTask docs. Avoid pushing tasks that + // are known to start beyond the array. + while ((1 << pow) > (int)ObjArrayMarkingStride && (chunk*2 < ShenandoahMarkTask::chunk_size())) { + pow--; + chunk *= 2; + bool pushed = q->push(ShenandoahMarkTask(array, chunk - 1, pow)); + assert(pushed, "overflow queue should always succeed pushing"); + } + + int chunk_size = 1 << pow; + + int from = (chunk - 1) * chunk_size; + int to = chunk * chunk_size; + +#ifdef ASSERT + int len = array->length(); + assert (0 <= from && from < len, "from is sane: %d/%d", from, len); + assert (0 < to && to <= len, "to is sane: %d/%d", to, len); +#endif + + array->oop_iterate_range(cl, from, to); +} + +#endif // SHARE_VM_GC_SHENANDOAH_SHENANDOAHTRAVERSALGC_INLINE_HPP diff --git a/src/hotspot/share/gc/shenandoah/shenandoahUtils.hpp b/src/hotspot/share/gc/shenandoah/shenandoahUtils.hpp --- a/src/hotspot/share/gc/shenandoah/shenandoahUtils.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahUtils.hpp @@ -87,6 +87,8 @@ type == VM_Operation::VMOp_ShenandoahVerifyHeapAfterEvacuation || type == VM_Operation::VMOp_ShenandoahInitPartialGC || type == VM_Operation::VMOp_ShenandoahFinalPartialGC || + type == VM_Operation::VMOp_ShenandoahInitTraversalGC || + type == VM_Operation::VMOp_ShenandoahFinalTraversalGC || type == VM_Operation::VMOp_ShenandoahInitUpdateRefs || type == VM_Operation::VMOp_ShenandoahFinalUpdateRefs || type == VM_Operation::VMOp_ShenandoahFullGC; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp b/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp --- a/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp @@ -942,7 +942,31 @@ _verify_marked_complete, // bitmaps might be stale, but alloc-after-mark should be well _verify_matrix_conservative, // matrix is conservatively consistent _verify_cset_none, // no cset references left after partial + _verify_liveness_complete, // no reliable liveness data anymore + _verify_regions_nocset // no cset regions, trash regions allowed + ); +} + +void ShenandoahVerifier::verify_before_traversal() { + verify_at_safepoint( + "Before Traversal", + _verify_forwarded_none, // cannot have forwarded objects + _verify_marked_disable, // bitmaps are not relevant before traversal + _verify_matrix_disable, // matrix is not used in traversal + _verify_cset_none, // no cset references before partial _verify_liveness_disable, // no reliable liveness data anymore + _verify_regions_notrash_nocset // no trash and no cset regions + ); +} + +void ShenandoahVerifier::verify_after_traversal() { + verify_at_safepoint( + "After Traversal", + _verify_forwarded_none, // cannot have forwarded objects + _verify_marked_next, // bitmaps might be stale, but alloc-after-mark should be well + _verify_matrix_disable, // matrix is conservatively consistent + _verify_cset_none, // no cset references left after partial + _verify_liveness_complete, // no reliable liveness data anymore _verify_regions_nocset // no cset regions, trash regions allowed ); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahVerifier.hpp b/src/hotspot/share/gc/shenandoah/shenandoahVerifier.hpp --- a/src/hotspot/share/gc/shenandoah/shenandoahVerifier.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahVerifier.hpp @@ -186,6 +186,8 @@ void verify_after_fullgc(); void verify_before_partial(); void verify_after_partial(); + void verify_before_traversal(); + void verify_after_traversal(); void verify_generic(VerifyOption option); static void verify_oop_fwdptr(oop obj, oop new_fwd); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahWorkerPolicy.cpp b/src/hotspot/share/gc/shenandoah/shenandoahWorkerPolicy.cpp --- a/src/hotspot/share/gc/shenandoah/shenandoahWorkerPolicy.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahWorkerPolicy.cpp @@ -33,6 +33,8 @@ uint ShenandoahWorkerPolicy::_prev_fullgc = 0; uint ShenandoahWorkerPolicy::_prev_stw_partial = 0; uint ShenandoahWorkerPolicy::_prev_conc_partial = 0; +uint ShenandoahWorkerPolicy::_prev_stw_traversal = 0; +uint ShenandoahWorkerPolicy::_prev_conc_traversal = 0; uint ShenandoahWorkerPolicy::_prev_conc_update_ref = 0; uint ShenandoahWorkerPolicy::_prev_par_update_ref = 0; uint ShenandoahWorkerPolicy::_prev_conc_cleanup = 0; @@ -101,6 +103,26 @@ return _prev_conc_partial; } +// Calculate workers for Stop-the-world traversal GC +uint ShenandoahWorkerPolicy::calc_workers_for_stw_traversal() { + uint active_workers = (_prev_stw_traversal == 0) ? ParallelGCThreads : _prev_stw_traversal; + _prev_stw_traversal = + AdaptiveSizePolicy::calc_active_workers(ParallelGCThreads, + active_workers, + Threads::number_of_non_daemon_threads()); + return _prev_stw_traversal; +} + +// Calculate workers for concurent traversal GC +uint ShenandoahWorkerPolicy::calc_workers_for_conc_traversal() { + uint active_workers = (_prev_conc_traversal == 0) ? ConcGCThreads : _prev_conc_traversal; + _prev_conc_traversal = + AdaptiveSizePolicy::calc_active_conc_workers(ConcGCThreads, + active_workers, + Threads::number_of_non_daemon_threads()); + return _prev_conc_traversal; +} + // Calculate workers for concurrent reference update uint ShenandoahWorkerPolicy::calc_workers_for_conc_update_ref() { uint active_workers = (_prev_conc_update_ref == 0) ? ConcGCThreads : _prev_conc_update_ref; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahWorkerPolicy.hpp b/src/hotspot/share/gc/shenandoah/shenandoahWorkerPolicy.hpp --- a/src/hotspot/share/gc/shenandoah/shenandoahWorkerPolicy.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahWorkerPolicy.hpp @@ -36,6 +36,8 @@ static uint _prev_fullgc; static uint _prev_stw_partial; static uint _prev_conc_partial; + static uint _prev_stw_traversal; + static uint _prev_conc_traversal; static uint _prev_conc_update_ref; static uint _prev_par_update_ref; static uint _prev_conc_cleanup; @@ -62,6 +64,12 @@ // Calculate workers for concurrent partial GC static uint calc_workers_for_conc_partial(); + // Calculate workers for Stop-the-world traversal GC + static uint calc_workers_for_stw_traversal(); + + // Calculate workers for concurrent traversal GC + static uint calc_workers_for_conc_traversal(); + // Calculate workers for concurrent reference update static uint calc_workers_for_conc_update_ref(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp b/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp --- a/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp @@ -255,6 +255,9 @@ experimental(bool, ShenandoahLoopOptsAfterExpansion, true, \ "Attempt more loop opts after write barrier expansion") \ \ + experimental(bool, ShenandoahWBWithMemBar, true, \ + "Emit write barrier with membar for concurrent change of flag") \ + \ experimental(bool, UseShenandoahOWST, true, \ "Use Shenandoah work stealing termination protocol") \ \ @@ -311,6 +314,12 @@ diagnostic(bool, ShenandoahReadBarrier, true, \ "Turn on/off read barriers in Shenandoah") \ \ + diagnostic(bool, ShenandoahStoreValEnqueueBarrier, false, \ + "Turn on/off enqueuing of oops after write barriers (MWF)") \ + \ + diagnostic(bool, ShenandoahMWF, false, \ + "Turn on/off enqueuing of oops after write barriers (MWF)") \ + \ diagnostic(bool, ShenandoahStoreValWriteBarrier, false, \ "Turn on/off store val write barriers in Shenandoah") \ \ diff --git a/src/hotspot/share/gc/shenandoah/shenandoah_specialized_oop_closures.hpp b/src/hotspot/share/gc/shenandoah/shenandoah_specialized_oop_closures.hpp --- a/src/hotspot/share/gc/shenandoah/shenandoah_specialized_oop_closures.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoah_specialized_oop_closures.hpp @@ -31,6 +31,9 @@ class ShenandoahUpdateHeapRefsClosure; class ShenandoahUpdateHeapRefsMatrixClosure; class ShenandoahPartialEvacuateUpdateHeapClosure; +class ShenandoahPartialEvacuateUpdateHeapClosure; +class ShenandoahTraversalClosure; +class ShenandoahTraversalMetadataClosure; #define SPECIALIZED_OOP_OOP_ITERATE_CLOSURES_SHENANDOAH(f) \ f(ShenandoahMarkUpdateRefsClosure,_nv) \ @@ -39,6 +42,8 @@ f(ShenandoahMarkRefsMetadataClosure,_nv) \ f(ShenandoahUpdateHeapRefsClosure,_nv) \ f(ShenandoahUpdateHeapRefsMatrixClosure,_nv) \ + f(ShenandoahTraversalClosure,_nv) \ + f(ShenandoahTraversalMetadataClosure,_nv) \ f(ShenandoahPartialEvacuateUpdateHeapClosure,_nv) #endif // SHARE_VM_GC_SHENANDOAH_SHENANDOAH_SPECIALIZED_OOP_CLOSURES_HPP diff --git a/src/hotspot/share/gc/shenandoah/vm_operations_shenandoah.cpp b/src/hotspot/share/gc/shenandoah/vm_operations_shenandoah.cpp --- a/src/hotspot/share/gc/shenandoah/vm_operations_shenandoah.cpp +++ b/src/hotspot/share/gc/shenandoah/vm_operations_shenandoah.cpp @@ -29,6 +29,7 @@ #include "gc/shenandoah/shenandoahHeap.inline.hpp" #include "gc/shenandoah/shenandoahMarkCompact.hpp" #include "gc/shenandoah/shenandoahPartialGC.hpp" +#include "gc/shenandoah/shenandoahTraversalGC.hpp" #include "gc/shenandoah/shenandoahUtils.hpp" #include "gc/shenandoah/shenandoahVerifier.hpp" #include "gc/shenandoah/shenandoahWorkGroup.hpp" @@ -72,6 +73,24 @@ ShenandoahHeap::heap()->entry_final_partial(); } +void VM_ShenandoahInitTraversalGC::doit() { + ShenandoahGCPauseMark mark(_gc_id, SvcGCMarker::OTHER); + + ShenandoahHeap* sh = ShenandoahHeap::heap(); + GCTraceTime(Info, gc) time("Pause Init Traversal", sh->gc_timer()); + + sh->traversal_gc()->init_traversal_collection(); +} + +void VM_ShenandoahFinalTraversalGC::doit() { + ShenandoahGCPauseMark mark(_gc_id, SvcGCMarker::OTHER); + + ShenandoahHeap* sh = ShenandoahHeap::heap(); + GCTraceTime(Info, gc) time("Pause Final Traversal", sh->gc_timer()); + + sh->traversal_gc()->final_traversal_collection(); +} + void VM_ShenandoahInitUpdateRefs::doit() { ShenandoahGCPauseMark mark(_gc_id, SvcGCMarker::OTHER); ShenandoahHeap::heap()->entry_init_updaterefs(); diff --git a/src/hotspot/share/gc/shenandoah/vm_operations_shenandoah.hpp b/src/hotspot/share/gc/shenandoah/vm_operations_shenandoah.hpp --- a/src/hotspot/share/gc/shenandoah/vm_operations_shenandoah.hpp +++ b/src/hotspot/share/gc/shenandoah/vm_operations_shenandoah.hpp @@ -38,6 +38,8 @@ // - VM_ShenandoahFullGC: do full GC // - VM_ShenandoahInitPartialGC: init partial GC // - VM_ShenandoahFinalPartialGC: finish partial GC +// - VM_ShenandoahInitTraversalGC: init traversal GC +// - VM_ShenandoahFinalTraversalGC: finish traversal GC class VM_ShenandoahOperation : public VM_Operation { protected: @@ -103,6 +105,22 @@ virtual void doit(); }; +class VM_ShenandoahInitTraversalGC: public VM_ShenandoahOperation { +public: + VM_ShenandoahInitTraversalGC() : VM_ShenandoahOperation() {}; + VM_Operation::VMOp_Type type() const { return VMOp_ShenandoahInitTraversalGC; } + const char* name() const { return "Shenandoah Init Traversal Collection"; } + virtual void doit(); +}; + +class VM_ShenandoahFinalTraversalGC: public VM_ShenandoahReferenceOperation { +public: + VM_ShenandoahFinalTraversalGC() : VM_ShenandoahReferenceOperation() {}; + VM_Operation::VMOp_Type type() const { return VMOp_ShenandoahFinalTraversalGC; } + const char* name() const { return "Shenandoah Final Traversal Collection"; } + virtual void doit(); +}; + class VM_ShenandoahInitUpdateRefs: public VM_ShenandoahOperation { public: VM_ShenandoahInitUpdateRefs() : VM_ShenandoahOperation() {}; diff --git a/src/hotspot/share/opto/compile.cpp b/src/hotspot/share/opto/compile.cpp --- a/src/hotspot/share/opto/compile.cpp +++ b/src/hotspot/share/opto/compile.cpp @@ -3381,7 +3381,7 @@ } case Op_If: { #ifdef ASSERT - if (ShenandoahWriteBarrierNode::is_evacuation_in_progress_test(n->as_If())) { + if (ShenandoahWriteBarrierNode::is_evacuation_in_progress_test(n->as_If()) && ShenandoahWBWithMemBar) { Node* c = n->in(0); int count = 0; for (;;) { diff --git a/src/hotspot/share/opto/graphKit.cpp b/src/hotspot/share/opto/graphKit.cpp --- a/src/hotspot/share/opto/graphKit.cpp +++ b/src/hotspot/share/opto/graphKit.cpp @@ -4202,6 +4202,111 @@ g1_write_barrier_pre_helper(*this, adr); } +void GraphKit::shenandoah_enqueue_barrier(Node* pre_val) { + + // Some sanity checks + assert(pre_val != NULL, "must be loaded already"); + // Nothing to be done if pre_val is null. + if (pre_val->bottom_type()->higher_equal(TypePtr::NULL_PTR)) return; + assert(pre_val->bottom_type()->basic_type() == T_OBJECT, "or we shouldn't be here"); + + IdealKit ideal(this, true); + + Node* tls = __ thread(); // ThreadLocalStorage + + Node* no_base = __ top(); + Node* zero = __ ConI(0); + Node* zeroX = __ ConX(0); + + float likely = PROB_LIKELY(0.999); + float unlikely = PROB_UNLIKELY(0.999); + + // Offsets into the thread + const int marking_offset = in_bytes(JavaThread::evacuation_in_progress_offset()); + /* + const int marking_offset = in_bytes(JavaThread::satb_mark_queue_offset() + // 648 + SATBMarkQueue::byte_offset_of_active()); + */ + const int index_offset = in_bytes(JavaThread::satb_mark_queue_offset() + // 656 + SATBMarkQueue::byte_offset_of_index()); + const int buffer_offset = in_bytes(JavaThread::satb_mark_queue_offset() + // 652 + SATBMarkQueue::byte_offset_of_buf()); + + // Now the actual pointers into the thread + Node* marking_adr = __ AddP(no_base, tls, __ ConX(marking_offset)); + Node* buffer_adr = __ AddP(no_base, tls, __ ConX(buffer_offset)); + Node* index_adr = __ AddP(no_base, tls, __ ConX(index_offset)); + + const Type* obj_type = pre_val->bottom_type(); + if (obj_type->meet(TypePtr::NULL_PTR) == obj_type->remove_speculative()) { + // dunno if it's NULL or not. + // if (pre_val != NULL) + __ if_then(pre_val, BoolTest::ne, null()); { + + // Now some of the values + Node* marking = __ load(__ ctrl(), marking_adr, TypeInt::BOOL, T_BOOLEAN, Compile::AliasIdxRaw); + + // if (!marking) + __ if_then(marking, BoolTest::ne, zero, unlikely); { + BasicType index_bt = TypeX_X->basic_type(); + assert(sizeof(size_t) == type2aelembytes(index_bt), "Loading G1 SATBMarkQueue::_index with wrong size."); + Node* index = __ load(__ ctrl(), index_adr, TypeX_X, index_bt, Compile::AliasIdxRaw); + + // is the queue for this thread full? + __ if_then(index, BoolTest::ne, zeroX, likely); { + + // decrement the index + Node* next_index = _gvn.transform(new SubXNode(index, __ ConX(sizeof(intptr_t)))); + + // Now get the buffer location we will log the previous value into and store it + Node* buffer = __ load(__ ctrl(), buffer_adr, TypeRawPtr::NOTNULL, T_ADDRESS, Compile::AliasIdxRaw); + Node *log_addr = __ AddP(no_base, buffer, next_index); + __ store(__ ctrl(), log_addr, pre_val, T_OBJECT, Compile::AliasIdxRaw, MemNode::unordered); + // update the index + __ store(__ ctrl(), index_adr, next_index, index_bt, Compile::AliasIdxRaw, MemNode::unordered); + + } __ else_(); { + // logging buffer is full, call the runtime + const TypeFunc *tf = OptoRuntime::g1_wb_pre_Type(); + __ make_leaf_call(tf, CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_pre), "g1_wb_pre", pre_val, tls); + } __ end_if(); // (!index) + } __ end_if(); // (!marking) + } __ end_if(); // (pre_val != NULL) + } else { + // We know it is not null. + // Now some of the values + Node* marking = __ load(__ ctrl(), marking_adr, TypeInt::BOOL, T_BOOLEAN, Compile::AliasIdxRaw); + + // if (!marking) + __ if_then(marking, BoolTest::ne, zero, unlikely); { + BasicType index_bt = TypeX_X->basic_type(); + assert(sizeof(size_t) == type2aelembytes(index_bt), "Loading G1 SATBMarkQueue::_index with wrong size."); + Node* index = __ load(__ ctrl(), index_adr, TypeX_X, index_bt, Compile::AliasIdxRaw); + + // is the queue for this thread full? + __ if_then(index, BoolTest::ne, zeroX, likely); { + + // decrement the index + Node* next_index = _gvn.transform(new SubXNode(index, __ ConX(sizeof(intptr_t)))); + + // Now get the buffer location we will log the previous value into and store it + Node* buffer = __ load(__ ctrl(), buffer_adr, TypeRawPtr::NOTNULL, T_ADDRESS, Compile::AliasIdxRaw); + Node *log_addr = __ AddP(no_base, buffer, next_index); + __ store(__ ctrl(), log_addr, pre_val, T_OBJECT, Compile::AliasIdxRaw, MemNode::unordered); + // update the index + __ store(__ ctrl(), index_adr, next_index, index_bt, Compile::AliasIdxRaw, MemNode::unordered); + + } __ else_(); { + // logging buffer is full, call the runtime + const TypeFunc *tf = OptoRuntime::g1_wb_pre_Type(); + __ make_leaf_call(tf, CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_pre), "g1_wb_pre", pre_val, tls); + } __ end_if(); // (!index) + } __ end_if(); // (!marking) + } + // Final sync IdealKit and GraphKit. + final_sync(ideal); +} + void GraphKit::shenandoah_write_barrier_pre(bool do_load, Node* obj, Node* adr, @@ -4773,10 +4878,13 @@ Node* GraphKit::shenandoah_storeval_barrier(Node* obj) { if (UseShenandoahGC) { if (ShenandoahStoreValWriteBarrier) { - return shenandoah_write_barrier_impl(obj); + obj = shenandoah_write_barrier(obj); + } + if (ShenandoahStoreValEnqueueBarrier && !ShenandoahMWF) { + shenandoah_enqueue_barrier(obj); } if (ShenandoahStoreValReadBarrier) { - return shenandoah_read_barrier_impl(obj, true, false, false); + obj = shenandoah_read_barrier_impl(obj, true, false, false); } } return obj; @@ -4834,7 +4942,7 @@ } } -static Node* shenandoah_write_barrier_helper(GraphKit& kit, Node* obj, const TypePtr* adr_type) { +Node* GraphKit::shenandoah_write_barrier_helper(GraphKit& kit, Node* obj, const TypePtr* adr_type) { ShenandoahWriteBarrierNode* wb = new ShenandoahWriteBarrierNode(kit.C, kit.control(), kit.memory(adr_type), obj); Node* n = kit.gvn().transform(wb); if (n == wb) { // New barrier needs memory projection. @@ -4848,7 +4956,11 @@ Node* GraphKit::shenandoah_write_barrier(Node* obj) { if (UseShenandoahGC && ShenandoahWriteBarrier) { - return shenandoah_write_barrier_impl(obj); + obj = shenandoah_write_barrier_impl(obj); + if (ShenandoahStoreValEnqueueBarrier && ShenandoahMWF) { + shenandoah_enqueue_barrier(obj); + } + return obj; } else { return obj; } diff --git a/src/hotspot/share/opto/graphKit.hpp b/src/hotspot/share/opto/graphKit.hpp --- a/src/hotspot/share/opto/graphKit.hpp +++ b/src/hotspot/share/opto/graphKit.hpp @@ -782,6 +782,7 @@ const TypeOopPtr* val_type, Node* pre_val, BasicType bt); + void shenandoah_enqueue_barrier(Node* val); void g1_write_barrier_post(Node* store, Node* obj, @@ -939,6 +940,7 @@ private: Node* shenandoah_read_barrier_impl(Node* obj, bool use_ctrl, bool use_mem, bool allow_fromspace); Node* shenandoah_write_barrier_impl(Node* obj); + Node* shenandoah_write_barrier_helper(GraphKit& kit, Node* obj, const TypePtr* adr_type); }; // Helper class to support building of control flow branches. Upon diff --git a/src/hotspot/share/opto/loopTransform.cpp b/src/hotspot/share/opto/loopTransform.cpp --- a/src/hotspot/share/opto/loopTransform.cpp +++ b/src/hotspot/share/opto/loopTransform.cpp @@ -2854,7 +2854,7 @@ if (PrintOpto) { tty->print_cr("should_peel"); } phase->do_peeling(this,old_new); } else if (should_unswitch) { - phase->do_unswitching(this, old_new); + phase->do_unswitching(this, old_new, false); } return true; } @@ -2872,7 +2872,7 @@ // to completely unroll this loop or do loop unswitching. if (cl->is_normal_loop()) { if (should_unswitch) { - phase->do_unswitching(this, old_new); + phase->do_unswitching(this, old_new, false); return true; } bool should_maximally_unroll = policy_maximally_unroll(phase); @@ -3000,7 +3000,7 @@ return false; } } else if (policy_unswitching(phase)) { - phase->do_unswitching(this, old_new); + phase->do_unswitching(this, old_new, false); } } diff --git a/src/hotspot/share/opto/loopUnswitch.cpp b/src/hotspot/share/opto/loopUnswitch.cpp --- a/src/hotspot/share/opto/loopUnswitch.cpp +++ b/src/hotspot/share/opto/loopUnswitch.cpp @@ -76,12 +76,12 @@ if (head->unswitch_count() + 1 > head->unswitch_max()) { return false; } - return phase->find_unswitching_candidate(this) != NULL; + return phase->find_unswitching_candidate(this, false) != NULL; } //------------------------------find_unswitching_candidate----------------------------- // Find candidate "if" for unswitching -IfNode* PhaseIdealLoop::find_unswitching_candidate(const IdealLoopTree *loop) const { +IfNode* PhaseIdealLoop::find_unswitching_candidate(const IdealLoopTree *loop, bool do_evac) const { // Find first invariant test that doesn't exit the loop LoopNode *head = loop->_head->as_Loop(); @@ -101,6 +101,7 @@ if (loop->is_invariant(bol) && !loop->is_loop_exit(iff)) { unswitch_iff = iff; } else if (ShenandoahWriteBarrierNode::is_evacuation_in_progress_test(iff) && + do_evac && (loop_has_sfpts == -1 || loop_has_sfpts == 0)) { assert(!loop->is_loop_exit(iff), "both branches should be in the loop"); if (loop_has_sfpts == -1) { @@ -132,12 +133,12 @@ // Clone loop with an invariant test (that does not exit) and // insert a clone of the test that selects which version to // execute. -void PhaseIdealLoop::do_unswitching (IdealLoopTree *loop, Node_List &old_new) { +void PhaseIdealLoop::do_unswitching (IdealLoopTree *loop, Node_List &old_new, bool do_evac) { // Find first invariant test that doesn't exit the loop LoopNode *head = loop->_head->as_Loop(); - IfNode* unswitch_iff = find_unswitching_candidate((const IdealLoopTree *)loop); + IfNode* unswitch_iff = find_unswitching_candidate((const IdealLoopTree *)loop, do_evac); if (ShenandoahWriteBarrierNode::is_evacuation_in_progress_test(unswitch_iff)) { ShenandoahWriteBarrierNode::move_evacuation_test_out_of_loop(unswitch_iff, this); } diff --git a/src/hotspot/share/opto/loopnode.hpp b/src/hotspot/share/opto/loopnode.hpp --- a/src/hotspot/share/opto/loopnode.hpp +++ b/src/hotspot/share/opto/loopnode.hpp @@ -1089,10 +1089,10 @@ // Clone loop with an invariant test (that does not exit) and // insert a clone of the test that selects which version to // execute. - void do_unswitching (IdealLoopTree *loop, Node_List &old_new); + void do_unswitching (IdealLoopTree *loop, Node_List &old_new, bool do_evac); // Find candidate "if" for unswitching - IfNode* find_unswitching_candidate(const IdealLoopTree *loop) const; + IfNode* find_unswitching_candidate(const IdealLoopTree *loop, bool do_evac) const; // Range Check Elimination uses this function! // Constrain the main loop iterations so the affine function: diff --git a/src/hotspot/share/opto/shenandoahSupport.cpp b/src/hotspot/share/opto/shenandoahSupport.cpp --- a/src/hotspot/share/opto/shenandoahSupport.cpp +++ b/src/hotspot/share/opto/shenandoahSupport.cpp @@ -595,10 +595,12 @@ Node* ShenandoahWriteBarrierNode::evacuation_in_progress_test_ctrl(Node* iff) { assert(is_evacuation_in_progress_test(iff), "bad input"); Node* c = iff; + if (ShenandoahWBWithMemBar) { do { assert(c->in(0)->is_Proj() && c->in(0)->in(0)->is_MemBar(), "where's the mem bar?"); c = c->in(0)->in(0); } while (c->adr_type() != TypeRawPtr::BOTTOM); + } return c->in(0); } @@ -3480,6 +3482,7 @@ evacuation_in_progress_adr_type, TypeInt::BOOL, MemNode::unordered); phase->register_new_node(evacuation_in_progress, ctrl); + if (ShenandoahWBWithMemBar) { Node* mb = MemBarNode::make(phase->C, Op_MemBarAcquire, Compile::AliasIdxRaw); mb->init_req(TypeFunc::Control, ctrl); mb->init_req(TypeFunc::Memory, raw_mem); @@ -3498,12 +3501,15 @@ wb_mem = new ProjNode(mb,TypeFunc::Memory); phase->register_new_node(wb_mem, mb); + ctrl = ctrl_proj; + } + Node* evacuation_in_progress_cmp = new CmpINode(evacuation_in_progress, phase->igvn().zerocon(T_INT)); - phase->register_new_node(evacuation_in_progress_cmp, ctrl_proj); + phase->register_new_node(evacuation_in_progress_cmp, ctrl); Node* evacuation_in_progress_test = new BoolNode(evacuation_in_progress_cmp, BoolTest::ne); - phase->register_new_node(evacuation_in_progress_test, ctrl_proj); - evacuation_iff = new IfNode(ctrl_proj, evacuation_in_progress_test, PROB_UNLIKELY(0.999), COUNT_UNKNOWN); - phase->register_control(evacuation_iff, loop, ctrl_proj); + phase->register_new_node(evacuation_in_progress_test, ctrl); + evacuation_iff = new IfNode(ctrl, evacuation_in_progress_test, PROB_UNLIKELY(0.999), COUNT_UNKNOWN); + phase->register_control(evacuation_iff, loop, ctrl); assert(is_evacuation_in_progress_test(evacuation_iff), "inconsistent"); @@ -3916,6 +3922,7 @@ // move test and its mem barriers out of the loop assert(is_evacuation_in_progress_test(iff), "inconsistent"); + if (ShenandoahWBWithMemBar) { IdealLoopTree *loop = phase->get_loop(iff); Node* loop_head = loop->_head; Node* entry_c = loop_head->in(LoopNode::EntryControl); @@ -3990,9 +3997,26 @@ } assert(phase->is_dominator(phase->get_ctrl(load->in(MemNode::Address)), entry_c), "address not out of loop?"); + } else { + IdealLoopTree *loop = phase->get_loop(iff); + Node* loop_head = loop->_head; + Node* entry_c = loop_head->in(LoopNode::EntryControl); + + Node* load = iff->in(1)->in(1)->in(1); + assert(load->Opcode() == Op_LoadUB, "inconsistent"); + Node* mem_ctrl = NULL; + Node* mem = dom_mem(load->in(MemNode::Memory), loop_head, Compile::AliasIdxRaw, mem_ctrl, phase); + phase->igvn().replace_input_of(load, MemNode::Memory, mem); + phase->igvn().replace_input_of(load, 0, entry_c); + phase->set_ctrl_and_loop(load, entry_c); + } + } void ShenandoahWriteBarrierNode::backtoback_evacs(IfNode* iff, IfNode* dom_if, PhaseIdealLoop* phase) { + if (!ShenandoahWBWithMemBar) { + return; + } // move all mem barriers from this evac test to the dominating one, // removing duplicates in the process IdealLoopTree *loop = phase->get_loop(dom_if); @@ -4109,14 +4133,14 @@ if ((!head->is_CountedLoop() || head->as_CountedLoop()->is_main_loop() || head->as_CountedLoop()->is_normal_loop()) && !seen.test_set(head->_idx) && loop->policy_unswitching(phase)) { - IfNode* iff = phase->find_unswitching_candidate(loop); + IfNode* iff = phase->find_unswitching_candidate(loop, true); if (iff != NULL && is_evacuation_in_progress_test(iff)) { if (head->is_strip_mined()) { head->verify_strip_mined(0); head->clear_strip_mined(); head->in(LoopNode::EntryControl)->as_Loop()->clear_strip_mined(); } - phase->do_unswitching(loop, old_new); + phase->do_unswitching(loop, old_new, true); } } } diff --git a/src/hotspot/share/runtime/sharedRuntime.cpp b/src/hotspot/share/runtime/sharedRuntime.cpp --- a/src/hotspot/share/runtime/sharedRuntime.cpp +++ b/src/hotspot/share/runtime/sharedRuntime.cpp @@ -36,6 +36,7 @@ #include "compiler/compileBroker.hpp" #include "compiler/disassembler.hpp" #include "gc/shared/gcLocker.inline.hpp" +#include "gc/shenandoah/shenandoahBarrierSet.hpp" #include "interpreter/interpreter.hpp" #include "interpreter/interpreterRuntime.hpp" #include "logging/log.hpp" @@ -211,6 +212,8 @@ } assert(oopDesc::is_oop(orig, true /* ignore mark word */), "Error"); // store the original value that was in the field reference +if (UseShenandoahGC) { ShenandoahBarrierSet::enqueue(orig); } +return; thread->satb_mark_queue().enqueue(orig); JRT_END diff --git a/src/hotspot/share/runtime/thread.cpp b/src/hotspot/share/runtime/thread.cpp --- a/src/hotspot/share/runtime/thread.cpp +++ b/src/hotspot/share/runtime/thread.cpp @@ -1909,7 +1909,7 @@ // from the list of active threads. We must do this after any deferred // card marks have been flushed (above) so that any entries that are // added to the thread's dirty card queue as a result are not lost. - if (UseG1GC || (UseShenandoahGC && (ShenandoahSATBBarrier || ShenandoahConditionalSATBBarrier || ShenandoahKeepAliveBarrier))) { + if (UseG1GC || (UseShenandoahGC && (ShenandoahSATBBarrier || ShenandoahConditionalSATBBarrier || ShenandoahKeepAliveBarrier || ShenandoahStoreValEnqueueBarrier))) { flush_barrier_queues(); } if (UseShenandoahGC && UseTLAB && gclab().is_initialized()) { @@ -1994,7 +1994,7 @@ } #if INCLUDE_ALL_GCS - if (UseG1GC || (UseShenandoahGC && (ShenandoahSATBBarrier || ShenandoahConditionalSATBBarrier || ShenandoahKeepAliveBarrier))) { + if (UseG1GC || (UseShenandoahGC && (ShenandoahSATBBarrier || ShenandoahConditionalSATBBarrier || ShenandoahKeepAliveBarrier || ShenandoahStoreValEnqueueBarrier))) { flush_barrier_queues(); } if (UseShenandoahGC && UseTLAB && gclab().is_initialized()) { diff --git a/src/hotspot/share/runtime/vm_operations.hpp b/src/hotspot/share/runtime/vm_operations.hpp --- a/src/hotspot/share/runtime/vm_operations.hpp +++ b/src/hotspot/share/runtime/vm_operations.hpp @@ -102,6 +102,8 @@ template(ShenandoahVerifyHeapAfterEvacuation) \ template(ShenandoahInitPartialGC) \ template(ShenandoahFinalPartialGC) \ + template(ShenandoahInitTraversalGC) \ + template(ShenandoahFinalTraversalGC) \ template(ShenandoahInitUpdateRefs) \ template(ShenandoahFinalUpdateRefs) \ template(Exit) \