--- old/src/cpu/aarch64/vm/sharedRuntime_aarch64.cpp 2019-11-11 16:09:40.266343806 +0100 +++ new/src/cpu/aarch64/vm/sharedRuntime_aarch64.cpp 2019-11-11 16:09:40.176343811 +0100 @@ -1944,7 +1944,7 @@ __ ldr(r0, Address(r0, -JNIHandles::weak_tag_value)); __ verify_oop(r0); #if INCLUDE_ALL_GCS - if (UseG1GC || (UseShenandoahGC && ShenandoahSATBBarrier)) { + if (UseG1GC || (UseShenandoahGC && ShenandoahKeepAliveBarrier)) { __ g1_write_barrier_pre(noreg /* obj */, r0 /* pre_val */, rthread /* thread */, --- old/src/cpu/aarch64/vm/shenandoahBarrierSetAssembler_aarch64.cpp 2019-11-11 16:09:40.824343775 +0100 +++ new/src/cpu/aarch64/vm/shenandoahBarrierSetAssembler_aarch64.cpp 2019-11-11 16:09:40.737343780 +0100 @@ -120,6 +120,21 @@ __ leave(); } +void ShenandoahBarrierSetAssembler::storeval_barrier(MacroAssembler* masm, Register dst, Register tmp) { + if (ShenandoahStoreValEnqueueBarrier) { + // Save possibly live regs. + RegSet live_regs = RegSet::range(r0, r4) - dst; + __ push(live_regs, sp); + __ strd(v0, __ pre(sp, 2 * -wordSize)); + + __ g1_write_barrier_pre(noreg, dst, rthread, tmp, true, false); + + // Restore possibly live regs. + __ ldrd(v0, __ post(sp, 2 * wordSize)); + __ pop(live_regs, sp); + } +} + void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler* masm, Register dst) { if (ShenandoahLoadRefBarrier) { Label is_null; --- old/src/cpu/aarch64/vm/shenandoahBarrierSetAssembler_aarch64.hpp 2019-11-11 16:09:41.349343746 +0100 +++ new/src/cpu/aarch64/vm/shenandoahBarrierSetAssembler_aarch64.hpp 2019-11-11 16:09:41.263343751 +0100 @@ -44,6 +44,8 @@ public: static ShenandoahBarrierSetAssembler* bsasm(); + void storeval_barrier(MacroAssembler* masm, Register dst, Register tmp); + #ifdef COMPILER1 void gen_load_reference_barrier_stub(LIR_Assembler* ce, ShenandoahLoadReferenceBarrierStub* stub); #endif --- old/src/cpu/aarch64/vm/templateInterpreter_aarch64.cpp 2019-11-11 16:09:41.866343717 +0100 +++ new/src/cpu/aarch64/vm/templateInterpreter_aarch64.cpp 2019-11-11 16:09:41.779343722 +0100 @@ -693,7 +693,7 @@ const int referent_offset = java_lang_ref_Reference::referent_offset; guarantee(referent_offset > 0, "referent offset not initialized"); - if (UseG1GC || UseShenandoahGC) { + if (UseG1GC || (UseShenandoahGC && ShenandoahKeepAliveBarrier)) { Label slow_path; const Register local_0 = c_rarg0; // Check if local 0 != NULL @@ -1187,7 +1187,7 @@ // Resolve jweak. __ ldr(r0, Address(r0, -JNIHandles::weak_tag_value)); #if INCLUDE_ALL_GCS - if (UseG1GC || (UseShenandoahGC && ShenandoahSATBBarrier)) { + if (UseG1GC || (UseShenandoahGC && ShenandoahKeepAliveBarrier)) { __ enter(); // Barrier may call runtime. __ g1_write_barrier_pre(noreg /* obj */, r0 /* pre_val */, --- old/src/cpu/aarch64/vm/templateTable_aarch64.cpp 2019-11-11 16:09:42.417343687 +0100 +++ new/src/cpu/aarch64/vm/templateTable_aarch64.cpp 2019-11-11 16:09:42.327343692 +0100 @@ -38,6 +38,9 @@ #include "runtime/sharedRuntime.hpp" #include "runtime/stubRoutines.hpp" #include "runtime/synchronizer.hpp" +#if INCLUDE_ALL_GCS +#include "shenandoahBarrierSetAssembler_aarch64.hpp" +#endif #ifndef CC_INTERP @@ -152,7 +155,6 @@ #if INCLUDE_ALL_GCS case BarrierSet::G1SATBCT: case BarrierSet::G1SATBCTLogging: - case BarrierSet::ShenandoahBarrierSet: { // flatten object address if needed if (obj.index() == noreg && obj.offset() == 0) { @@ -188,6 +190,35 @@ } } + break; + case BarrierSet::ShenandoahBarrierSet: + { + // flatten object address if needed + if (obj.index() == noreg && obj.offset() == 0) { + if (obj.base() != r3) { + __ mov(r3, obj.base()); + } + } else { + __ lea(r3, obj); + } + if (ShenandoahSATBBarrier) { + __ g1_write_barrier_pre(r3 /* obj */, + r1 /* pre_val */, + rthread /* thread */, + r10 /* tmp */, + val != noreg /* tosca_live */, + false /* expand_call */); + } + if (val == noreg) { + __ store_heap_oop_null(Address(r3, 0)); + } else { + if (ShenandoahStoreValEnqueueBarrier) { + ShenandoahBarrierSetAssembler::bsasm()->storeval_barrier(_masm, val, r10); + } + __ store_heap_oop(Address(r3, 0), val); + } + + } break; #endif // INCLUDE_ALL_GCS case BarrierSet::CardTableModRef: --- old/src/cpu/x86/vm/macroAssembler_x86.cpp 2019-11-11 16:09:42.986343656 +0100 +++ new/src/cpu/x86/vm/macroAssembler_x86.cpp 2019-11-11 16:09:42.897343661 +0100 @@ -4195,7 +4195,7 @@ if (UseShenandoahGC) { Address gc_state(thread, in_bytes(JavaThread::gc_state_offset())); - testb(gc_state, ShenandoahHeap::MARKING); + testb(gc_state, ShenandoahHeap::MARKING | ShenandoahHeap::TRAVERSAL); jcc(Assembler::zero, done); } else { assert(UseG1GC, "Should be"); --- old/src/cpu/x86/vm/shenandoahBarrierSetAssembler_x86.cpp 2019-11-11 16:09:43.578343623 +0100 +++ new/src/cpu/x86/vm/shenandoahBarrierSetAssembler_x86.cpp 2019-11-11 16:09:43.490343628 +0100 @@ -166,6 +166,47 @@ #endif } +void ShenandoahBarrierSetAssembler::storeval_barrier(MacroAssembler* masm, Register dst, Register tmp) { + if (ShenandoahStoreValEnqueueBarrier) { + storeval_barrier_impl(masm, dst, tmp); + } +} + +void ShenandoahBarrierSetAssembler::storeval_barrier_impl(MacroAssembler* masm, Register dst, Register tmp) { + assert(ShenandoahStoreValEnqueueBarrier, "should be enabled"); + + if (dst == noreg) return; + + if (ShenandoahStoreValEnqueueBarrier) { + // The set of registers to be saved+restored is the same as in the write-barrier above. + // Those are the commonly used registers in the interpreter. + __ pusha(); + // __ push_callee_saved_registers(); + __ subptr(rsp, 2 * Interpreter::stackElementSize); + __ movdbl(Address(rsp, 0), xmm0); + +#ifdef _LP64 + Register thread = r15_thread; +#else + Register thread = rcx; + if (thread == dst || thread == tmp) { + thread = rdi; + } + if (thread == dst || thread == tmp) { + thread = rbx; + } + __ get_thread(thread); +#endif + assert_different_registers(dst, tmp, thread); + + __ g1_write_barrier_pre(noreg, dst, thread, tmp, true, false); + __ movdbl(xmm0, Address(rsp, 0)); + __ addptr(rsp, 2 * Interpreter::stackElementSize); + //__ pop_callee_saved_registers(); + __ popa(); + } +} + void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler* masm, Register dst) { if (ShenandoahLoadRefBarrier) { Label done; --- old/src/cpu/x86/vm/shenandoahBarrierSetAssembler_x86.hpp 2019-11-11 16:09:44.102343594 +0100 +++ new/src/cpu/x86/vm/shenandoahBarrierSetAssembler_x86.hpp 2019-11-11 16:09:44.015343599 +0100 @@ -41,9 +41,12 @@ void load_reference_barrier_not_null(MacroAssembler* masm, Register dst); + void storeval_barrier_impl(MacroAssembler* masm, Register dst, Register tmp); + public: static ShenandoahBarrierSetAssembler* bsasm(); + void storeval_barrier(MacroAssembler* masm, Register dst, Register tmp); #ifdef COMPILER1 void gen_load_reference_barrier_stub(LIR_Assembler* ce, ShenandoahLoadReferenceBarrierStub* stub); #endif --- old/src/cpu/x86/vm/templateInterpreter_x86_32.cpp 2019-11-11 16:09:44.615343566 +0100 +++ new/src/cpu/x86/vm/templateInterpreter_x86_32.cpp 2019-11-11 16:09:44.533343570 +0100 @@ -863,6 +863,7 @@ // Generate the G1 pre-barrier code to log the value of // the referent field in an SATB buffer. + if (!UseShenandoahGC || ShenandoahKeepAliveBarrier) { __ get_thread(rcx); __ g1_write_barrier_pre(noreg /* obj */, rax /* pre_val */, @@ -870,6 +871,7 @@ rbx /* tmp */, true /* tosca_save */, true /* expand_call */); + } // _areturn __ pop(rsi); // get sender sp --- old/src/cpu/x86/vm/templateInterpreter_x86_64.cpp 2019-11-11 16:09:45.147343537 +0100 +++ new/src/cpu/x86/vm/templateInterpreter_x86_64.cpp 2019-11-11 16:09:45.064343541 +0100 @@ -815,12 +815,14 @@ // Generate the G1 pre-barrier code to log the value of // the referent field in an SATB buffer. + if (!UseShenandoahGC || ShenandoahKeepAliveBarrier) { __ g1_write_barrier_pre(noreg /* obj */, rax /* pre_val */, r15_thread /* thread */, rbx /* tmp */, true /* tosca_live */, true /* expand_call */); + } // _areturn __ pop(rdi); // get return address --- old/src/cpu/x86/vm/templateTable_x86_32.cpp 2019-11-11 16:09:45.688343507 +0100 +++ new/src/cpu/x86/vm/templateTable_x86_32.cpp 2019-11-11 16:09:45.599343512 +0100 @@ -36,6 +36,9 @@ #include "runtime/stubRoutines.hpp" #include "runtime/synchronizer.hpp" #include "utilities/macros.hpp" +#if INCLUDE_ALL_GCS +#include "shenandoahBarrierSetAssembler_x86.hpp" +#endif #ifndef CC_INTERP #define __ _masm-> @@ -129,7 +132,6 @@ #if INCLUDE_ALL_GCS case BarrierSet::G1SATBCT: case BarrierSet::G1SATBCTLogging: - case BarrierSet::ShenandoahBarrierSet: { // flatten object address if needed // We do it regardless of precise because we need the registers @@ -164,6 +166,41 @@ } __ restore_bcp(); + } + break; + case BarrierSet::ShenandoahBarrierSet: + { + // flatten object address if needed + // We do it regardless of precise because we need the registers + if (obj.index() == noreg && obj.disp() == 0) { + if (obj.base() != rdx) { + __ movl(rdx, obj.base()); + } + } else { + __ leal(rdx, obj); + } + __ get_thread(rcx); + __ save_bcp(); + if (ShenandoahSATBBarrier) { + __ g1_write_barrier_pre(rdx /* obj */, + rbx /* pre_val */, + rcx /* thread */, + rsi /* tmp */, + val != noreg /* tosca_live */, + false /* expand_call */); + } + + // Do the actual store + // noreg means NULL + if (val == noreg) { + __ movptr(Address(rdx, 0), NULL_WORD); + // No post barrier for NULL + } else { + ShenandoahBarrierSetAssembler::bsasm()->storeval_barrier(_masm, val, rsi); + __ movl(Address(rdx, 0), val); + } + __ restore_bcp(); + } break; #endif // INCLUDE_ALL_GCS --- old/src/cpu/x86/vm/templateTable_x86_64.cpp 2019-11-11 16:09:46.253343476 +0100 +++ new/src/cpu/x86/vm/templateTable_x86_64.cpp 2019-11-11 16:09:46.170343480 +0100 @@ -36,6 +36,9 @@ #include "runtime/stubRoutines.hpp" #include "runtime/synchronizer.hpp" #include "utilities/macros.hpp" +#if INCLUDE_ALL_GCS +#include "shenandoahBarrierSetAssembler_x86.hpp" +#endif #ifndef CC_INTERP @@ -140,7 +143,6 @@ #if INCLUDE_ALL_GCS case BarrierSet::G1SATBCT: case BarrierSet::G1SATBCTLogging: - case BarrierSet::ShenandoahBarrierSet: { // flatten object address if needed if (obj.index() == noreg && obj.disp() == 0) { @@ -174,6 +176,32 @@ } } break; + case BarrierSet::ShenandoahBarrierSet: + { + // flatten object address if needed + if (obj.index() == noreg && obj.disp() == 0) { + if (obj.base() != rdx) { + __ movq(rdx, obj.base()); + } + } else { + __ leaq(rdx, obj); + } + if (ShenandoahSATBBarrier) { + __ g1_write_barrier_pre(rdx /* obj */, + rbx /* pre_val */, + r15_thread /* thread */, + r8 /* tmp */, + val != noreg /* tosca_live */, + false /* expand_call */); + } + if (val == noreg) { + __ store_heap_oop_null(Address(rdx, 0)); + } else { + ShenandoahBarrierSetAssembler::bsasm()->storeval_barrier(_masm, val, r8); + __ store_heap_oop(Address(rdx, 0), val); + } + } + break; #endif // INCLUDE_ALL_GCS case BarrierSet::CardTableModRef: case BarrierSet::CardTableExtension: --- old/src/share/vm/c1/c1_LIRGenerator.cpp 2019-11-11 16:09:46.815343445 +0100 +++ new/src/share/vm/c1/c1_LIRGenerator.cpp 2019-11-11 16:09:46.729343449 +0100 @@ -1430,9 +1430,13 @@ #if INCLUDE_ALL_GCS case BarrierSet::G1SATBCT: case BarrierSet::G1SATBCTLogging: - case BarrierSet::ShenandoahBarrierSet: G1SATBCardTableModRef_pre_barrier(addr_opr, pre_val, do_load, patch, info); break; + case BarrierSet::ShenandoahBarrierSet: + if (ShenandoahSATBBarrier) { + G1SATBCardTableModRef_pre_barrier(addr_opr, pre_val, do_load, patch, info); + } + break; #endif // INCLUDE_ALL_GCS case BarrierSet::CardTableModRef: case BarrierSet::CardTableExtension: @@ -1456,6 +1460,7 @@ G1SATBCardTableModRef_post_barrier(addr, new_val); break; case BarrierSet::ShenandoahBarrierSet: + ShenandoahBarrierSetC1::bsc1()->storeval_barrier(this, new_val, NULL, false); break; #endif // INCLUDE_ALL_GCS case BarrierSet::CardTableModRef: --- old/src/share/vm/classfile/symbolTable.cpp 2019-11-11 16:09:47.377343414 +0100 +++ new/src/share/vm/classfile/symbolTable.cpp 2019-11-11 16:09:47.292343418 +0100 @@ -721,7 +721,7 @@ // considered dead. The SATB part of G1 needs to get notified about this // potential resurrection, otherwise the marking might not find the object. #if INCLUDE_ALL_GCS - if ((UseG1GC || (UseShenandoahGC && ShenandoahSATBBarrier)) && string != NULL) { + if ((UseG1GC || (UseShenandoahGC && ShenandoahKeepAliveBarrier)) && string != NULL) { G1SATBCardTableModRefBS::enqueue(string); } #endif --- old/src/share/vm/gc_implementation/shenandoah/heuristics/shenandoahAggressiveHeuristics.cpp 2019-11-11 16:09:47.912343384 +0100 +++ new/src/share/vm/gc_implementation/shenandoah/heuristics/shenandoahAggressiveHeuristics.cpp 2019-11-11 16:09:47.827343389 +0100 @@ -48,6 +48,7 @@ // Final configuration checks SHENANDOAH_CHECK_FLAG_SET(ShenandoahLoadRefBarrier); SHENANDOAH_CHECK_FLAG_SET(ShenandoahSATBBarrier); + SHENANDOAH_CHECK_FLAG_SET(ShenandoahKeepAliveBarrier); SHENANDOAH_CHECK_FLAG_SET(ShenandoahCASBarrier); SHENANDOAH_CHECK_FLAG_SET(ShenandoahCloneBarrier); } --- old/src/share/vm/gc_implementation/shenandoah/heuristics/shenandoahCompactHeuristics.cpp 2019-11-11 16:09:48.435343355 +0100 +++ new/src/share/vm/gc_implementation/shenandoah/heuristics/shenandoahCompactHeuristics.cpp 2019-11-11 16:09:48.349343360 +0100 @@ -43,6 +43,7 @@ // Final configuration checks SHENANDOAH_CHECK_FLAG_SET(ShenandoahLoadRefBarrier); SHENANDOAH_CHECK_FLAG_SET(ShenandoahSATBBarrier); + SHENANDOAH_CHECK_FLAG_SET(ShenandoahKeepAliveBarrier); SHENANDOAH_CHECK_FLAG_SET(ShenandoahCASBarrier); SHENANDOAH_CHECK_FLAG_SET(ShenandoahCloneBarrier); } --- old/src/share/vm/gc_implementation/shenandoah/heuristics/shenandoahStaticHeuristics.cpp 2019-11-11 16:09:48.965343326 +0100 +++ new/src/share/vm/gc_implementation/shenandoah/heuristics/shenandoahStaticHeuristics.cpp 2019-11-11 16:09:48.879343331 +0100 @@ -41,6 +41,7 @@ // Final configuration checks SHENANDOAH_CHECK_FLAG_SET(ShenandoahLoadRefBarrier); SHENANDOAH_CHECK_FLAG_SET(ShenandoahSATBBarrier); + SHENANDOAH_CHECK_FLAG_SET(ShenandoahKeepAliveBarrier); SHENANDOAH_CHECK_FLAG_SET(ShenandoahCASBarrier); SHENANDOAH_CHECK_FLAG_SET(ShenandoahCloneBarrier); } --- old/src/share/vm/gc_implementation/shenandoah/shenandoahAsserts.cpp 2019-11-11 16:09:49.493343297 +0100 +++ new/src/share/vm/gc_implementation/shenandoah/shenandoahAsserts.cpp 2019-11-11 16:09:49.408343302 +0100 @@ -26,7 +26,9 @@ #include "gc_implementation/shenandoah/shenandoahAsserts.hpp" #include "gc_implementation/shenandoah/shenandoahForwarding.hpp" #include "gc_implementation/shenandoah/shenandoahHeap.inline.hpp" +#include "gc_implementation/shenandoah/shenandoahHeapRegionSet.inline.hpp" #include "gc_implementation/shenandoah/shenandoahMarkingContext.inline.hpp" +#include "gc_implementation/shenandoah/shenandoahTraversalGC.hpp" #include "gc_implementation/shenandoah/shenandoahUtils.hpp" #include "memory/resourceArea.hpp" @@ -65,6 +67,9 @@ msg.append(" " PTR_FORMAT " - klass " PTR_FORMAT " %s\n", p2i(obj), p2i(obj->klass()), obj->klass()->external_name()); msg.append(" %3s allocated after mark start\n", ctx->allocated_after_mark_start((HeapWord *) obj) ? "" : "not"); + if (heap->traversal_gc() != NULL) { + msg.append(" %3s in traversal set\n", heap->traversal_gc()->traversal_set()->is_in((HeapWord*) obj) ? "" : "not"); + } msg.append(" %3s marked \n", ctx->is_marked(obj) ? "" : "not"); msg.append(" %3s in collection set\n", heap->in_collection_set(obj) ? "" : "not"); msg.append(" mark:%s\n", mw_ss.as_string()); --- old/src/share/vm/gc_implementation/shenandoah/shenandoahBarrierSet.cpp 2019-11-11 16:09:50.022343268 +0100 +++ new/src/share/vm/gc_implementation/shenandoah/shenandoahBarrierSet.cpp 2019-11-11 16:09:49.936343273 +0100 @@ -46,6 +46,7 @@ #include "shenandoahBarrierSetAssembler_stub.hpp" #endif +template class ShenandoahUpdateRefsForOopClosure: public ExtendedOopClosure { private: ShenandoahHeap* _heap; @@ -53,14 +54,23 @@ template inline void do_oop_work(T* p) { - _heap->maybe_update_with_forwarded(p); + oop o; + if (STOREVAL_EVAC_BARRIER) { + o = _heap->evac_update_with_forwarded(p); + if (!oopDesc::is_null(o)) { + _bs->enqueue(o); + } + } else { + _heap->maybe_update_with_forwarded(p); + } } public: ShenandoahUpdateRefsForOopClosure() : _heap(ShenandoahHeap::heap()), _bs(ShenandoahBarrierSet::barrier_set()) { assert(UseShenandoahGC && ShenandoahCloneBarrier, "should be enabled"); } - void do_oop(oop* p) { do_oop_work(p); } - void do_oop(narrowOop* p) { do_oop_work(p); } + + virtual void do_oop(oop* p) { do_oop_work(p); } + virtual void do_oop(narrowOop* p) { do_oop_work(p); } }; ShenandoahBarrierSet::ShenandoahBarrierSet(ShenandoahHeap* heap) : @@ -150,10 +160,10 @@ // return *v; } -template +template void ShenandoahBarrierSet::write_ref_array_loop(HeapWord* start, size_t count) { - assert(UseShenandoahGC && ShenandoahCloneBarrier, "Should be enabled"); - ShenandoahUpdateRefsForOopClosure cl; + assert(UseShenandoahGC && ShenandoahCloneBarrier, "should be enabled"); + ShenandoahUpdateRefsForOopClosure cl; T* dst = (T*) start; for (size_t i = 0; i < count; i++) { cl.do_oop(dst++); @@ -165,11 +175,19 @@ if (!ShenandoahCloneBarrier) return; if (!need_update_refs_barrier()) return; - ShenandoahEvacOOMScope oom_evac_scope; - if (UseCompressedOops) { - write_ref_array_loop(start, count); + if (_heap->is_concurrent_traversal_in_progress()) { + ShenandoahEvacOOMScope oom_evac_scope; + if (UseCompressedOops) { + write_ref_array_loop(start, count); + } else { + write_ref_array_loop(start, count); + } } else { - write_ref_array_loop(start, count); + if (UseCompressedOops) { + write_ref_array_loop(start, count); + } else { + write_ref_array_loop(start, count); + } } } @@ -202,28 +220,25 @@ } template -void ShenandoahBarrierSet::write_ref_field_pre_static(T* field, oop newVal) { - T heap_oop = oopDesc::load_heap_oop(field); - - shenandoah_assert_not_in_cset_loc_except(field, ShenandoahHeap::heap()->cancelled_gc()); - - if (!oopDesc::is_null(heap_oop)) { - ShenandoahBarrierSet::barrier_set()->enqueue(oopDesc::decode_heap_oop(heap_oop)); - } -} - -template inline void ShenandoahBarrierSet::inline_write_ref_field_pre(T* field, oop newVal) { - write_ref_field_pre_static(field, newVal); + newVal = load_reference_barrier(newVal); + storeval_barrier(newVal); + if (ShenandoahSATBBarrier) { + T heap_oop = oopDesc::load_heap_oop(field); + shenandoah_assert_not_in_cset_loc_except(field, ShenandoahHeap::heap()->cancelled_gc()); + if (!oopDesc::is_null(heap_oop)) { + ShenandoahBarrierSet::barrier_set()->enqueue(oopDesc::decode_heap_oop(heap_oop)); + } + } } // These are the more general virtual versions. void ShenandoahBarrierSet::write_ref_field_pre_work(oop* field, oop new_val) { - write_ref_field_pre_static(field, new_val); + inline_write_ref_field_pre(field, new_val); } void ShenandoahBarrierSet::write_ref_field_pre_work(narrowOop* field, oop new_val) { - write_ref_field_pre_static(field, new_val); + inline_write_ref_field_pre(field, new_val); } void ShenandoahBarrierSet::write_ref_field_work(void* v, oop o, bool release) { @@ -242,11 +257,16 @@ // it would be NULL in any case. But we *are* interested in any oop* // that potentially need to be updated. - ShenandoahEvacOOMScope oom_evac_scope; oop obj = oop(mr.start()); shenandoah_assert_correct(NULL, obj); - ShenandoahUpdateRefsForOopClosure cl; - obj->oop_iterate(&cl); + if (_heap->is_concurrent_traversal_in_progress()) { + ShenandoahEvacOOMScope oom_evac_scope; + ShenandoahUpdateRefsForOopClosure cl; + obj->oop_iterate(&cl); + } else { + ShenandoahUpdateRefsForOopClosure cl; + obj->oop_iterate(&cl); + } } oop ShenandoahBarrierSet::load_reference_barrier_not_null(oop obj) { @@ -269,7 +289,7 @@ oop ShenandoahBarrierSet::load_reference_barrier_mutator(oop obj) { assert(ShenandoahLoadRefBarrier, "should be enabled"); - assert(_heap->is_gc_in_progress_mask(ShenandoahHeap::EVACUATION), "evac should be in progress"); + assert(_heap->is_gc_in_progress_mask(ShenandoahHeap::EVACUATION | ShenandoahHeap::TRAVERSAL), "evac should be in progress"); shenandoah_assert_in_cset(NULL, obj); oop fwd = resolve_forwarded_not_null(obj); @@ -289,7 +309,10 @@ size_t max = ShenandoahEvacAssist; if (max > 0) { - ShenandoahMarkingContext* ctx = _heap->complete_marking_context(); + // Traversal is special: it uses incomplete marking context, because it coalesces evac with mark. + // Other code uses complete marking context, because evac happens after the mark. + ShenandoahMarkingContext* ctx = _heap->is_concurrent_traversal_in_progress() ? + _heap->marking_context() : _heap->complete_marking_context(); ShenandoahHeapRegion* r = _heap->heap_region_containing(obj); assert(r->is_cset(), "sanity"); @@ -314,7 +337,7 @@ oop ShenandoahBarrierSet::load_reference_barrier_impl(oop obj) { assert(ShenandoahLoadRefBarrier, "should be enabled"); if (!oopDesc::is_null(obj)) { - bool evac_in_progress = _heap->is_gc_in_progress_mask(ShenandoahHeap::EVACUATION); + bool evac_in_progress = _heap->is_gc_in_progress_mask(ShenandoahHeap::EVACUATION | ShenandoahHeap::TRAVERSAL); oop fwd = resolve_forwarded_not_null(obj); if (evac_in_progress && _heap->in_collection_set(obj) && @@ -334,7 +357,21 @@ } } +void ShenandoahBarrierSet::storeval_barrier(oop obj) { + if (ShenandoahStoreValEnqueueBarrier && !oopDesc::is_null(obj) && _heap->is_concurrent_traversal_in_progress()) { + enqueue(obj); + } +} + +void ShenandoahBarrierSet::keep_alive_barrier(oop obj) { + if (ShenandoahKeepAliveBarrier && _heap->is_concurrent_mark_in_progress()) { + enqueue(obj); + } +} + void ShenandoahBarrierSet::enqueue(oop obj) { + shenandoah_assert_not_forwarded_if(NULL, obj, _heap->is_concurrent_traversal_in_progress()); + // Filter marked objects before hitting the SATB queues. The same predicate would // be used by SATBMQ::filter to eliminate already marked objects downstream, but // filtering here helps to avoid wasteful SATB queueing work to begin with. @@ -371,5 +408,8 @@ if (ShenandoahSATBBarrier && success && result != NULL) { enqueue(result); } + if (new_value != NULL) { + storeval_barrier(new_value); + } return result; } --- old/src/share/vm/gc_implementation/shenandoah/shenandoahBarrierSet.hpp 2019-11-11 16:09:50.556343239 +0100 +++ new/src/share/vm/gc_implementation/shenandoah/shenandoahBarrierSet.hpp 2019-11-11 16:09:50.469343243 +0100 @@ -91,8 +91,6 @@ void write_ref_array_pre(narrowOop* dst, int count, bool dest_uninitialized); - template static void write_ref_field_pre_static(T* field, oop newVal); - // We export this to make it available in cases where the static // type of the barrier set is known. Note that it is non-virtual. template inline void inline_write_ref_field_pre(T* field, oop newVal); @@ -108,10 +106,8 @@ static inline oop resolve_forwarded_not_null(oop p); static inline oop resolve_forwarded(oop p); - static oopDesc* write_barrier_IRT(oopDesc* src); - static oopDesc* write_barrier_JRT(oopDesc* src); - - oop write_barrier_mutator(oop obj); + void storeval_barrier(oop obj); + void keep_alive_barrier(oop obj); oop load_reference_barrier(oop obj); oop load_reference_barrier_mutator(oop obj); @@ -124,7 +120,7 @@ private: inline bool need_update_refs_barrier(); - template + template void write_ref_array_loop(HeapWord* start, size_t count); oop load_reference_barrier_impl(oop obj); --- old/src/share/vm/gc_implementation/shenandoah/shenandoahBarrierSetC1.cpp 2019-11-11 16:09:51.083343209 +0100 +++ new/src/share/vm/gc_implementation/shenandoah/shenandoahBarrierSetC1.cpp 2019-11-11 16:09:50.994343214 +0100 @@ -112,3 +112,11 @@ } return obj; } + +LIR_Opr ShenandoahBarrierSetC1::storeval_barrier(LIRGenerator* gen, LIR_Opr obj, CodeEmitInfo* info, bool patch) { + if (ShenandoahStoreValEnqueueBarrier) { + obj = ensure_in_register(gen, obj); + gen->G1SATBCardTableModRef_pre_barrier(LIR_OprFact::illegalOpr, obj, false, false, NULL); + } + return obj; +} --- old/src/share/vm/gc_implementation/shenandoah/shenandoahBarrierSetC1.hpp 2019-11-11 16:09:51.615343180 +0100 +++ new/src/share/vm/gc_implementation/shenandoah/shenandoahBarrierSetC1.hpp 2019-11-11 16:09:51.527343185 +0100 @@ -62,10 +62,13 @@ }; class ShenandoahBarrierSetC1 : public CHeapObj{ +private: + CodeBlob* _pre_barrier_c1_runtime_code_blob; public: static ShenandoahBarrierSetC1* bsc1(); LIR_Opr load_reference_barrier(LIRGenerator* gen, LIR_Opr obj, CodeEmitInfo* info, bool need_null_check); + LIR_Opr storeval_barrier(LIRGenerator* gen, LIR_Opr obj, CodeEmitInfo* info, bool patch); private: LIR_Opr load_reference_barrier_impl(LIRGenerator* gen, LIR_Opr obj, CodeEmitInfo* info, bool need_null_check); LIR_Opr ensure_in_register(LIRGenerator* gen, LIR_Opr obj); --- old/src/share/vm/gc_implementation/shenandoah/shenandoahClosures.inline.hpp 2019-11-11 16:09:52.149343151 +0100 +++ new/src/share/vm/gc_implementation/shenandoah/shenandoahClosures.inline.hpp 2019-11-11 16:09:52.061343156 +0100 @@ -38,7 +38,8 @@ } obj = ShenandoahBarrierSet::resolve_forwarded_not_null(obj); shenandoah_assert_not_forwarded_if(NULL, obj, - ShenandoahHeap::heap()->is_concurrent_mark_in_progress()); + (ShenandoahHeap::heap()->is_concurrent_mark_in_progress() || + ShenandoahHeap::heap()->is_concurrent_traversal_in_progress())); return _mark_context->is_marked(obj); } --- old/src/share/vm/gc_implementation/shenandoah/shenandoahConcurrentMark.hpp 2019-11-11 16:09:52.677343122 +0100 +++ new/src/share/vm/gc_implementation/shenandoah/shenandoahConcurrentMark.hpp 2019-11-11 16:09:52.591343126 +0100 @@ -32,6 +32,7 @@ class ShenandoahStrDedupQueue; class ShenandoahConcurrentMark: public CHeapObj { + friend class ShenandoahTraversalGC; private: ShenandoahHeap* _heap; ShenandoahObjToScanQueueSet* _task_queues; --- old/src/share/vm/gc_implementation/shenandoah/shenandoahConcurrentMark.inline.hpp 2019-11-11 16:09:53.206343093 +0100 +++ new/src/share/vm/gc_implementation/shenandoah/shenandoahConcurrentMark.inline.hpp 2019-11-11 16:09:53.119343097 +0100 @@ -40,7 +40,7 @@ void ShenandoahConcurrentMark::do_task(ShenandoahObjToScanQueue* q, T* cl, jushort* live_data, ShenandoahMarkTask* task) { oop obj = task->obj(); - shenandoah_assert_not_forwarded(NULL, obj); + shenandoah_assert_not_forwarded_except(NULL, obj, _heap->is_concurrent_traversal_in_progress() && _heap->cancelled_gc()); shenandoah_assert_marked(NULL, obj); shenandoah_assert_not_in_cset_except(NULL, obj, _heap->cancelled_gc()); --- old/src/share/vm/gc_implementation/shenandoah/shenandoahControlThread.cpp 2019-11-11 16:09:53.726343064 +0100 +++ new/src/share/vm/gc_implementation/shenandoah/shenandoahControlThread.cpp 2019-11-11 16:09:53.643343068 +0100 @@ -33,6 +33,7 @@ #include "gc_implementation/shenandoah/shenandoahHeuristics.hpp" #include "gc_implementation/shenandoah/shenandoahMonitoringSupport.hpp" #include "gc_implementation/shenandoah/shenandoahPhaseTimings.hpp" +#include "gc_implementation/shenandoah/shenandoahTraversalGC.hpp" #include "gc_implementation/shenandoah/shenandoahUtils.hpp" #include "gc_implementation/shenandoah/shenandoahVMOperations.hpp" #include "gc_implementation/shenandoah/shenandoahWorkerPolicy.hpp" @@ -99,8 +100,10 @@ ShenandoahHeap* heap = ShenandoahHeap::heap(); - GCMode default_mode = concurrent_normal; - GCCause::Cause default_cause = GCCause::_shenandoah_concurrent_gc; + GCMode default_mode = heap->is_traversal_mode() ? + concurrent_traversal : concurrent_normal; + GCCause::Cause default_cause = heap->is_traversal_mode() ? + GCCause::_shenandoah_traversal_gc : GCCause::_shenandoah_concurrent_gc; int sleep = ShenandoahControlIntervalMin; double last_shrink_time = os::elapsedTime(); @@ -220,6 +223,9 @@ switch (mode) { case none: break; + case concurrent_traversal: + service_concurrent_traversal_cycle(cause); + break; case concurrent_normal: service_concurrent_normal_cycle(cause); break; @@ -311,6 +317,30 @@ terminate(); } +void ShenandoahControlThread::service_concurrent_traversal_cycle(GCCause::Cause cause) { + ShenandoahGCSession session(cause); + + ShenandoahHeap* heap = ShenandoahHeap::heap(); + TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters()); + + // Reset for upcoming cycle + heap->entry_reset(); + + heap->vmop_entry_init_traversal(); + + if (check_cancellation_or_degen(ShenandoahHeap::_degenerated_traversal)) return; + + heap->entry_traversal(); + if (check_cancellation_or_degen(ShenandoahHeap::_degenerated_traversal)) return; + + heap->vmop_entry_final_traversal(); + + heap->entry_cleanup(); + + heap->heuristics()->record_success_concurrent(); + heap->shenandoah_policy()->record_success_concurrent(); +} + void ShenandoahControlThread::service_concurrent_normal_cycle(GCCause::Cause cause) { // Normal cycle goes via all concurrent phases. If allocation failure (af) happens during // any of the concurrent phases, it first degrades to Degenerated GC and completes GC there. --- old/src/share/vm/gc_implementation/shenandoah/shenandoahControlThread.hpp 2019-11-11 16:09:54.255343035 +0100 +++ new/src/share/vm/gc_implementation/shenandoah/shenandoahControlThread.hpp 2019-11-11 16:09:54.167343040 +0100 @@ -58,6 +58,7 @@ private: typedef enum { none, + concurrent_traversal, concurrent_normal, stw_degenerated, stw_full @@ -96,6 +97,7 @@ void service_concurrent_normal_cycle(GCCause::Cause cause); void service_stw_full_cycle(GCCause::Cause cause); void service_stw_degenerated_cycle(GCCause::Cause cause, ShenandoahHeap::ShenandoahDegenPoint point); + void service_concurrent_traversal_cycle(GCCause::Cause cause); void service_uncommit(double shrink_before); bool try_set_alloc_failure_gc(); --- old/src/share/vm/gc_implementation/shenandoah/shenandoahFreeSet.cpp 2019-11-11 16:09:54.784343006 +0100 +++ new/src/share/vm/gc_implementation/shenandoah/shenandoahFreeSet.cpp 2019-11-11 16:09:54.697343010 +0100 @@ -25,6 +25,7 @@ #include "gc_implementation/shenandoah/shenandoahFreeSet.hpp" #include "gc_implementation/shenandoah/shenandoahHeap.inline.hpp" +#include "gc_implementation/shenandoah/shenandoahTraversalGC.hpp" ShenandoahFreeSet::ShenandoahFreeSet(ShenandoahHeap* heap,size_t max_regions) : _heap(heap), @@ -175,6 +176,15 @@ // Record actual allocation size req.set_actual_size(size); + + if (req.is_gc_alloc() && _heap->is_concurrent_traversal_in_progress()) { + // Traversal needs to traverse through GC allocs. Adjust TAMS to the new top + // so that these allocations appear below TAMS, and thus get traversed. + // See top of shenandoahTraversal.cpp for an explanation. + _heap->marking_context()->capture_top_at_mark_start(r); + _heap->traversal_gc()->traversal_set()->add_region_check_for_duplicates(r); + OrderAccess::fence(); + } } if (result == NULL || has_no_alloc_capacity(r)) { --- old/src/share/vm/gc_implementation/shenandoah/shenandoahHeap.cpp 2019-11-11 16:09:55.319342976 +0100 +++ new/src/share/vm/gc_implementation/shenandoah/shenandoahHeap.cpp 2019-11-11 16:09:55.233342981 +0100 @@ -51,6 +51,7 @@ #include "gc_implementation/shenandoah/shenandoahPassiveMode.hpp" #include "gc_implementation/shenandoah/shenandoahRootProcessor.hpp" #include "gc_implementation/shenandoah/shenandoahTaskqueue.hpp" +#include "gc_implementation/shenandoah/shenandoahTraversalMode.hpp" #include "gc_implementation/shenandoah/shenandoahUtils.hpp" #include "gc_implementation/shenandoah/shenandoahVerifier.hpp" #include "gc_implementation/shenandoah/shenandoahCodeRoots.hpp" @@ -357,6 +358,10 @@ _pacer = NULL; } + _traversal_gc = strcmp(ShenandoahGCMode, "traversal") == 0 ? + new ShenandoahTraversalGC(this, _num_regions) : + NULL; + _control_thread = new ShenandoahControlThread(); log_info(gc, init)("Initialize Shenandoah heap: " SIZE_FORMAT "%s initial, " SIZE_FORMAT "%s min, " SIZE_FORMAT "%s max", @@ -375,7 +380,9 @@ void ShenandoahHeap::initialize_heuristics() { if (ShenandoahGCMode != NULL) { - if (strcmp(ShenandoahGCMode, "normal") == 0) { + if (strcmp(ShenandoahGCMode, "traversal") == 0) { + _gc_mode = new ShenandoahTraversalMode(); + } else if (strcmp(ShenandoahGCMode, "normal") == 0) { _gc_mode = new ShenandoahNormalMode(); } else if (strcmp(ShenandoahGCMode, "passive") == 0) { _gc_mode = new ShenandoahPassiveMode(); @@ -409,6 +416,7 @@ _regions(NULL), _free_set(NULL), _collection_set(NULL), + _traversal_gc(NULL), _update_refs_iterator(this), _bytes_allocated_since_gc_start(0), _max_workers((uint)MAX2(ConcGCThreads, ParallelGCThreads)), @@ -431,6 +439,7 @@ log_info(gc, init)("Reference processing: %s", ParallelRefProcEnabled ? "parallel" : "serial"); _scm = new ShenandoahConcurrentMark(); + _full_gc = new ShenandoahMarkCompact(); _used = 0; @@ -498,6 +507,7 @@ if (is_concurrent_mark_in_progress()) st->print("marking, "); if (is_evacuation_in_progress()) st->print("evacuating, "); if (is_update_refs_in_progress()) st->print("updating refs, "); + if (is_concurrent_traversal_in_progress()) st->print("traversal, "); if (is_degenerated_gc_in_progress()) st->print("degenerated gc, "); if (is_full_gc_in_progress()) st->print("full gc, "); if (is_full_gc_move_in_progress()) st->print("full gc move, "); @@ -1580,6 +1590,18 @@ concurrent_mark()->preclean_weak_refs(); } +void ShenandoahHeap::op_init_traversal() { + traversal_gc()->init_traversal_collection(); +} + +void ShenandoahHeap::op_traversal() { + traversal_gc()->concurrent_traversal_collection(); +} + +void ShenandoahHeap::op_final_traversal() { + traversal_gc()->final_traversal_collection(); +} + void ShenandoahHeap::op_full(GCCause::Cause cause) { ShenandoahMetricsSnapshot metrics; metrics.snap_before(); @@ -1608,6 +1630,24 @@ metrics.snap_before(); switch (point) { + case _degenerated_traversal: + { + // Drop the collection set. Note: this leaves some already forwarded objects + // behind, which may be problematic, see comments for ShenandoahEvacAssist + // workarounds in ShenandoahTraversalHeuristics. + + ShenandoahHeapLocker locker(lock()); + collection_set()->clear_current_index(); + for (size_t i = 0; i < collection_set()->count(); i++) { + ShenandoahHeapRegion* r = collection_set()->next(); + r->make_regular_bypass(); + } + collection_set()->clear(); + } + op_final_traversal(); + op_cleanup(); + return; + // The cases below form the Duff's-like device: it describes the actual GC cycle, // but enters it at different points, depending on which concurrent phase had // degenerated. @@ -1624,6 +1664,13 @@ set_process_references(heuristics()->can_process_references()); set_unload_classes(heuristics()->can_unload_classes()); + if (is_traversal_mode()) { + // Not possible to degenerate from here, upgrade to Full GC right away. + cancel_gc(GCCause::_shenandoah_upgrade_to_full_gc); + op_degenerated_fail(); + return; + } + op_reset(); op_init_mark(); @@ -1753,7 +1800,7 @@ } void ShenandoahHeap::force_satb_flush_all_threads() { - if (!is_concurrent_mark_in_progress()) { + if (!is_concurrent_mark_in_progress() && !is_concurrent_traversal_in_progress()) { // No need to flush SATBs return; } @@ -1787,6 +1834,11 @@ JavaThread::satb_mark_queue_set().set_active_all_threads(in_progress, !in_progress); } +void ShenandoahHeap::set_concurrent_traversal_in_progress(bool in_progress) { + set_gc_state_mask(TRAVERSAL | HAS_FORWARDED | UPDATEREFS, in_progress); + JavaThread::satb_mark_queue_set().set_active_all_threads(in_progress, !in_progress); +} + void ShenandoahHeap::set_evacuation_in_progress(bool in_progress) { assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Only call this at safepoint"); set_gc_state_mask(EVACUATION, in_progress); @@ -2310,6 +2362,26 @@ VMThread::execute(&op); } +void ShenandoahHeap::vmop_entry_init_traversal() { + TraceCollectorStats tcs(monitoring_support()->stw_collection_counters()); + ShenandoahGCPhase total(ShenandoahPhaseTimings::total_pause_gross); + ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_traversal_gc_gross); + + try_inject_alloc_failure(); + VM_ShenandoahInitTraversalGC op; + VMThread::execute(&op); +} + +void ShenandoahHeap::vmop_entry_final_traversal() { + TraceCollectorStats tcs(monitoring_support()->stw_collection_counters()); + ShenandoahGCPhase total(ShenandoahPhaseTimings::total_pause_gross); + ShenandoahGCPhase phase(ShenandoahPhaseTimings::final_traversal_gc_gross); + + try_inject_alloc_failure(); + VM_ShenandoahFinalTraversalGC op; + VMThread::execute(&op); +} + void ShenandoahHeap::vmop_entry_full(GCCause::Cause cause) { TraceCollectorStats tcs(monitoring_support()->full_stw_collection_counters()); ShenandoahGCPhase total(ShenandoahPhaseTimings::total_pause_gross); @@ -2398,6 +2470,36 @@ op_final_updaterefs(); } +void ShenandoahHeap::entry_init_traversal() { + ShenandoahGCPhase total_phase(ShenandoahPhaseTimings::total_pause); + ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_traversal_gc); + + static const char* msg = "Pause Init Traversal"; + GCTraceTime time(msg, PrintGC, _gc_timer, tracer()->gc_id()); + EventMark em("%s", msg); + + ShenandoahWorkerScope scope(workers(), + ShenandoahWorkerPolicy::calc_workers_for_stw_traversal(), + "init traversal"); + + op_init_traversal(); +} + +void ShenandoahHeap::entry_final_traversal() { + ShenandoahGCPhase total_phase(ShenandoahPhaseTimings::total_pause); + ShenandoahGCPhase phase(ShenandoahPhaseTimings::final_traversal_gc); + + static const char* msg = "Pause Final Traversal"; + GCTraceTime time(msg, PrintGC, _gc_timer, tracer()->gc_id()); + EventMark em("%s", msg); + + ShenandoahWorkerScope scope(workers(), + ShenandoahWorkerPolicy::calc_workers_for_stw_traversal(), + "final traversal"); + + op_final_traversal(); +} + void ShenandoahHeap::entry_full(GCCause::Cause cause) { ShenandoahGCPhase total_phase(ShenandoahPhaseTimings::total_pause); ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc); @@ -2523,6 +2625,21 @@ } } +void ShenandoahHeap::entry_traversal() { + static const char* msg = "Concurrent traversal"; + GCTraceTime time(msg, PrintGC, NULL, tracer()->gc_id(), true); + EventMark em("%s", msg); + + TraceCollectorStats tcs(monitoring_support()->concurrent_collection_counters()); + + ShenandoahWorkerScope scope(workers(), + ShenandoahWorkerPolicy::calc_workers_for_conc_traversal(), + "concurrent traversal"); + + try_inject_alloc_failure(); + op_traversal(); +} + void ShenandoahHeap::entry_uncommit(double shrink_before) { static const char *msg = "Concurrent uncommit"; GCTraceTime time(msg, PrintGC, NULL, tracer()->gc_id(), true); @@ -2651,6 +2768,8 @@ switch (point) { case _degenerated_unset: return "Pause Degenerated GC ()"; + case _degenerated_traversal: + return "Pause Degenerated GC (Traversal)"; case _degenerated_outside_cycle: return "Pause Degenerated GC (Outside of Cycle)"; case _degenerated_mark: --- old/src/share/vm/gc_implementation/shenandoah/shenandoahHeap.hpp 2019-11-11 16:09:55.890342945 +0100 +++ new/src/share/vm/gc_implementation/shenandoah/shenandoahHeap.hpp 2019-11-11 16:09:55.803342949 +0100 @@ -49,6 +49,7 @@ class ShenandoahMode; class ShenandoahPhaseTimings; class ShenandoahPacer; +class ShenandoahTraversalGC; class ShenandoahVerifier; class ShenandoahWorkGang; class VMStructs; @@ -235,7 +236,10 @@ EVACUATION_BITPOS = 2, // Heap is under updating: needs SVRB/SVWB barriers. - UPDATEREFS_BITPOS = 3 + UPDATEREFS_BITPOS = 3, + + // Heap is under traversal collection + TRAVERSAL_BITPOS = 4 }; enum GCState { @@ -243,7 +247,8 @@ HAS_FORWARDED = 1 << HAS_FORWARDED_BITPOS, MARKING = 1 << MARKING_BITPOS, EVACUATION = 1 << EVACUATION_BITPOS, - UPDATEREFS = 1 << UPDATEREFS_BITPOS + UPDATEREFS = 1 << UPDATEREFS_BITPOS, + TRAVERSAL = 1 << TRAVERSAL_BITPOS }; private: @@ -265,6 +270,7 @@ void set_degenerated_gc_in_progress(bool in_progress); void set_full_gc_in_progress(bool in_progress); void set_full_gc_move_in_progress(bool in_progress); + void set_concurrent_traversal_in_progress(bool in_progress); void set_has_forwarded_objects(bool cond); inline bool is_stable() const; @@ -275,6 +281,7 @@ inline bool is_degenerated_gc_in_progress() const; inline bool is_full_gc_in_progress() const; inline bool is_full_gc_move_in_progress() const; + inline bool is_concurrent_traversal_in_progress() const; inline bool has_forwarded_objects() const; inline bool is_gc_in_progress_mask(uint mask) const; @@ -285,6 +292,7 @@ public: enum ShenandoahDegenPoint { _degenerated_unset, + _degenerated_traversal, _degenerated_outside_cycle, _degenerated_mark, _degenerated_evac, @@ -296,6 +304,8 @@ switch (point) { case _degenerated_unset: return ""; + case _degenerated_traversal: + return "Traversal"; case _degenerated_outside_cycle: return "Outside of Cycle"; case _degenerated_mark: @@ -333,6 +343,8 @@ void vmop_entry_final_evac(); void vmop_entry_init_updaterefs(); void vmop_entry_final_updaterefs(); + void vmop_entry_init_traversal(); + void vmop_entry_final_traversal(); void vmop_entry_full(GCCause::Cause cause); void vmop_degenerated(ShenandoahDegenPoint point); @@ -343,6 +355,8 @@ void entry_final_evac(); void entry_init_updaterefs(); void entry_final_updaterefs(); + void entry_init_traversal(); + void entry_final_traversal(); void entry_full(GCCause::Cause cause); void entry_degenerated(int point); @@ -354,6 +368,7 @@ void entry_cleanup(); void entry_evac(); void entry_updaterefs(); + void entry_traversal(); void entry_uncommit(double shrink_before); private: @@ -363,6 +378,8 @@ void op_final_evac(); void op_init_updaterefs(); void op_final_updaterefs(); + void op_init_traversal(); + void op_final_traversal(); void op_full(GCCause::Cause cause); void op_degenerated(ShenandoahDegenPoint point); void op_degenerated_fail(); @@ -375,6 +392,7 @@ void op_conc_evac(); void op_stw_evac(); void op_updaterefs(); + void op_traversal(); void op_uncommit(double shrink_before); // Messages for GC trace event, they have to be immortal for @@ -393,6 +411,7 @@ ShenandoahHeuristics* _heuristics; ShenandoahFreeSet* _free_set; ShenandoahConcurrentMark* _scm; + ShenandoahTraversalGC* _traversal_gc; ShenandoahMarkCompact* _full_gc; ShenandoahPacer* _pacer; ShenandoahVerifier* _verifier; @@ -408,6 +427,8 @@ ShenandoahHeuristics* heuristics() const { return _heuristics; } ShenandoahFreeSet* free_set() const { return _free_set; } ShenandoahConcurrentMark* concurrent_mark() { return _scm; } + ShenandoahTraversalGC* traversal_gc() const { return _traversal_gc; } + bool is_traversal_mode() const { return _traversal_gc != NULL; } ShenandoahPacer* pacer() const { return _pacer; } ShenandoahPhaseTimings* phase_timings() const { return _phase_timings; } @@ -637,6 +658,9 @@ // public: template + inline oop evac_update_with_forwarded(T* p); + + template inline oop maybe_update_with_forwarded(T* p); template --- old/src/share/vm/gc_implementation/shenandoah/shenandoahHeap.inline.hpp 2019-11-11 16:09:56.424342915 +0100 +++ new/src/share/vm/gc_implementation/shenandoah/shenandoahHeap.inline.hpp 2019-11-11 16:09:56.341342920 +0100 @@ -98,6 +98,29 @@ } } +template +inline oop ShenandoahHeap::evac_update_with_forwarded(T* p) { + T o = oopDesc::load_heap_oop(p); + if (!oopDesc::is_null(o)) { + oop heap_oop = oopDesc::decode_heap_oop_not_null(o); + if (in_collection_set(heap_oop)) { + oop forwarded_oop = ShenandoahBarrierSet::resolve_forwarded_not_null(heap_oop); + if (forwarded_oop == heap_oop) { + forwarded_oop = evacuate_object(heap_oop, Thread::current()); + } + oop prev = cas_oop(forwarded_oop, p, heap_oop); + if (prev == heap_oop) { + return forwarded_oop; + } else { + return NULL; + } + } + return heap_oop; + } else { + return NULL; + } +} + inline oop ShenandoahHeap::cas_oop(oop n, oop* addr, oop c) { assert(is_ptr_aligned(addr, sizeof(narrowOop)), err_msg("Address should be aligned: " PTR_FORMAT, p2i(addr))); return (oop) Atomic::cmpxchg_ptr(n, addr, c); @@ -272,13 +295,17 @@ } inline bool ShenandoahHeap::is_idle() const { - return _gc_state.is_unset(MARKING | EVACUATION | UPDATEREFS); + return _gc_state.is_unset(MARKING | EVACUATION | UPDATEREFS | TRAVERSAL); } inline bool ShenandoahHeap::is_concurrent_mark_in_progress() const { return _gc_state.is_set(MARKING); } +inline bool ShenandoahHeap::is_concurrent_traversal_in_progress() const { + return _gc_state.is_set(TRAVERSAL); +} + inline bool ShenandoahHeap::is_evacuation_in_progress() const { return _gc_state.is_set(EVACUATION); } --- old/src/share/vm/gc_implementation/shenandoah/shenandoahHeapRegion.cpp 2019-11-11 16:09:56.950342886 +0100 +++ new/src/share/vm/gc_implementation/shenandoah/shenandoahHeapRegion.cpp 2019-11-11 16:09:56.867342891 +0100 @@ -100,7 +100,8 @@ void ShenandoahHeapRegion::make_regular_bypass() { _heap->assert_heaplock_owned_by_current_thread(); - assert (_heap->is_full_gc_in_progress(), "only for full GC"); + assert (_heap->is_full_gc_in_progress() || _heap->is_degenerated_gc_in_progress(), + "only for full or degen GC"); switch (_state) { case _empty_uncommitted: --- old/src/share/vm/gc_implementation/shenandoah/shenandoahHeapRegionCounters.cpp 2019-11-11 16:09:57.480342857 +0100 +++ new/src/share/vm/gc_implementation/shenandoah/shenandoahHeapRegionCounters.cpp 2019-11-11 16:09:57.394342862 +0100 @@ -81,10 +81,11 @@ ShenandoahHeap* heap = ShenandoahHeap::heap(); jlong status = 0; - if (heap->is_concurrent_mark_in_progress()) status |= 1 << 0; - if (heap->is_evacuation_in_progress()) status |= 1 << 1; - if (heap->is_update_refs_in_progress()) status |= 1 << 2; - _status->set_value(status); + if (heap->is_concurrent_mark_in_progress()) status |= 1 << 0; + if (heap->is_evacuation_in_progress()) status |= 1 << 1; + if (heap->is_update_refs_in_progress()) status |= 1 << 2; + if (heap->is_concurrent_traversal_in_progress()) status |= 1 << 3; + _status->set_value(status); _timestamp->set_value(os::elapsed_counter()); --- old/src/share/vm/gc_implementation/shenandoah/shenandoahHeapRegionCounters.hpp 2019-11-11 16:09:57.998342828 +0100 +++ new/src/share/vm/gc_implementation/shenandoah/shenandoahHeapRegionCounters.hpp 2019-11-11 16:09:57.910342833 +0100 @@ -39,6 +39,7 @@ * - bit 0 set when marking in progress * - bit 1 set when evacuation in progress * - bit 2 set when update refs in progress + * - bit 3 set when traversal in progress * * one variable counter per region, with $max_regions (see above) counters: * - sun.gc.shenandoah.regions.region.$i.data --- old/src/share/vm/gc_implementation/shenandoah/shenandoahHeuristics.cpp 2019-11-11 16:09:58.526342799 +0100 +++ new/src/share/vm/gc_implementation/shenandoah/shenandoahHeuristics.cpp 2019-11-11 16:09:58.440342804 +0100 @@ -39,6 +39,14 @@ else return 0; } +int ShenandoahHeuristics::compare_by_garbage_then_alloc_seq_ascending(RegionData a, RegionData b) { + int r = compare_by_garbage(a, b); + if (r != 0) { + return r; + } + return compare_by_alloc_seq_ascending(a, b); +} + int ShenandoahHeuristics::compare_by_alloc_seq_ascending(RegionData a, RegionData b) { if (a._seqnum_last_alloc == b._seqnum_last_alloc) return 0; --- old/src/share/vm/gc_implementation/shenandoah/shenandoahHeuristics.hpp 2019-11-11 16:09:59.047342771 +0100 +++ new/src/share/vm/gc_implementation/shenandoah/shenandoahHeuristics.hpp 2019-11-11 16:09:58.961342775 +0100 @@ -100,6 +100,7 @@ ShenandoahSharedFlag _metaspace_oom; static int compare_by_garbage(RegionData a, RegionData b); + static int compare_by_garbage_then_alloc_seq_ascending(RegionData a, RegionData b); static int compare_by_alloc_seq_ascending(RegionData a, RegionData b); static int compare_by_alloc_seq_descending(RegionData a, RegionData b); --- old/src/share/vm/gc_implementation/shenandoah/shenandoahMarkCompact.cpp 2019-11-11 16:09:59.568342742 +0100 +++ new/src/share/vm/gc_implementation/shenandoah/shenandoahMarkCompact.cpp 2019-11-11 16:09:59.487342746 +0100 @@ -39,6 +39,7 @@ #include "gc_implementation/shenandoah/shenandoahHeuristics.hpp" #include "gc_implementation/shenandoah/shenandoahMarkingContext.inline.hpp" #include "gc_implementation/shenandoah/shenandoahRootProcessor.hpp" +#include "gc_implementation/shenandoah/shenandoahTraversalGC.hpp" #include "gc_implementation/shenandoah/shenandoahTaskqueue.inline.hpp" #include "gc_implementation/shenandoah/shenandoahUtils.hpp" #include "gc_implementation/shenandoah/shenandoahVerifier.hpp" @@ -86,28 +87,34 @@ ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_prepare); // Full GC is supposed to recover from any GC state: - // 0. Remember if we have forwarded objects + // a0. Remember if we have forwarded objects bool has_forwarded_objects = heap->has_forwarded_objects(); - // a. Cancel concurrent mark, if in progress - if (heap->is_concurrent_mark_in_progress()) { - heap->concurrent_mark()->cancel(); - heap->stop_concurrent_marking(); - } - assert(!heap->is_concurrent_mark_in_progress(), "sanity"); - - // b1. Cancel evacuation, if in progress + // a1. Cancel evacuation, if in progress if (heap->is_evacuation_in_progress()) { heap->set_evacuation_in_progress(false); } assert(!heap->is_evacuation_in_progress(), "sanity"); - // b2. Cancel update-refs, if in progress + // a2. Cancel update-refs, if in progress if (heap->is_update_refs_in_progress()) { heap->set_update_refs_in_progress(false); } assert(!heap->is_update_refs_in_progress(), "sanity"); + // a3. Cancel concurrent traversal GC, if in progress + if (heap->is_concurrent_traversal_in_progress()) { + heap->traversal_gc()->reset(); + heap->set_concurrent_traversal_in_progress(false); + } + + // b. Cancel concurrent mark, if in progress + if (heap->is_concurrent_mark_in_progress()) { + heap->concurrent_mark()->cancel(); + heap->stop_concurrent_marking(); + } + assert(!heap->is_concurrent_mark_in_progress(), "sanity"); + // c. Reset the bitmaps for new marking heap->reset_mark_bitmap(); assert(heap->marking_context()->is_bitmap_clear(), "sanity"); --- old/src/share/vm/gc_implementation/shenandoah/shenandoahNormalMode.cpp 2019-11-11 16:10:00.101342713 +0100 +++ new/src/share/vm/gc_implementation/shenandoah/shenandoahNormalMode.cpp 2019-11-11 16:10:00.019342717 +0100 @@ -35,6 +35,7 @@ // Final configuration checks SHENANDOAH_CHECK_FLAG_SET(ShenandoahLoadRefBarrier); SHENANDOAH_CHECK_FLAG_SET(ShenandoahSATBBarrier); + SHENANDOAH_CHECK_FLAG_SET(ShenandoahKeepAliveBarrier); SHENANDOAH_CHECK_FLAG_SET(ShenandoahCASBarrier); SHENANDOAH_CHECK_FLAG_SET(ShenandoahCloneBarrier); } --- old/src/share/vm/gc_implementation/shenandoah/shenandoahOopClosures.hpp 2019-11-11 16:10:00.619342684 +0100 +++ new/src/share/vm/gc_implementation/shenandoah/shenandoahOopClosures.hpp 2019-11-11 16:10:00.538342688 +0100 @@ -24,11 +24,14 @@ #ifndef SHARE_VM_GC_SHENANDOAH_SHENANDOAHOOPCLOSURES_HPP #define SHARE_VM_GC_SHENANDOAH_SHENANDOAHOOPCLOSURES_HPP +#include "memory/iterator.hpp" #include "gc_implementation/shenandoah/shenandoahTaskqueue.hpp" class ShenandoahHeap; class ShenandoahStrDedupQueue; class ShenandoahMarkingContext; +class ShenandoahTraversalGC; +class OopClosure; enum UpdateRefsMode { NONE, // No reference updating @@ -200,4 +203,154 @@ virtual void do_oop(oop* p) { do_oop_nv(p); } }; +class ShenandoahTraversalSuperClosure : public MetadataAwareOopClosure { +private: + ShenandoahTraversalGC* const _traversal_gc; + ShenandoahStrDedupQueue* const _dedup_queue; + Thread* const _thread; + ShenandoahObjToScanQueue* const _queue; + ShenandoahMarkingContext* const _mark_context; +protected: + ShenandoahTraversalSuperClosure(ShenandoahObjToScanQueue* q, ReferenceProcessor* rp, ShenandoahStrDedupQueue* dq = NULL); + + template + void work(T* p); + +}; + +class ShenandoahTraversalRootsClosure : public ShenandoahTraversalSuperClosure { +private: + template + inline void do_oop_work(T* p) { work(p); } + +public: + ShenandoahTraversalRootsClosure(ShenandoahObjToScanQueue* q, ReferenceProcessor* rp) : + ShenandoahTraversalSuperClosure(q, rp) {} + + virtual void do_oop(narrowOop* p) { do_oop_work(p); } + virtual void do_oop(oop* p) { do_oop_work(p); } + + virtual bool do_metadata() { return false; } +}; + +class ShenandoahTraversalClosure : public ShenandoahTraversalSuperClosure { +private: + template + inline void do_oop_work(T* p) { work(p); } + +public: + ShenandoahTraversalClosure(ShenandoahObjToScanQueue* q, ReferenceProcessor* rp) : + ShenandoahTraversalSuperClosure(q, rp) {} + + virtual void do_oop(narrowOop* p) { do_oop_work(p); } + virtual void do_oop(oop* p) { do_oop_work(p); } + + virtual bool do_metadata() { return false; } +}; + +class ShenandoahTraversalMetadataClosure : public ShenandoahTraversalSuperClosure { +private: + template + inline void do_oop_work(T* p) { work(p); } + +public: + ShenandoahTraversalMetadataClosure(ShenandoahObjToScanQueue* q, ReferenceProcessor* rp) : + ShenandoahTraversalSuperClosure(q, rp) {} + + virtual void do_oop(narrowOop* p) { do_oop_work(p); } + virtual void do_oop(oop* p) { do_oop_work(p); } + + virtual bool do_metadata() { return true; } +}; + +class ShenandoahTraversalDedupClosure : public ShenandoahTraversalSuperClosure { +private: + template + inline void do_oop_work(T* p) { work(p); } + +public: + ShenandoahTraversalDedupClosure(ShenandoahObjToScanQueue* q, ReferenceProcessor* rp, ShenandoahStrDedupQueue* dq) : + ShenandoahTraversalSuperClosure(q, rp, dq) {} + + virtual void do_oop(narrowOop* p) { do_oop_work(p); } + virtual void do_oop(oop* p) { do_oop_work(p); } + + virtual bool do_metadata() { return false; } +}; + +class ShenandoahTraversalMetadataDedupClosure : public ShenandoahTraversalSuperClosure { +private: + template + inline void do_oop_work(T* p) { work(p); } + +public: + ShenandoahTraversalMetadataDedupClosure(ShenandoahObjToScanQueue* q, ReferenceProcessor* rp, ShenandoahStrDedupQueue* dq) : + ShenandoahTraversalSuperClosure(q, rp, dq) {} + + virtual void do_oop(narrowOop* p) { do_oop_work(p); } + virtual void do_oop(oop* p) { do_oop_work(p); } + + virtual bool do_metadata() { return true; } +}; + +class ShenandoahTraversalDegenClosure : public ShenandoahTraversalSuperClosure { +private: + template + inline void do_oop_work(T* p) { work(p); } + +public: + ShenandoahTraversalDegenClosure(ShenandoahObjToScanQueue* q, ReferenceProcessor* rp) : + ShenandoahTraversalSuperClosure(q, rp) {} + + virtual void do_oop(narrowOop* p) { do_oop_work(p); } + virtual void do_oop(oop* p) { do_oop_work(p); } + + virtual bool do_metadata() { return false; } +}; + +class ShenandoahTraversalMetadataDegenClosure : public ShenandoahTraversalSuperClosure { +private: + template + inline void do_oop_work(T* p) { work(p); } + +public: + ShenandoahTraversalMetadataDegenClosure(ShenandoahObjToScanQueue* q, ReferenceProcessor* rp) : + ShenandoahTraversalSuperClosure(q, rp) {} + + virtual void do_oop(narrowOop* p) { do_oop_work(p); } + virtual void do_oop(oop* p) { do_oop_work(p); } + + virtual bool do_metadata() { return true; } +}; + +class ShenandoahTraversalDedupDegenClosure : public ShenandoahTraversalSuperClosure { +private: + template + inline void do_oop_work(T* p) { work(p); } + +public: + ShenandoahTraversalDedupDegenClosure(ShenandoahObjToScanQueue* q, ReferenceProcessor* rp, ShenandoahStrDedupQueue* dq) : + ShenandoahTraversalSuperClosure(q, rp, dq) {} + + virtual void do_oop(narrowOop* p) { do_oop_work(p); } + virtual void do_oop(oop* p) { do_oop_work(p); } + + virtual bool do_metadata() { return false; } +}; + +class ShenandoahTraversalMetadataDedupDegenClosure : public ShenandoahTraversalSuperClosure { +private: + template + inline void do_oop_work(T* p) { work(p); } + +public: + ShenandoahTraversalMetadataDedupDegenClosure(ShenandoahObjToScanQueue* q, ReferenceProcessor* rp, ShenandoahStrDedupQueue* dq) : + ShenandoahTraversalSuperClosure(q, rp, dq) {} + + virtual void do_oop(narrowOop* p) { do_oop_work(p); } + virtual void do_oop(oop* p) { do_oop_work(p); } + + virtual bool do_metadata() { return true; } +}; + #endif // SHARE_VM_GC_SHENANDOAH_SHENANDOAHOOPCLOSURES_HPP --- old/src/share/vm/gc_implementation/shenandoah/shenandoahOopClosures.inline.hpp 2019-11-11 16:10:01.145342655 +0100 +++ new/src/share/vm/gc_implementation/shenandoah/shenandoahOopClosures.inline.hpp 2019-11-11 16:10:01.061342660 +0100 @@ -26,6 +26,7 @@ #include "gc_implementation/shenandoah/shenandoahHeap.inline.hpp" #include "gc_implementation/shenandoah/shenandoahConcurrentMark.inline.hpp" +#include "gc_implementation/shenandoah/shenandoahTraversalGC.inline.hpp" template inline void ShenandoahMarkRefsSuperClosure::work(T *p) { @@ -37,4 +38,9 @@ _heap->maybe_update_with_forwarded(p); } +template +inline void ShenandoahTraversalSuperClosure::work(T* p) { + _traversal_gc->process_oop(p, _thread, _queue, _mark_context, _dedup_queue); +} + #endif // SHARE_VM_GC_SHENANDOAH_SHENANDOAHOOPCLOSURES_INLINE_HPP --- old/src/share/vm/gc_implementation/shenandoah/shenandoahPacer.cpp 2019-11-11 16:10:01.661342627 +0100 +++ new/src/share/vm/gc_implementation/shenandoah/shenandoahPacer.cpp 2019-11-11 16:10:01.575342631 +0100 @@ -127,6 +127,33 @@ } /* + * Traversal walks the entire heap once, and therefore we have to make assumptions about its + * liveness, like concurrent mark does. + */ + +void ShenandoahPacer::setup_for_traversal() { + assert(ShenandoahPacing, "Only be here when pacing is enabled"); + + size_t live = update_and_get_progress_history(); + size_t free = _heap->free_set()->available(); + + size_t non_taxable = free * ShenandoahPacingCycleSlack / 100; + size_t taxable = free - non_taxable; + + double tax = 1.0 * live / taxable; // base tax for available free space + tax *= ShenandoahPacingSurcharge; // additional surcharge to help unclutter heap + + restart_with(non_taxable, tax); + + log_info(gc, ergo)("Pacer for Traversal. Expected Live: " SIZE_FORMAT "%s, Free: " SIZE_FORMAT "%s, " + "Non-Taxable: " SIZE_FORMAT "%s, Alloc Tax Rate: %.1fx", + byte_size_in_proper_unit(live), proper_unit_for_byte_size(live), + byte_size_in_proper_unit(free), proper_unit_for_byte_size(free), + byte_size_in_proper_unit(non_taxable), proper_unit_for_byte_size(non_taxable), + tax); +} + +/* * In idle phase, we have to pace the application to let control thread react with GC start. * * Here, we have rendezvous with concurrent thread that adds up the budget as it acknowledges --- old/src/share/vm/gc_implementation/shenandoah/shenandoahPacer.hpp 2019-11-11 16:10:02.191342597 +0100 +++ new/src/share/vm/gc_implementation/shenandoah/shenandoahPacer.hpp 2019-11-11 16:10:02.105342602 +0100 @@ -67,6 +67,7 @@ void setup_for_mark(); void setup_for_evac(); void setup_for_updaterefs(); + void setup_for_traversal(); inline void report_mark(size_t words); inline void report_evac(size_t words); --- old/src/share/vm/gc_implementation/shenandoah/shenandoahPassiveMode.cpp 2019-11-11 16:10:02.709342569 +0100 +++ new/src/share/vm/gc_implementation/shenandoah/shenandoahPassiveMode.cpp 2019-11-11 16:10:02.630342573 +0100 @@ -42,6 +42,8 @@ // Disable known barriers by default. SHENANDOAH_ERGO_DISABLE_FLAG(ShenandoahLoadRefBarrier); SHENANDOAH_ERGO_DISABLE_FLAG(ShenandoahSATBBarrier); + SHENANDOAH_ERGO_DISABLE_FLAG(ShenandoahKeepAliveBarrier); + SHENANDOAH_ERGO_DISABLE_FLAG(ShenandoahStoreValEnqueueBarrier); SHENANDOAH_ERGO_DISABLE_FLAG(ShenandoahCASBarrier); SHENANDOAH_ERGO_DISABLE_FLAG(ShenandoahCloneBarrier); --- old/src/share/vm/gc_implementation/shenandoah/shenandoahPhaseTimings.cpp 2019-11-11 16:10:03.238342540 +0100 +++ new/src/share/vm/gc_implementation/shenandoah/shenandoahPhaseTimings.cpp 2019-11-11 16:10:03.151342544 +0100 @@ -84,6 +84,10 @@ guarantee(phase == init_evac || phase == scan_roots || phase == update_roots || + phase == init_traversal_gc_work || + phase == init_weak_traversal_gc_work || + phase == final_traversal_gc_work || + phase == final_traversal_update_roots || phase == final_update_refs_roots || phase == full_gc_roots || phase == degen_gc_update_roots || --- old/src/share/vm/gc_implementation/shenandoah/shenandoahPhaseTimings.hpp 2019-11-11 16:10:03.767342511 +0100 +++ new/src/share/vm/gc_implementation/shenandoah/shenandoahPhaseTimings.hpp 2019-11-11 16:10:03.683342515 +0100 @@ -170,6 +170,89 @@ f(degen_gc_update_string_dedup_roots, " DU: String Dedup Roots") \ f(degen_gc_update_finish_queues, " DU: Finish Queues") \ \ + f(init_traversal_gc_gross, "Pause Init Traversal (G)") \ + f(init_traversal_gc, "Pause Init Traversal (N)") \ + f(traversal_gc_prepare, " Prepare") \ + f(traversal_gc_accumulate_stats, " Accumulate Stats") \ + f(traversal_gc_make_parsable, " Make Parsable") \ + f(traversal_gc_resize_tlabs, " Resize TLABs") \ + f(traversal_gc_prepare_sync_pinned, " Sync Pinned") \ + \ + /* Per-thread timer block, should have "roots" counters in consistent order */ \ + f(init_traversal_gc_work, " Work") \ + f(init_traversal_gc_thread_roots, " TI: Thread Roots") \ + f(init_traversal_gc_code_roots, " TI: Code Cache Roots") \ + f(init_traversal_gc_string_table_roots, " TI: String Table Roots") \ + f(init_traversal_gc_universe_roots, " TI: Universe Roots") \ + f(init_traversal_gc_jni_roots, " TI: JNI Roots") \ + f(init_traversal_gc_jni_weak_roots, " TI: JNI Weak Roots") \ + f(init_traversal_gc_synchronizer_roots, " TI: Synchronizer Roots") \ + f(init_traversal_gc_management_roots, " TI: Management Roots") \ + f(init_traversal_gc_system_dict_roots, " TI: System Dict Roots") \ + f(init_traversal_gc_cldg_roots, " TI: CLDG Roots") \ + f(init_traversal_gc_jvmti_roots, " TI: JVMTI Roots") \ + f(init_traversal_gc_string_dedup_table_roots, " TI: Dedup Table Roots") \ + f(init_traversal_gc_string_dedup_queue_roots, " TI: Dedup Queue Roots") \ + f(init_traversal_gc_finish_queues, " TI: Finish Queues") \ + \ + /* Per-thread timer block, should have "roots" counters in consistent order */ \ + f(init_weak_traversal_gc_work, " Weak") \ + f(init_weak_traversal_gc_thread_roots, " TW: Thread Roots") \ + f(init_weak_traversal_gc_code_roots, " TW: Code Cache Roots") \ + f(init_weak_traversal_gc_string_table_roots, " TW: String Table Roots") \ + f(init_weak_traversal_gc_universe_roots, " TW: Universe Roots") \ + f(init_weak_traversal_gc_jni_roots, " TW: JNI Roots") \ + f(init_weak_traversal_gc_jni_weak_roots, " TW: JNI Weak Roots") \ + f(init_weak_traversal_gc_synchronizer_roots, " TW: Synchronizer Roots") \ + f(init_weak_traversal_gc_management_roots, " TW: Management Roots") \ + f(init_weak_traversal_gc_system_dict_roots, " TW: System Dict Roots") \ + f(init_weak_traversal_gc_cldg_roots, " TW: CLDG Roots") \ + f(init_weak_traversal_gc_jvmti_roots, " TW: JVMTI Roots") \ + f(init_weak_traversal_gc_string_dedup_table_roots, " TW: Dedup Table Roots") \ + f(init_weak_traversal_gc_string_dedup_queue_roots, " TW: Dedup Queue Roots") \ + f(init_weak_traversal_gc_finish_queues, " TW: Finish Queues") \ + \ + f(final_traversal_gc_gross, "Pause Final Traversal (G)") \ + f(final_traversal_gc, "Pause Final Traversal (N)") \ + \ + /* Per-thread timer block, should have "roots" counters in consistent order */ \ + f(final_traversal_gc_work, " Work") \ + f(final_traversal_gc_thread_roots, " TF: Thread Roots") \ + f(final_traversal_gc_code_roots, " TF: Code Cache Roots") \ + f(final_traversal_gc_string_table_roots, " TF: String Table Roots") \ + f(final_traversal_gc_universe_roots, " TF: Universe Roots") \ + f(final_traversal_gc_jni_roots, " TF: JNI Roots") \ + f(final_traversal_gc_jni_weak_roots, " TF: JNI Weak Roots") \ + f(final_traversal_gc_synchronizer_roots, " TF: Synchronizer Roots") \ + f(final_traversal_gc_management_roots, " TF: Management Roots") \ + f(final_traversal_gc_system_dict_roots, " TF: System Dict Roots") \ + f(final_traversal_gc_cldg_roots, " TF: CLDG Roots") \ + f(final_traversal_gc_jvmti_roots, " TF: JVMTI Roots") \ + f(final_traversal_gc_string_dedup_table_roots, " TF: Dedup Table Roots") \ + f(final_traversal_gc_string_dedup_queue_roots, " TF: Dedup Queue Roots") \ + f(final_traversal_gc_finish_queues, " TF: Finish Queues") \ + f(final_traversal_gc_termination, " TF: Termination") \ + \ + /* Per-thread timer block, should have "roots" counters in consistent order */ \ + f(final_traversal_update_roots, " Update Roots") \ + f(final_traversal_update_thread_roots, " TU: Thread Roots") \ + f(final_traversal_update_code_roots, " TU: Code Cache Roots") \ + f(final_traversal_update_string_table_roots, " TU: String Table Roots") \ + f(final_traversal_update_universe_roots, " TU: Universe Roots") \ + f(final_traversal_update_jni_roots, " TU: JNI Roots") \ + f(final_traversal_update_jni_weak_roots, " TU: JNI Weak Roots") \ + f(final_traversal_update_synchronizer_roots, " TU: Synchronizer Roots") \ + f(final_traversal_update_management_roots, " TU: Management Roots") \ + f(final_traversal_update_system_dict_roots, " TU: System Dict Roots") \ + f(final_traversal_update_cldg_roots, " TU: CLDG Roots") \ + f(final_traversal_update_jvmti_roots, " TU: JVMTI Roots") \ + f(final_traversal_update_string_dedup_table_roots, " TU: Dedup Table Roots") \ + f(final_traversal_update_string_dedup_queue_roots, " TU: Dedup Queue Roots") \ + f(final_traversal_update_finish_queues, " TU: Finish Queues") \ + \ + f(traversal_gc_sync_pinned, " Sync Pinned") \ + f(traversal_gc_cleanup, " Cleanup") \ + \ f(full_gc_gross, "Pause Full GC (G)") \ f(full_gc, "Pause Full GC (N)") \ f(full_gc_heapdumps, " Heap Dumps") \ --- old/src/share/vm/gc_implementation/shenandoah/shenandoahTimingTracker.cpp 2019-11-11 16:10:04.301342481 +0100 +++ new/src/share/vm/gc_implementation/shenandoah/shenandoahTimingTracker.cpp 2019-11-11 16:10:04.214342486 +0100 @@ -64,8 +64,10 @@ ShenandoahTerminationTracker::ShenandoahTerminationTracker(ShenandoahPhaseTimings::Phase phase) : _phase(phase) { assert(_current_termination_phase == ShenandoahPhaseTimings::_num_phases, "Should be invalid"); assert(phase == ShenandoahPhaseTimings::termination || + phase == ShenandoahPhaseTimings::final_traversal_gc_termination || phase == ShenandoahPhaseTimings::full_gc_mark_termination || phase == ShenandoahPhaseTimings::conc_termination || + phase == ShenandoahPhaseTimings::conc_traversal_termination || phase == ShenandoahPhaseTimings::weakrefs_termination || phase == ShenandoahPhaseTimings::full_gc_weakrefs_termination, "Only these phases"); --- old/src/share/vm/gc_implementation/shenandoah/shenandoahUtils.cpp 2019-11-11 16:10:04.820342453 +0100 +++ new/src/share/vm/gc_implementation/shenandoah/shenandoahUtils.cpp 2019-11-11 16:10:04.738342457 +0100 @@ -125,6 +125,9 @@ case ShenandoahPhaseTimings::init_evac: case ShenandoahPhaseTimings::final_update_refs_roots: case ShenandoahPhaseTimings::degen_gc_update_roots: + case ShenandoahPhaseTimings::init_traversal_gc_work: + case ShenandoahPhaseTimings::final_traversal_gc_work: + case ShenandoahPhaseTimings::final_traversal_update_roots: case ShenandoahPhaseTimings::full_gc_roots: return true; default: --- old/src/share/vm/gc_implementation/shenandoah/shenandoahUtils.hpp 2019-11-11 16:10:05.342342424 +0100 +++ new/src/share/vm/gc_implementation/shenandoah/shenandoahUtils.hpp 2019-11-11 16:10:05.258342428 +0100 @@ -100,6 +100,8 @@ return type == VM_Operation::VMOp_ShenandoahInitMark || type == VM_Operation::VMOp_ShenandoahFinalMarkStartEvac || type == VM_Operation::VMOp_ShenandoahFinalEvac || + type == VM_Operation::VMOp_ShenandoahInitTraversalGC || + type == VM_Operation::VMOp_ShenandoahFinalTraversalGC || type == VM_Operation::VMOp_ShenandoahInitUpdateRefs || type == VM_Operation::VMOp_ShenandoahFinalUpdateRefs || type == VM_Operation::VMOp_ShenandoahFullGC || --- old/src/share/vm/gc_implementation/shenandoah/shenandoahVMOperations.cpp 2019-11-11 16:10:05.873342394 +0100 +++ new/src/share/vm/gc_implementation/shenandoah/shenandoahVMOperations.cpp 2019-11-11 16:10:05.788342399 +0100 @@ -66,6 +66,16 @@ ShenandoahHeap::heap()->entry_full(_gc_cause); } +void VM_ShenandoahInitTraversalGC::doit() { + ShenandoahGCPauseMark mark(SvcGCMarker::OTHER); + ShenandoahHeap::heap()->entry_init_traversal(); +} + +void VM_ShenandoahFinalTraversalGC::doit() { + ShenandoahGCPauseMark mark(SvcGCMarker::OTHER); + ShenandoahHeap::heap()->entry_final_traversal(); +} + void VM_ShenandoahInitUpdateRefs::doit() { ShenandoahGCPauseMark mark(SvcGCMarker::OTHER); ShenandoahHeap::heap()->entry_init_updaterefs(); --- old/src/share/vm/gc_implementation/shenandoah/shenandoahVMOperations.hpp 2019-11-11 16:10:06.401342365 +0100 +++ new/src/share/vm/gc_implementation/shenandoah/shenandoahVMOperations.hpp 2019-11-11 16:10:06.312342370 +0100 @@ -36,6 +36,8 @@ // - VM_ShenandoahInitUpdateRefs: initiate update references // - VM_ShenandoahFinalUpdateRefs: finish up update references // - VM_ShenandoahFullGC: do full GC +// - VM_ShenandoahInitTraversalGC: init traversal GC +// - VM_ShenandoahFinalTraversalGC: finish traversal GC class VM_ShenandoahOperation : public VM_Operation { public: @@ -99,6 +101,22 @@ virtual void doit(); }; +class VM_ShenandoahInitTraversalGC: public VM_ShenandoahOperation { +public: + VM_ShenandoahInitTraversalGC() : VM_ShenandoahOperation() {}; + VM_Operation::VMOp_Type type() const { return VMOp_ShenandoahInitTraversalGC; } + const char* name() const { return "Shenandoah Init Traversal Collection"; } + virtual void doit(); +}; + +class VM_ShenandoahFinalTraversalGC: public VM_ShenandoahReferenceOperation { +public: + VM_ShenandoahFinalTraversalGC() : VM_ShenandoahReferenceOperation() {}; + VM_Operation::VMOp_Type type() const { return VMOp_ShenandoahFinalTraversalGC; } + const char* name() const { return "Shenandoah Final Traversal Collection"; } + virtual void doit(); +}; + class VM_ShenandoahInitUpdateRefs: public VM_ShenandoahOperation { public: VM_ShenandoahInitUpdateRefs() : VM_ShenandoahOperation() {}; --- old/src/share/vm/gc_implementation/shenandoah/shenandoahVerifier.cpp 2019-11-11 16:10:06.931342336 +0100 +++ new/src/share/vm/gc_implementation/shenandoah/shenandoahVerifier.cpp 2019-11-11 16:10:06.844342341 +0100 @@ -862,6 +862,30 @@ ); } +void ShenandoahVerifier::verify_before_traversal() { + verify_at_safepoint( + "Before Traversal", + _verify_forwarded_none, // cannot have forwarded objects + _verify_marked_disable, // bitmaps are not relevant before traversal + _verify_cset_none, // no cset references before traversal + _verify_liveness_disable, // no reliable liveness data anymore + _verify_regions_notrash_nocset, // no trash and no cset regions + _verify_gcstate_stable // nothing forwarded before traversal + ); +} + +void ShenandoahVerifier::verify_after_traversal() { + verify_at_safepoint( + "After Traversal", + _verify_forwarded_none, // cannot have forwarded objects + _verify_marked_complete, // should have complete marking after traversal + _verify_cset_none, // no cset references left after traversal + _verify_liveness_disable, // liveness data is not collected for new allocations + _verify_regions_nocset, // no cset regions, trash regions allowed + _verify_gcstate_stable // nothing forwarded after traversal + ); +} + void ShenandoahVerifier::verify_before_fullgc() { verify_at_safepoint( "Before Full GC", --- old/src/share/vm/gc_implementation/shenandoah/shenandoahVerifier.hpp 2019-11-11 16:10:07.471342306 +0100 +++ new/src/share/vm/gc_implementation/shenandoah/shenandoahVerifier.hpp 2019-11-11 16:10:07.384342311 +0100 @@ -178,6 +178,8 @@ void verify_after_updaterefs(); void verify_before_fullgc(); void verify_after_fullgc(); + void verify_before_traversal(); + void verify_after_traversal(); void verify_after_degenerated(); void verify_generic(VerifyOption option); }; --- old/src/share/vm/gc_implementation/shenandoah/shenandoahWorkerPolicy.cpp 2019-11-11 16:10:07.997342277 +0100 +++ new/src/share/vm/gc_implementation/shenandoah/shenandoahWorkerPolicy.cpp 2019-11-11 16:10:07.912342282 +0100 @@ -32,6 +32,8 @@ uint ShenandoahWorkerPolicy::_prev_conc_evac = 0; uint ShenandoahWorkerPolicy::_prev_fullgc = 0; uint ShenandoahWorkerPolicy::_prev_degengc = 0; +uint ShenandoahWorkerPolicy::_prev_stw_traversal = 0; +uint ShenandoahWorkerPolicy::_prev_conc_traversal = 0; uint ShenandoahWorkerPolicy::_prev_conc_update_ref = 0; uint ShenandoahWorkerPolicy::_prev_par_update_ref = 0; uint ShenandoahWorkerPolicy::_prev_conc_cleanup = 0; @@ -81,6 +83,26 @@ return _prev_fullgc; } +// Calculate workers for Stop-the-world traversal GC +uint ShenandoahWorkerPolicy::calc_workers_for_stw_traversal() { + uint active_workers = (_prev_stw_traversal == 0) ? ParallelGCThreads : _prev_stw_traversal; + _prev_stw_traversal = + AdaptiveSizePolicy::calc_active_workers(ParallelGCThreads, + active_workers, + Threads::number_of_non_daemon_threads()); + return _prev_stw_traversal; +} + +// Calculate workers for concurent traversal GC +uint ShenandoahWorkerPolicy::calc_workers_for_conc_traversal() { + uint active_workers = (_prev_conc_traversal == 0) ? ConcGCThreads : _prev_conc_traversal; + _prev_conc_traversal = + AdaptiveSizePolicy::calc_active_conc_workers(ConcGCThreads, + active_workers, + Threads::number_of_non_daemon_threads()); + return _prev_conc_traversal; +} + // Calculate workers for concurrent reference update uint ShenandoahWorkerPolicy::calc_workers_for_conc_update_ref() { uint active_workers = (_prev_conc_update_ref == 0) ? ConcGCThreads : _prev_conc_update_ref; --- old/src/share/vm/gc_implementation/shenandoah/shenandoahWorkerPolicy.hpp 2019-11-11 16:10:08.524342248 +0100 +++ new/src/share/vm/gc_implementation/shenandoah/shenandoahWorkerPolicy.hpp 2019-11-11 16:10:08.437342253 +0100 @@ -33,6 +33,8 @@ static uint _prev_conc_evac; static uint _prev_fullgc; static uint _prev_degengc; + static uint _prev_stw_traversal; + static uint _prev_conc_traversal; static uint _prev_conc_update_ref; static uint _prev_par_update_ref; static uint _prev_conc_cleanup; @@ -57,6 +59,12 @@ // Calculate workers for parallel degenerated gc static uint calc_workers_for_stw_degenerated(); + // Calculate workers for Stop-the-world traversal GC + static uint calc_workers_for_stw_traversal(); + + // Calculate workers for concurrent traversal GC + static uint calc_workers_for_conc_traversal(); + // Calculate workers for concurrent reference update static uint calc_workers_for_conc_update_ref(); --- old/src/share/vm/gc_implementation/shenandoah/shenandoah_globals.hpp 2019-11-11 16:10:09.048342220 +0100 +++ new/src/share/vm/gc_implementation/shenandoah/shenandoah_globals.hpp 2019-11-11 16:10:08.962342224 +0100 @@ -70,6 +70,7 @@ product(ccstr, ShenandoahGCMode, "normal", \ "The GC mode to use in Shenandoah GC. Possible values" \ " *) normal - normal GC (mark-evac-update)" \ + " *) traversal - traversal GC (single-pass)" \ " *) passive - disable concurrent GC, do stop-the-world GC") \ \ experimental(ccstr, ShenandoahUpdateRefsEarly, "adaptive", \ @@ -324,6 +325,12 @@ diagnostic(bool, ShenandoahSATBBarrier, true, \ "Turn on/off SATB barriers in Shenandoah") \ \ + diagnostic(bool, ShenandoahKeepAliveBarrier, true, \ + "Turn on/off keep alive barriers in Shenandoah") \ + \ + diagnostic(bool, ShenandoahStoreValEnqueueBarrier, false, \ + "Turn on/off enqueuing of oops for storeval barriers") \ + \ diagnostic(bool, ShenandoahCASBarrier, true, \ "Turn on/off CAS barriers in Shenandoah") \ \ --- old/src/share/vm/gc_interface/gcCause.hpp 2019-11-11 16:10:09.581342190 +0100 +++ new/src/share/vm/gc_interface/gcCause.hpp 2019-11-11 16:10:09.496342195 +0100 @@ -77,6 +77,7 @@ _shenandoah_metadata_gc_clear_softrefs, _shenandoah_allocation_failure_evac, _shenandoah_concurrent_gc, + _shenandoah_traversal_gc, _shenandoah_upgrade_to_full_gc, _last_ditch_collection, --- old/src/share/vm/oops/oop.inline.hpp 2019-11-11 16:10:10.127342160 +0100 +++ new/src/share/vm/oops/oop.inline.hpp 2019-11-11 16:10:10.042342165 +0100 @@ -307,6 +307,9 @@ } #if INCLUDE_ALL_GCS if (UseShenandoahGC) { + if (exchange_value != NULL) { + ShenandoahBarrierSet::barrier_set()->storeval_barrier(exchange_value); + } result = ShenandoahBarrierSet::barrier_set()->load_reference_barrier(result); } #endif --- old/src/share/vm/opto/graphKit.cpp 2019-11-11 16:10:10.665342130 +0100 +++ new/src/share/vm/opto/graphKit.cpp 2019-11-11 16:10:10.579342135 +0100 @@ -1539,10 +1539,13 @@ switch (bs->kind()) { case BarrierSet::G1SATBCT: case BarrierSet::G1SATBCTLogging: - case BarrierSet::ShenandoahBarrierSet: g1_write_barrier_pre(do_load, obj, adr, adr_idx, val, val_type, pre_val, bt); break; - + case BarrierSet::ShenandoahBarrierSet: + if (ShenandoahSATBBarrier) { + g1_write_barrier_pre(do_load, obj, adr, adr_idx, val, val_type, pre_val, bt); + } + break; case BarrierSet::CardTableModRef: case BarrierSet::CardTableExtension: case BarrierSet::ModRef: @@ -1590,14 +1593,17 @@ case BarrierSet::G1SATBCTLogging: g1_write_barrier_post(store, obj, adr, adr_idx, val, bt, use_precise); break; - + case BarrierSet::ShenandoahBarrierSet: + if (ShenandoahStoreValEnqueueBarrier) { + g1_write_barrier_pre(false, NULL, NULL, max_juint, NULL, NULL, val, bt); + } + break; case BarrierSet::CardTableModRef: case BarrierSet::CardTableExtension: write_barrier_post(store, obj, adr, adr_idx, val, use_precise); break; case BarrierSet::ModRef: - case BarrierSet::ShenandoahBarrierSet: break; case BarrierSet::Other: @@ -3948,7 +3954,7 @@ if (UseShenandoahGC) { Node* gc_state = __ AddP(no_base, tls, __ ConX(in_bytes(JavaThread::gc_state_offset()))); Node* ld = __ load(__ ctrl(), gc_state, TypeInt::BYTE, T_BYTE, Compile::AliasIdxRaw); - marking = __ AndI(ld, __ ConI(ShenandoahHeap::MARKING)); + marking = __ AndI(ld, __ ConI(ShenandoahHeap::MARKING | ShenandoahHeap::TRAVERSAL)); assert(ShenandoahBarrierC2Support::is_gc_state_load(ld), "Should match the shape"); } else { assert(UseG1GC, "should be"); --- old/src/share/vm/opto/ifnode.cpp 2019-11-11 16:10:11.250342098 +0100 +++ new/src/share/vm/opto/ifnode.cpp 2019-11-11 16:10:11.164342103 +0100 @@ -626,7 +626,7 @@ cmpx->is_Cmp() && cmpx->in(2) == phase->intcon(0) && cmpx->in(1)->in(1)->is_shenandoah_state_load() && cmpx->in(1)->in(2)->is_Con() && - cmpx->in(1)->in(2) == phase->intcon(ShenandoahHeap::MARKING)) { + cmpx->in(1)->in(2) == phase->intcon(ShenandoahHeap::MARKING | ShenandoahHeap::TRAVERSAL)) { return true; } --- old/src/share/vm/prims/jni.cpp 2019-11-11 16:10:11.787342069 +0100 +++ new/src/share/vm/prims/jni.cpp 2019-11-11 16:10:11.702342073 +0100 @@ -2630,7 +2630,7 @@ // If G1 is enabled and we are accessing the value of the referent // field in a reference object then we need to register a non-null // referent with the SATB barrier. - if (UseG1GC || (UseShenandoahGC && ShenandoahSATBBarrier)) { + if (UseG1GC || (UseShenandoahGC && ShenandoahKeepAliveBarrier)) { bool needs_barrier = false; if (ret != NULL && --- old/src/share/vm/prims/jvm.cpp 2019-11-11 16:10:12.376342036 +0100 +++ new/src/share/vm/prims/jvm.cpp 2019-11-11 16:10:12.291342041 +0100 @@ -587,7 +587,7 @@ // If G1 is enabled then we need to register a non-null referent // with the SATB barrier. #if INCLUDE_ALL_GCS - if (UseG1GC || UseShenandoahGC) { + if (UseG1GC || (UseShenandoahGC && ShenandoahKeepAliveBarrier)) { oop referent = java_lang_ref_Reference::referent(clone); if (referent != NULL) { G1SATBCardTableModRefBS::enqueue(referent); --- old/src/share/vm/prims/jvmtiGetLoadedClasses.cpp 2019-11-11 16:10:12.953342004 +0100 +++ new/src/share/vm/prims/jvmtiGetLoadedClasses.cpp 2019-11-11 16:10:12.869342009 +0100 @@ -46,7 +46,7 @@ // to get notified about this potential resurrection, otherwise the marking // might not find the object. #if INCLUDE_ALL_GCS - if ((o != NULL) && (UseG1GC || UseShenandoahGC)) { + if ((o != NULL) && (UseG1GC || (UseShenandoahGC && ShenandoahKeepAliveBarrier))) { G1SATBCardTableModRefBS::enqueue(o); } #endif --- old/src/share/vm/prims/jvmtiTagMap.cpp 2019-11-11 16:10:13.477341975 +0100 +++ new/src/share/vm/prims/jvmtiTagMap.cpp 2019-11-11 16:10:13.395341980 +0100 @@ -1521,7 +1521,7 @@ oop o = entry->object(); assert(o != NULL && Universe::heap()->is_in_reserved(o), "sanity check"); #if INCLUDE_ALL_GCS - if (UseG1GC || UseShenandoahGC) { + if (UseG1GC || (UseShenandoahGC && ShenandoahKeepAliveBarrier)) { // The reference in this tag map could be the only (implicitly weak) // reference to that object. If we hand it out, we need to keep it live wrt // SATB marking similar to other j.l.ref.Reference referents. --- old/src/share/vm/prims/unsafe.cpp 2019-11-11 16:10:14.035341945 +0100 +++ new/src/share/vm/prims/unsafe.cpp 2019-11-11 16:10:13.949341949 +0100 @@ -218,7 +218,7 @@ static void ensure_satb_referent_alive(oop o, jlong offset, oop v) { #if INCLUDE_ALL_GCS - if ((UseG1GC || (UseShenandoahGC && ShenandoahSATBBarrier)) && v != NULL && is_java_lang_ref_Reference_access(o, offset)) { + if ((UseG1GC || (UseShenandoahGC && ShenandoahKeepAliveBarrier)) && v != NULL && is_java_lang_ref_Reference_access(o, offset)) { G1SATBCardTableModRefBS::enqueue(v); } #endif --- old/src/share/vm/runtime/arguments.cpp 2019-11-11 16:10:14.582341915 +0100 +++ new/src/share/vm/runtime/arguments.cpp 2019-11-11 16:10:14.495341919 +0100 @@ -1732,6 +1732,8 @@ FLAG_SET_DEFAULT(ShenandoahSATBBarrier, false); FLAG_SET_DEFAULT(ShenandoahLoadRefBarrier, false); + FLAG_SET_DEFAULT(ShenandoahKeepAliveBarrier, false); + FLAG_SET_DEFAULT(ShenandoahStoreValEnqueueBarrier, false); FLAG_SET_DEFAULT(ShenandoahCASBarrier, false); FLAG_SET_DEFAULT(ShenandoahCloneBarrier, false); @@ -1843,6 +1845,8 @@ if (ShenandoahVerifyOptoBarriers && (!FLAG_IS_DEFAULT(ShenandoahSATBBarrier) || !FLAG_IS_DEFAULT(ShenandoahLoadRefBarrier) || + !FLAG_IS_DEFAULT(ShenandoahKeepAliveBarrier) || + !FLAG_IS_DEFAULT(ShenandoahStoreValEnqueueBarrier) || !FLAG_IS_DEFAULT(ShenandoahCASBarrier) || !FLAG_IS_DEFAULT(ShenandoahCloneBarrier) )) { --- old/src/share/vm/runtime/jniHandles.cpp 2019-11-11 16:10:15.150341883 +0100 +++ new/src/share/vm/runtime/jniHandles.cpp 2019-11-11 16:10:15.064341888 +0100 @@ -116,7 +116,7 @@ oop result = jweak_ref(handle); result = guard_value(result); #if INCLUDE_ALL_GCS - if (result != NULL && (UseG1GC || (UseShenandoahGC && ShenandoahSATBBarrier))) { + if (result != NULL && (UseG1GC || (UseShenandoahGC && ShenandoahKeepAliveBarrier))) { G1SATBCardTableModRefBS::enqueue(result); } #endif // INCLUDE_ALL_GCS --- old/src/share/vm/runtime/thread.cpp 2019-11-11 16:10:15.686341854 +0100 +++ new/src/share/vm/runtime/thread.cpp 2019-11-11 16:10:15.598341859 +0100 @@ -1956,7 +1956,7 @@ // from the list of active threads. We must do this after any deferred // card marks have been flushed (above) so that any entries that are // added to the thread's dirty card queue as a result are not lost. - if (UseG1GC || (UseShenandoahGC && ShenandoahSATBBarrier)) { + if (UseG1GC || (UseShenandoahGC)) { flush_barrier_queues(); } if (UseShenandoahGC && UseTLAB && gclab().is_initialized()) { @@ -2046,7 +2046,7 @@ } #if INCLUDE_ALL_GCS - if (UseG1GC || (UseShenandoahGC && ShenandoahSATBBarrier)) { + if (UseG1GC || (UseShenandoahGC)) { flush_barrier_queues(); } if (UseShenandoahGC && UseTLAB && gclab().is_initialized()) { --- old/src/share/vm/runtime/vm_operations.hpp 2019-11-11 16:10:16.261341822 +0100 +++ new/src/share/vm/runtime/vm_operations.hpp 2019-11-11 16:10:16.177341827 +0100 @@ -97,6 +97,8 @@ template(ShenandoahInitMark) \ template(ShenandoahFinalMarkStartEvac) \ template(ShenandoahFinalEvac) \ + template(ShenandoahInitTraversalGC) \ + template(ShenandoahFinalTraversalGC) \ template(ShenandoahInitUpdateRefs) \ template(ShenandoahFinalUpdateRefs) \ template(ShenandoahDegeneratedGC) \ --- old/test/gc/shenandoah/TestAllocHumongousFragment.java 2019-11-11 16:10:16.794341793 +0100 +++ new/test/gc/shenandoah/TestAllocHumongousFragment.java 2019-11-11 16:10:16.708341797 +0100 @@ -41,6 +41,13 @@ * * @run main/othervm -verbose:gc -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -Xmx1g -Xms1g -XX:ShenandoahTargetNumRegions=2048 -XX:ShenandoahGCHeuristics=adaptive TestAllocHumongousFragment * @run main/othervm -verbose:gc -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -Xmx1g -Xms1g -XX:ShenandoahTargetNumRegions=2048 -XX:ShenandoahGCHeuristics=static TestAllocHumongousFragment + * + * @run main/othervm -verbose:gc -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -Xmx1g -Xms1g -XX:ShenandoahTargetNumRegions=2048 -XX:ShenandoahGCMode=traversal -XX:ShenandoahGCHeuristics=aggressive -XX:+ShenandoahOOMDuringEvacALot -XX:+ShenandoahVerify TestAllocHumongousFragment + * @run main/othervm -verbose:gc -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -Xmx1g -Xms1g -XX:ShenandoahTargetNumRegions=2048 -XX:ShenandoahGCMode=traversal -XX:ShenandoahGCHeuristics=aggressive -XX:+ShenandoahAllocFailureALot -XX:+ShenandoahVerify TestAllocHumongousFragment + * @run main/othervm -verbose:gc -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -Xmx1g -Xms1g -XX:ShenandoahTargetNumRegions=2048 -XX:ShenandoahGCMode=traversal -XX:ShenandoahGCHeuristics=aggressive -XX:+ShenandoahOOMDuringEvacALot TestAllocHumongousFragment + * @run main/othervm -verbose:gc -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -Xmx1g -Xms1g -XX:ShenandoahTargetNumRegions=2048 -XX:ShenandoahGCMode=traversal -XX:ShenandoahGCHeuristics=aggressive -XX:+ShenandoahAllocFailureALot TestAllocHumongousFragment + * @run main/othervm -verbose:gc -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -Xmx1g -Xms1g -XX:ShenandoahTargetNumRegions=2048 -XX:ShenandoahGCMode=traversal -XX:+ShenandoahVerify TestAllocHumongousFragment + * @run main/othervm -verbose:gc -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -Xmx1g -Xms1g -XX:ShenandoahTargetNumRegions=2048 -XX:ShenandoahGCMode=traversal TestAllocHumongousFragment */ import java.util.*; --- old/test/gc/shenandoah/TestAllocIntArrays.java 2019-11-11 16:10:17.323341764 +0100 +++ new/test/gc/shenandoah/TestAllocIntArrays.java 2019-11-11 16:10:17.238341768 +0100 @@ -44,6 +44,14 @@ * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -Xmx1g -Xms1g -XX:ShenandoahGCHeuristics=static TestAllocIntArrays * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -Xmx1g -Xms1g -XX:ShenandoahGCHeuristics=compact TestAllocIntArrays * + * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -Xmx1g -Xms1g -XX:ShenandoahGCMode=traversal -XX:ShenandoahGCHeuristics=aggressive -XX:+ShenandoahOOMDuringEvacALot -XX:+ShenandoahVerify TestAllocIntArrays + * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -Xmx1g -Xms1g -XX:ShenandoahGCMode=traversal -XX:ShenandoahGCHeuristics=aggressive -XX:+ShenandoahAllocFailureALot -XX:+ShenandoahVerify TestAllocIntArrays + * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -Xmx1g -Xms1g -XX:ShenandoahGCMode=traversal -XX:ShenandoahGCHeuristics=aggressive -XX:+ShenandoahOOMDuringEvacALot TestAllocIntArrays + * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -Xmx1g -Xms1g -XX:ShenandoahGCMode=traversal -XX:ShenandoahGCHeuristics=aggressive -XX:+ShenandoahAllocFailureALot TestAllocIntArrays + * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -Xmx1g -Xms1g -XX:ShenandoahGCMode=traversal -XX:ShenandoahGCHeuristics=aggressive TestAllocIntArrays + * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -Xmx1g -Xms1g -XX:ShenandoahGCMode=traversal -XX:+ShenandoahVerify TestAllocIntArrays + * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -Xmx1g -Xms1g -XX:ShenandoahGCMode=traversal TestAllocIntArrays + * * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -Xmx1g -Xms1g -XX:-UseTLAB -XX:+ShenandoahVerify TestAllocIntArrays */ --- old/test/gc/shenandoah/TestAllocObjectArrays.java 2019-11-11 16:10:17.852341734 +0100 +++ new/test/gc/shenandoah/TestAllocObjectArrays.java 2019-11-11 16:10:17.764341739 +0100 @@ -44,6 +44,14 @@ * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -Xmx1g -Xms1g -XX:ShenandoahGCHeuristics=static TestAllocObjectArrays * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -Xmx1g -Xms1g -XX:ShenandoahGCHeuristics=compact TestAllocObjectArrays * + * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -Xmx1g -Xms1g -XX:ShenandoahGCMode=traversal -XX:ShenandoahGCHeuristics=aggressive -XX:+ShenandoahOOMDuringEvacALot -XX:+ShenandoahVerify TestAllocObjectArrays + * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -Xmx1g -Xms1g -XX:ShenandoahGCMode=traversal -XX:ShenandoahGCHeuristics=aggressive -XX:+ShenandoahAllocFailureALot -XX:+ShenandoahVerify TestAllocObjectArrays + * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -Xmx1g -Xms1g -XX:ShenandoahGCMode=traversal -XX:ShenandoahGCHeuristics=aggressive -XX:+ShenandoahOOMDuringEvacALot TestAllocObjectArrays + * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -Xmx1g -Xms1g -XX:ShenandoahGCMode=traversal -XX:ShenandoahGCHeuristics=aggressive -XX:+ShenandoahAllocFailureALot TestAllocObjectArrays + * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -Xmx1g -Xms1g -XX:ShenandoahGCMode=traversal -XX:ShenandoahGCHeuristics=aggressive TestAllocObjectArrays + * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -Xmx1g -Xms1g -XX:ShenandoahGCMode=traversal -XX:+ShenandoahVerify TestAllocObjectArrays + * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -Xmx1g -Xms1g -XX:ShenandoahGCMode=traversal TestAllocObjectArrays + * * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -Xmx1g -Xms1g -XX:-UseTLAB -XX:+ShenandoahVerify TestAllocObjectArrays */ --- old/test/gc/shenandoah/TestAllocObjects.java 2019-11-11 16:10:18.376341706 +0100 +++ new/test/gc/shenandoah/TestAllocObjects.java 2019-11-11 16:10:18.291341710 +0100 @@ -43,6 +43,14 @@ * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -Xmx1g -Xms1g -XX:ShenandoahGCHeuristics=adaptive TestAllocObjects * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -Xmx1g -Xms1g -XX:ShenandoahGCHeuristics=static TestAllocObjects * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -Xmx1g -Xms1g -XX:ShenandoahGCHeuristics=compact TestAllocObjects + * + * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -Xmx1g -Xms1g -XX:ShenandoahGCMode=traversal -XX:ShenandoahGCHeuristics=aggressive -XX:+ShenandoahOOMDuringEvacALot -XX:+ShenandoahVerify TestAllocObjects + * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -Xmx1g -Xms1g -XX:ShenandoahGCMode=traversal -XX:ShenandoahGCHeuristics=aggressive -XX:+ShenandoahAllocFailureALot -XX:+ShenandoahVerify TestAllocObjects + * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -Xmx1g -Xms1g -XX:ShenandoahGCMode=traversal -XX:ShenandoahGCHeuristics=aggressive -XX:+ShenandoahOOMDuringEvacALot TestAllocObjects + * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -Xmx1g -Xms1g -XX:ShenandoahGCMode=traversal -XX:ShenandoahGCHeuristics=aggressive -XX:+ShenandoahAllocFailureALot TestAllocObjects + * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -Xmx1g -Xms1g -XX:ShenandoahGCMode=traversal -XX:ShenandoahGCHeuristics=aggressive TestAllocObjects + * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -Xmx1g -Xms1g -XX:ShenandoahGCMode=traversal -XX:+ShenandoahVerify TestAllocObjects + * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -Xmx1g -Xms1g -XX:ShenandoahGCMode=traversal TestAllocObjects */ import java.util.Random; --- old/test/gc/shenandoah/TestGCThreadGroups.java 2019-11-11 16:10:18.905341676 +0100 +++ new/test/gc/shenandoah/TestGCThreadGroups.java 2019-11-11 16:10:18.820341681 +0100 @@ -34,6 +34,9 @@ * @run main/othervm -XX:ConcGCThreads=2 -XX:ParallelGCThreads=4 -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -Xmx16m -XX:ShenandoahGCHeuristics=static -Dtarget=1000 TestGCThreadGroups * @run main/othervm -XX:ConcGCThreads=2 -XX:ParallelGCThreads=4 -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -Xmx16m -XX:ShenandoahGCHeuristics=compact -Dtarget=100 TestGCThreadGroups * @run main/othervm -XX:ConcGCThreads=2 -XX:ParallelGCThreads=4 -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -Xmx16m -XX:ShenandoahGCHeuristics=aggressive -Dtarget=100 TestGCThreadGroups + * + * @run main/othervm -XX:ConcGCThreads=2 -XX:ParallelGCThreads=4 -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -Xmx16m -XX:ShenandoahGCMode=traversal -Dtarget=100 TestGCThreadGroups + * @run main/othervm -XX:ConcGCThreads=2 -XX:ParallelGCThreads=4 -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -Xmx16m -XX:ShenandoahGCMode=traversal -XX:ShenandoahGCHeuristics=aggressive -Dtarget=100 TestGCThreadGroups */ public class TestGCThreadGroups { --- old/test/gc/shenandoah/TestHeapUncommit.java 2019-11-11 16:10:19.433341647 +0100 +++ new/test/gc/shenandoah/TestHeapUncommit.java 2019-11-11 16:10:19.347341652 +0100 @@ -39,6 +39,11 @@ * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -Xmx1g -XX:+ShenandoahUncommit -XX:ShenandoahGCHeuristics=static -XX:ShenandoahUncommitDelay=0 TestHeapUncommit * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -Xmx1g -XX:+ShenandoahUncommit -XX:ShenandoahGCHeuristics=compact -XX:ShenandoahUncommitDelay=0 TestHeapUncommit * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -Xmx1g -XX:+ShenandoahUncommit -XX:ShenandoahGCHeuristics=aggressive -XX:ShenandoahUncommitDelay=0 TestHeapUncommit + * + * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -Xmx1g -XX:+ShenandoahUncommit -XX:ShenandoahGCMode=traversal -XX:ShenandoahUncommitDelay=0 TestHeapUncommit + * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -Xmx1g -XX:+ShenandoahUncommit -XX:ShenandoahGCMode=traversal -XX:ShenandoahUncommitDelay=0 -XX:+ShenandoahVerify TestHeapUncommit + * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -Xmx1g -XX:+ShenandoahUncommit -XX:ShenandoahGCMode=traversal -XX:ShenandoahGCHeuristics=aggressive -XX:ShenandoahUncommitDelay=0 TestHeapUncommit + * */ import java.util.Random; --- old/test/gc/shenandoah/TestLotsOfCycles.java 2019-11-11 16:10:19.960341618 +0100 +++ new/test/gc/shenandoah/TestLotsOfCycles.java 2019-11-11 16:10:19.875341623 +0100 @@ -35,6 +35,11 @@ * @run main/othervm/timeout=480 -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -Xmx16m -XX:ShenandoahGCHeuristics=adaptive -Dtarget=10000 TestLotsOfCycles * @run main/othervm/timeout=480 -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -Xmx16m -XX:ShenandoahGCHeuristics=static -Dtarget=10000 TestLotsOfCycles * @run main/othervm/timeout=480 -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -Xmx16m -XX:ShenandoahGCHeuristics=compact -Dtarget=1000 TestLotsOfCycles + * + * @run main/othervm/timeout=480 -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -Xmx16m -XX:ShenandoahGCMode=traversal -XX:ShenandoahGCHeuristics=aggressive -XX:+ShenandoahOOMDuringEvacALot -Dtarget=1000 TestLotsOfCycles + * @run main/othervm/timeout=480 -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -Xmx16m -XX:ShenandoahGCMode=traversal -XX:ShenandoahGCHeuristics=aggressive -XX:+ShenandoahAllocFailureALot -Dtarget=1000 TestLotsOfCycles + * @run main/othervm/timeout=480 -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -Xmx16m -XX:ShenandoahGCMode=traversal -XX:ShenandoahGCHeuristics=aggressive -Dtarget=1000 TestLotsOfCycles + * @run main/othervm/timeout=480 -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -Xmx16m -XX:ShenandoahGCMode=traversal -Dtarget=1000 TestLotsOfCycles */ public class TestLotsOfCycles { --- old/test/gc/shenandoah/TestPeriodicGC.java 2019-11-11 16:10:20.486341589 +0100 +++ new/test/gc/shenandoah/TestPeriodicGC.java 2019-11-11 16:10:20.401341594 +0100 @@ -86,6 +86,26 @@ ); } + testWith("Short period with traversal mode", + true, + "-verbose:gc", + "-XX:+UnlockDiagnosticVMOptions", + "-XX:+UnlockExperimentalVMOptions", + "-XX:+UseShenandoahGC", + "-XX:ShenandoahGCMode=traversal", + "-XX:ShenandoahGuaranteedGCInterval=1000" + ); + + testWith("Long period with traversal mode", + false, + "-verbose:gc", + "-XX:+UnlockDiagnosticVMOptions", + "-XX:+UnlockExperimentalVMOptions", + "-XX:+UseShenandoahGC", + "-XX:ShenandoahGCMode=traversal", + "-XX:ShenandoahGuaranteedGCInterval=100000" // deliberately too long + ); + testWith("Short period with aggressive", false, "-verbose:gc", --- old/test/gc/shenandoah/TestRefprocSanity.java 2019-11-11 16:10:21.012341560 +0100 +++ new/test/gc/shenandoah/TestRefprocSanity.java 2019-11-11 16:10:20.925341565 +0100 @@ -29,6 +29,10 @@ * @run main/othervm -Xmx1g -Xms1g -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC TestRefprocSanity * @run main/othervm -Xmx1g -Xms1g -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:+ShenandoahVerify TestRefprocSanity * @run main/othervm -Xmx1g -Xms1g -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:ShenandoahGCHeuristics=aggressive TestRefprocSanity + * + * @run main/othervm -Xmx1g -Xms1g -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:ShenandoahGCMode=traversal TestRefprocSanity + * @run main/othervm -Xmx1g -Xms1g -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:ShenandoahGCMode=traversal -XX:+ShenandoahVerify TestRefprocSanity + * @run main/othervm -Xmx1g -Xms1g -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:ShenandoahGCMode=traversal -XX:ShenandoahGCHeuristics=aggressive TestRefprocSanity */ import java.lang.ref.*; --- old/test/gc/shenandoah/TestRegionSampling.java 2019-11-11 16:10:21.539341531 +0100 +++ new/test/gc/shenandoah/TestRegionSampling.java 2019-11-11 16:10:21.454341536 +0100 @@ -33,6 +33,9 @@ * * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -Xmx1g -Xms1g -XX:ShenandoahGCMode=passive -XX:+ShenandoahDegeneratedGC -XX:+ShenandoahRegionSampling TestRegionSampling * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -Xmx1g -Xms1g -XX:ShenandoahGCMode=passive -XX:-ShenandoahDegeneratedGC -XX:+ShenandoahRegionSampling TestRegionSampling + * + * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -Xmx1g -Xms1g -XX:ShenandoahGCMode=traversal -XX:ShenandoahGCHeuristics=aggressive -XX:+ShenandoahRegionSampling TestRegionSampling + * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -Xmx1g -Xms1g -XX:ShenandoahGCMode=traversal -XX:+ShenandoahRegionSampling TestRegionSampling */ public class TestRegionSampling { --- old/test/gc/shenandoah/TestRetainObjects.java 2019-11-11 16:10:22.063341502 +0100 +++ new/test/gc/shenandoah/TestRetainObjects.java 2019-11-11 16:10:21.978341507 +0100 @@ -42,6 +42,12 @@ * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -Xms1g -Xmx1g -XX:ShenandoahGCHeuristics=static TestRetainObjects * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -Xms1g -Xmx1g -XX:ShenandoahGCHeuristics=compact TestRetainObjects * + * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -Xms1g -Xmx1g -XX:ShenandoahGCMode=traversal -XX:ShenandoahGCHeuristics=aggressive -XX:+ShenandoahOOMDuringEvacALot TestRetainObjects + * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -Xms1g -Xmx1g -XX:ShenandoahGCMode=traversal -XX:ShenandoahGCHeuristics=aggressive -XX:+ShenandoahAllocFailureALot TestRetainObjects + * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -Xms1g -Xmx1g -XX:ShenandoahGCMode=traversal -XX:ShenandoahGCHeuristics=aggressive TestRetainObjects + * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -Xms1g -Xmx1g -XX:ShenandoahGCMode=traversal -XX:+ShenandoahVerify TestRetainObjects + * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -Xms1g -Xmx1g -XX:ShenandoahGCMode=traversal TestRetainObjects + * * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -Xms1g -Xmx1g -XX:-UseTLAB TestRetainObjects */ --- old/test/gc/shenandoah/TestSieveObjects.java 2019-11-11 16:10:22.591341473 +0100 +++ new/test/gc/shenandoah/TestSieveObjects.java 2019-11-11 16:10:22.505341478 +0100 @@ -42,6 +42,12 @@ * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -Xmx1g -Xms1g -XX:ShenandoahGCHeuristics=static TestSieveObjects * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -Xmx1g -Xms1g -XX:ShenandoahGCHeuristics=compact TestSieveObjects * + * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -Xmx1g -Xms1g -XX:ShenandoahGCMode=traversal -XX:ShenandoahGCHeuristics=aggressive -XX:+ShenandoahOOMDuringEvacALot TestSieveObjects + * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -Xmx1g -Xms1g -XX:ShenandoahGCMode=traversal -XX:ShenandoahGCHeuristics=aggressive -XX:+ShenandoahAllocFailureALot TestSieveObjects + * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -Xmx1g -Xms1g -XX:ShenandoahGCMode=traversal -XX:ShenandoahGCHeuristics=aggressive TestSieveObjects + * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -Xmx1g -Xms1g -XX:ShenandoahGCMode=traversal -XX:+ShenandoahVerify TestSieveObjects + * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -Xmx1g -Xms1g -XX:ShenandoahGCMode=traversal TestSieveObjects + * * @run main/othervm/timeout=240 -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -Xmx1g -Xms1g -XX:-UseTLAB -XX:+ShenandoahVerify TestSieveObjects */ --- old/test/gc/shenandoah/TestStringDedup.java 2019-11-11 16:10:23.120341444 +0100 +++ new/test/gc/shenandoah/TestStringDedup.java 2019-11-11 16:10:23.032341449 +0100 @@ -32,6 +32,9 @@ * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:+UseStringDeduplication -Xmx256M TestStringDedup * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:ShenandoahGCHeuristics=aggressive -XX:+UseStringDeduplication -Xmx256M TestStringDedup * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:ShenandoahGCHeuristics=compact -XX:+UseStringDeduplication -Xmx256M TestStringDedup + * + * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:ShenandoahGCMode=traversal -XX:ShenandoahGCHeuristics=aggressive -XX:+UseStringDeduplication -Xmx256M TestStringDedup + * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:ShenandoahGCMode=traversal -XX:+UseStringDeduplication -Xmx256M TestStringDedup */ import java.lang.reflect.*; --- old/test/gc/shenandoah/TestStringDedupStress.java 2019-11-11 16:10:23.646341415 +0100 +++ new/test/gc/shenandoah/TestStringDedupStress.java 2019-11-11 16:10:23.561341420 +0100 @@ -84,6 +84,30 @@ * -XX:ShenandoahGCHeuristics=aggressive -XX:ShenandoahUpdateRefsEarly=off -XX:+ShenandoahOOMDuringEvacALot -DtargetStrings=2000000 * -verbose:gc * TestStringDedupStress + * + * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:+UseStringDeduplication -Xmx1g + * -XX:ShenandoahGCMode=traversal + * -verbose:gc + * TestStringDedupStress + * + * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:+UseStringDeduplication -Xmx1g + * -XX:ShenandoahGCMode=traversal -XX:ShenandoahGCHeuristics=aggressive -DtargetStrings=2000000 + * -verbose:gc + * TestStringDedupStress + * + * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:+UseStringDeduplication -Xmx1g + * -XX:ShenandoahGCMode=traversal + * -XX:+ShenandoahOOMDuringEvacALot + * -DtargetStrings=2000000 + * -verbose:gc + * TestStringDedupStress + * + * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:+UseStringDeduplication -Xmx1g + * -XX:ShenandoahGCMode=traversal -XX:ShenandoahGCHeuristics=aggressive + * -XX:+ShenandoahOOMDuringEvacALot + * -DtargetStrings=2000000 + * -verbose:gc + * TestStringDedupStress */ import java.lang.management.*; --- old/test/gc/shenandoah/TestStringInternCleanup.java 2019-11-11 16:10:24.174341386 +0100 +++ new/test/gc/shenandoah/TestStringInternCleanup.java 2019-11-11 16:10:24.088341391 +0100 @@ -40,6 +40,10 @@ * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:+ClassUnloadingWithConcurrentMark -Xmx64m -XX:ShenandoahGCHeuristics=static TestStringInternCleanup * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:+ClassUnloadingWithConcurrentMark -Xmx64m -XX:ShenandoahGCHeuristics=compact TestStringInternCleanup * + * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:+ClassUnloadingWithConcurrentMark -Xmx64m -XX:ShenandoahGCMode=traversal -XX:+ShenandoahVerify TestStringInternCleanup + * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:+ClassUnloadingWithConcurrentMark -Xmx64m -XX:ShenandoahGCMode=traversal -XX:+ShenandoahVerify -XX:ShenandoahGCHeuristics=aggressive TestStringInternCleanup + * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:+ClassUnloadingWithConcurrentMark -Xmx64m -XX:ShenandoahGCMode=traversal TestStringInternCleanup + * * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:-ClassUnloadingWithConcurrentMark -Xmx64m TestStringInternCleanup */ --- old/test/gc/shenandoah/TestVerifyJCStress.java 2019-11-11 16:10:24.698341357 +0100 +++ new/test/gc/shenandoah/TestVerifyJCStress.java 2019-11-11 16:10:24.613341362 +0100 @@ -49,6 +49,12 @@ * -XX:ShenandoahGCHeuristics=static * -XX:+ShenandoahVerify -XX:+IgnoreUnrecognizedVMOptions -XX:+ShenandoahVerifyOptoBarriers * TestVerifyJCStress + * + * @run main/othervm -Xmx1g -Xms1g + * -XX:+UnlockExperimentalVMOptions -XX:+UnlockDiagnosticVMOptions -XX:+UseShenandoahGC + * -XX:ShenandoahGCMode=traversal + * -XX:+ShenandoahVerify -XX:+IgnoreUnrecognizedVMOptions -XX:+ShenandoahVerifyOptoBarriers + * TestVerifyJCStress */ import java.util.*; --- old/test/gc/shenandoah/TestWrongArrayMember.java 2019-11-11 16:10:25.225341328 +0100 +++ new/test/gc/shenandoah/TestWrongArrayMember.java 2019-11-11 16:10:25.142341333 +0100 @@ -26,6 +26,7 @@ * @key gc * * @run main/othervm -Xmx128m -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC TestWrongArrayMember + * @run main/othervm -Xmx128m -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:ShenandoahGCMode=traversal TestWrongArrayMember */ public class TestWrongArrayMember { --- old/test/gc/shenandoah/jni/TestCriticalNativeArgs.sh 2019-11-11 16:10:25.753341299 +0100 +++ new/test/gc/shenandoah/jni/TestCriticalNativeArgs.sh 2019-11-11 16:10:25.667341304 +0100 @@ -119,3 +119,27 @@ echo "Test Failed" exit 1 fi + +cmd="${TESTJAVA}${FS}bin${FS}java -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:ShenandoahGCMode=traversal -Xcomp -Xmx256M -XX:+CriticalJNINatives \ + -Djava.library.path=${THIS_DIR}${FS} TestCriticalNativeArgs" + +echo "$cmd" +eval $cmd + +if [ $? -ne 0 ] +then + echo "Test Failed" + exit 1 +fi + +cmd="${TESTJAVA}${FS}bin${FS}java -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:ShenandoahGCMode=traversal -XX:ShenandoahGCHeuristics=aggressive -Xcomp -Xmx512M -XX:+CriticalJNINatives \ + -Djava.library.path=${THIS_DIR}${FS} TestCriticalNativeArgs" + +echo "$cmd" +eval $cmd + +if [ $? -ne 0 ] +then + echo "Test Failed" + exit 1 +fi --- old/test/gc/shenandoah/jni/TestCriticalNativeStress.sh 2019-11-11 16:10:26.276341270 +0100 +++ new/test/gc/shenandoah/jni/TestCriticalNativeStress.sh 2019-11-11 16:10:26.192341275 +0100 @@ -120,3 +120,27 @@ echo "Test Failed" exit 1 fi + +cmd="${TESTJAVA}${FS}bin${FS}java -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:ShenandoahGCMode=traversal -Xcomp -Xmx256M -XX:+CriticalJNINatives \ + -Djava.library.path=${THIS_DIR}${FS} TestCriticalNativeStress" + +echo "$cmd" +eval $cmd + +if [ $? -ne 0 ] +then + echo "Test Failed" + exit 1 +fi + +cmd="${TESTJAVA}${FS}bin${FS}java -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:ShenandoahGCMode=traversal -XX:ShenandoahGCHeuristics=aggressive -Xcomp -Xmx512M -XX:+CriticalJNINatives \ + -Djava.library.path=${THIS_DIR}${FS} TestCriticalNativeStress" + +echo "$cmd" +eval $cmd + +if [ $? -ne 0 ] +then + echo "Test Failed" + exit 1 +fi --- old/test/gc/shenandoah/mxbeans/TestChurnNotifications.java 2019-11-11 16:10:26.792341242 +0100 +++ new/test/gc/shenandoah/mxbeans/TestChurnNotifications.java 2019-11-11 16:10:26.710341246 +0100 @@ -33,6 +33,9 @@ * @run main/othervm -Xmx128m -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:ShenandoahGCHeuristics=static -Dprecise=false TestChurnNotifications * @run main/othervm -Xmx128m -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:ShenandoahGCHeuristics=compact -Dprecise=false TestChurnNotifications * @run main/othervm -Xmx128m -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:ShenandoahGCHeuristics=aggressive -Dprecise=false TestChurnNotifications + * + * @run main/othervm -Xmx128m -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:ShenandoahGCMode=traversal -XX:ShenandoahGCHeuristics=aggressive -Dprecise=false TestChurnNotifications + * @run main/othervm -Xmx128m -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:ShenandoahGCMode=traversal -Dprecise=false TestChurnNotifications */ import java.util.*; --- old/test/gc/shenandoah/mxbeans/TestPauseNotifications.java 2019-11-11 16:10:27.321341213 +0100 +++ new/test/gc/shenandoah/mxbeans/TestPauseNotifications.java 2019-11-11 16:10:27.237341217 +0100 @@ -34,6 +34,9 @@ * @run main/othervm -Xmx128m -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:ShenandoahGCHeuristics=adaptive TestPauseNotifications * @run main/othervm -Xmx128m -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:ShenandoahGCHeuristics=static TestPauseNotifications * @run main/othervm -Xmx128m -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:ShenandoahGCHeuristics=compact TestPauseNotifications + * + * @run main/othervm -Xmx128m -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:ShenandoahGCMode=traversal -XX:ShenandoahGCHeuristics=aggressive TestPauseNotifications + * @run main/othervm -Xmx128m -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:ShenandoahGCMode=traversal TestPauseNotifications */ import java.util.*; --- old/test/gc/shenandoah/oom/TestClassLoaderLeak.java 2019-11-11 16:10:27.848341184 +0100 +++ new/test/gc/shenandoah/oom/TestClassLoaderLeak.java 2019-11-11 16:10:27.761341188 +0100 @@ -124,6 +124,7 @@ String[][][] modeHeuristics = new String[][][] { {{"normal"}, {"adaptive", "compact", "static", "aggressive"}}, + {{"traversal"}, {"adaptive", "aggressive"}}, {{"passive"}, {"passive"}} }; --- old/test/gc/shenandoah/options/TestExplicitGC.java 2019-11-11 16:10:28.378341154 +0100 +++ new/test/gc/shenandoah/options/TestExplicitGC.java 2019-11-11 16:10:28.290341159 +0100 @@ -1,4 +1,4 @@ -/* +'/* * Copyright (c) 2017, 2018, Red Hat, Inc. All rights reserved. * * This code is free software; you can redistribute it and/or modify it @@ -51,11 +51,16 @@ "Pause Full" }; - String[] concurrent = new String[] { + String[] concNormal = new String[] { "Pause Init Mark", "Pause Final Mark", }; + String[] concTraversal = new String[] { + "Pause Init Traversal", + "Pause Final Traversal", + }; + { ProcessBuilder pb = ProcessTools.createJavaProcessBuilder( "-XX:+UnlockExperimentalVMOptions", @@ -67,9 +72,12 @@ for (String p : full) { output.shouldNotContain(p); } - for (String p : concurrent) { + for (String p : concNormal) { output.shouldContain(p); } + for (String p : concTraversal) { + output.shouldNotContain(p); + } } { @@ -84,7 +92,10 @@ for (String p : full) { output.shouldNotContain(p); } - for (String p : concurrent) { + for (String p : concNormal) { + output.shouldNotContain(p); + } + for (String p : concTraversal) { output.shouldNotContain(p); } } @@ -101,9 +112,12 @@ for (String p : full) { output.shouldNotContain(p); } - for (String p : concurrent) { + for (String p : concNormal) { output.shouldContain(p); } + for (String p : concTraversal) { + output.shouldNotContain(p); + } } { @@ -111,6 +125,27 @@ "-XX:+UnlockExperimentalVMOptions", "-XX:+UseShenandoahGC", "-verbose:gc", + "-XX:+ExplicitGCInvokesConcurrent", + "-XX:ShenandoahGCMode=traversal", + TestExplicitGC.class.getName(), + "test"); + OutputAnalyzer output = new OutputAnalyzer(pb.start()); + for (String p : full) { + output.shouldNotContain(p); + } + for (String p : concNormal) { + output.shouldNotContain(p); + } + for (String p : concTraversal) { + output.shouldContain(p); + } + } + + { + ProcessBuilder pb = ProcessTools.createJavaProcessBuilder( + "-XX:+UnlockExperimentalVMOptions", + "-XX:+UseShenandoahGC", + "-verbose:gc", "-XX:-ExplicitGCInvokesConcurrent", TestExplicitGC.class.getName(), "test"); @@ -118,7 +153,10 @@ for (String p : full) { output.shouldContain(p); } - for (String p : concurrent) { + for (String p : concNormal) { + output.shouldNotContain(p); + } + for (String p : concTraversal) { output.shouldNotContain(p); } } --- old/test/gc/shenandoah/options/TestHeuristicsUnlock.java 2019-11-11 16:10:28.905341125 +0100 +++ new/test/gc/shenandoah/options/TestHeuristicsUnlock.java 2019-11-11 16:10:28.821341130 +0100 @@ -45,6 +45,8 @@ testWith("-XX:ShenandoahGCHeuristics=static", Mode.PRODUCT); testWith("-XX:ShenandoahGCHeuristics=compact", Mode.PRODUCT); + testWith("-XX:ShenandoahGCMode=traversal", Mode.PRODUCT); + testWith("-XX:ShenandoahGCHeuristics=aggressive", Mode.DIAGNOSTIC); testWith("-XX:ShenandoahGCMode=passive", Mode.DIAGNOSTIC); } --- old/test/gc/shenandoah/options/TestWrongBarrierDisable.java 2019-11-11 16:10:29.426341097 +0100 +++ new/test/gc/shenandoah/options/TestWrongBarrierDisable.java 2019-11-11 16:10:29.342341101 +0100 @@ -40,13 +40,22 @@ "ShenandoahCASBarrier", "ShenandoahCloneBarrier", "ShenandoahSATBBarrier", + "ShenandoahKeepAliveBarrier", + }; + + String[] traversal = { + "ShenandoahLoadRefBarrier", + "ShenandoahCASBarrier", + "ShenandoahCloneBarrier", }; shouldFailAll("-XX:ShenandoahGCHeuristics=adaptive", concurrent); shouldFailAll("-XX:ShenandoahGCHeuristics=static", concurrent); shouldFailAll("-XX:ShenandoahGCHeuristics=compact", concurrent); shouldFailAll("-XX:ShenandoahGCHeuristics=aggressive", concurrent); + shouldFailAll("-XX:ShenandoahGCMode=traversal", traversal); shouldPassAll("-XX:ShenandoahGCMode=passive", concurrent); + shouldPassAll("-XX:ShenandoahGCMode=passive", traversal); } private static void shouldFailAll(String h, String[] barriers) throws Exception { --- /dev/null 2019-11-11 08:42:03.603823722 +0100 +++ new/src/share/vm/gc_implementation/shenandoah/heuristics/shenandoahTraversalAggressiveHeuristics.cpp 2019-11-11 16:10:29.863341073 +0100 @@ -0,0 +1,126 @@ +/* + * Copyright (c) 2018, 2019, Red Hat, Inc. All rights reserved. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" + +#include "gc_implementation/shenandoah/heuristics/shenandoahTraversalAggressiveHeuristics.hpp" +#include "gc_implementation/shenandoah/shenandoahCollectionSet.hpp" +#include "gc_implementation/shenandoah/shenandoahFreeSet.hpp" +#include "gc_implementation/shenandoah/shenandoahHeap.inline.hpp" +#include "gc_implementation/shenandoah/shenandoahHeuristics.hpp" +#include "gc_implementation/shenandoah/shenandoahTraversalGC.hpp" +#include "gc_implementation/shenandoah/shenandoahLogging.hpp" +#include "utilities/quickSort.hpp" + +ShenandoahTraversalAggressiveHeuristics::ShenandoahTraversalAggressiveHeuristics() : ShenandoahHeuristics(), + _last_cset_select(0) { + // Do not shortcut evacuation + SHENANDOAH_ERGO_OVERRIDE_DEFAULT(ShenandoahImmediateThreshold, 100); + + // Aggressive runs with max speed for allocation, to capture races against mutator + SHENANDOAH_ERGO_DISABLE_FLAG(ShenandoahPacing); + + // Aggressive evacuates everything, so it needs as much evac space as it can get + SHENANDOAH_ERGO_ENABLE_FLAG(ShenandoahEvacReserveOverflow); + + // If class unloading is globally enabled, aggressive does unloading even with + // concurrent cycles. + if (ClassUnloading) { + SHENANDOAH_ERGO_OVERRIDE_DEFAULT(ShenandoahUnloadClassesFrequency, 1); + } + +} + +bool ShenandoahTraversalAggressiveHeuristics::is_experimental() { + return false; +} + +bool ShenandoahTraversalAggressiveHeuristics::is_diagnostic() { + return true; +} + +const char* ShenandoahTraversalAggressiveHeuristics::name() { + return "traversal-aggressive"; +} + +void ShenandoahTraversalAggressiveHeuristics::choose_collection_set(ShenandoahCollectionSet* collection_set) { + ShenandoahHeap* heap = ShenandoahHeap::heap(); + + ShenandoahTraversalGC* traversal_gc = heap->traversal_gc(); + + ShenandoahHeapRegionSet* traversal_set = traversal_gc->traversal_set(); + traversal_set->clear(); + + RegionData *data = get_region_data_cache(heap->num_regions()); + size_t cnt = 0; + + // Step 0. Prepare all regions + for (size_t i = 0; i < heap->num_regions(); i++) { + ShenandoahHeapRegion* r = heap->get_region(i); + if (r->used() > 0) { + if (r->is_regular()) { + data[cnt]._region = r; + data[cnt]._garbage = r->garbage(); + data[cnt]._seqnum_last_alloc = r->seqnum_last_alloc_mutator(); + cnt++; + } + traversal_set->add_region(r); + } + } + + for (size_t i = 0; i < cnt; i++) { + if (data[i]._seqnum_last_alloc > _last_cset_select) continue; + + ShenandoahHeapRegion* r = data[i]._region; + assert (r->is_regular(), "should have been filtered before"); + + if (r->garbage() > 0) { + assert(!collection_set->is_in(r), "must not yet be in cset"); + collection_set->add_region(r); + } + } + + // Clear liveness data + // TODO: Merge it with step 0, but save live data in RegionData before. + for (size_t i = 0; i < heap->num_regions(); i++) { + ShenandoahHeapRegion* r = heap->get_region(i); + if (r->used() > 0) { + r->clear_live_data(); + } + } + + collection_set->update_region_status(); + + _last_cset_select = ShenandoahHeapRegion::seqnum_current_alloc(); +} + +void ShenandoahTraversalAggressiveHeuristics::choose_collection_set_from_regiondata(ShenandoahCollectionSet* set, + RegionData* data, size_t data_size, + size_t free) { + ShouldNotReachHere(); +} + +bool ShenandoahTraversalAggressiveHeuristics::should_start_gc() const { + log_info(gc)("Trigger: Start next cycle immediately"); + return true; +} --- /dev/null 2019-11-11 08:42:03.603823722 +0100 +++ new/src/share/vm/gc_implementation/shenandoah/heuristics/shenandoahTraversalAggressiveHeuristics.hpp 2019-11-11 16:10:30.347341046 +0100 @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2018, Red Hat, Inc. All rights reserved. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_GC_SHENANDOAH_HEURISTICS_SHENANDOAHTRAVERSALAGGRESSIVEHEURISTICS_HPP +#define SHARE_GC_SHENANDOAH_HEURISTICS_SHENANDOAHTRAVERSALAGGRESSIVEHEURISTICS_HPP + +#include "gc_implementation/shenandoah/shenandoahHeuristics.hpp" + +class ShenandoahTraversalAggressiveHeuristics : public ShenandoahHeuristics { +private: + uint64_t _last_cset_select; + +protected: + virtual void choose_collection_set_from_regiondata(ShenandoahCollectionSet* set, + RegionData* data, size_t data_size, + size_t free); + +public: + ShenandoahTraversalAggressiveHeuristics(); + + virtual bool is_experimental(); + + virtual bool is_diagnostic(); + + virtual const char* name(); + + virtual void choose_collection_set(ShenandoahCollectionSet* collection_set); + virtual bool should_start_gc() const; +}; + +#endif // SHARE_GC_SHENANDOAH_HEURISTICS_SHENANDOAHTRAVERSALAGGRESSIVEHEURISTICS_HPP --- /dev/null 2019-11-11 08:42:03.603823722 +0100 +++ new/src/share/vm/gc_implementation/shenandoah/heuristics/shenandoahTraversalHeuristics.cpp 2019-11-11 16:10:30.830341019 +0100 @@ -0,0 +1,252 @@ +/* + * Copyright (c) 2018, 2019, Red Hat, Inc. All rights reserved. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" + +#include "gc_implementation/shenandoah/heuristics/shenandoahTraversalHeuristics.hpp" +#include "gc_implementation/shenandoah/shenandoahCollectionSet.hpp" +#include "gc_implementation/shenandoah/shenandoahFreeSet.hpp" +#include "gc_implementation/shenandoah/shenandoahHeap.inline.hpp" +#include "gc_implementation/shenandoah/shenandoahHeuristics.hpp" +#include "gc_implementation/shenandoah/shenandoahTraversalGC.hpp" +#include "gc_implementation/shenandoah/shenandoahLogging.hpp" +#include "utilities/quickSort.hpp" + +ShenandoahTraversalHeuristics::ShenandoahTraversalHeuristics() : ShenandoahHeuristics(), + _last_cset_select(0) {} + +bool ShenandoahTraversalHeuristics::is_experimental() { + return false; +} + +bool ShenandoahTraversalHeuristics::is_diagnostic() { + return false; +} + +const char* ShenandoahTraversalHeuristics::name() { + return "traversal"; +} + +void ShenandoahTraversalHeuristics::choose_collection_set(ShenandoahCollectionSet* collection_set) { + ShenandoahHeap* heap = ShenandoahHeap::heap(); + + ShenandoahTraversalGC* traversal_gc = heap->traversal_gc(); + + ShenandoahHeapRegionSet* traversal_set = traversal_gc->traversal_set(); + traversal_set->clear(); + + RegionData *data = get_region_data_cache(heap->num_regions()); + size_t cnt = 0; + + // Step 0. Prepare all regions + + for (size_t i = 0; i < heap->num_regions(); i++) { + ShenandoahHeapRegion* r = heap->get_region(i); + if (r->used() > 0) { + if (r->is_regular()) { + data[cnt]._region = r; + data[cnt]._garbage = r->garbage(); + data[cnt]._seqnum_last_alloc = r->seqnum_last_alloc_mutator(); + cnt++; + } + traversal_set->add_region(r); + } + } + + // The logic for cset selection is similar to that of adaptive: + // + // 1. We cannot get cset larger than available free space. Otherwise we guarantee OOME + // during evacuation, and thus guarantee full GC. In practice, we also want to let + // application to allocate something. This is why we limit CSet to some fraction of + // available space. In non-overloaded heap, max_cset would contain all plausible candidates + // over garbage threshold. + // + // 2. We should not get cset too low so that free threshold would not be met right + // after the cycle. Otherwise we get back-to-back cycles for no reason if heap is + // too fragmented. In non-overloaded non-fragmented heap min_garbage would be around zero. + // + // Therefore, we start by sorting the regions by garbage. Then we unconditionally add the best candidates + // before we meet min_garbage. Then we add all candidates that fit with a garbage threshold before + // we hit max_cset. When max_cset is hit, we terminate the cset selection. Note that in this scheme, + // ShenandoahGarbageThreshold is the soft threshold which would be ignored until min_garbage is hit. + // + // The significant complication is that liveness data was collected at the previous cycle, and only + // for those regions that were allocated before previous cycle started. + + size_t capacity = heap->max_capacity(); + size_t actual_free = heap->free_set()->available(); + size_t free_target = capacity / 100 * ShenandoahMinFreeThreshold; + size_t min_garbage = free_target > actual_free ? (free_target - actual_free) : 0; + size_t max_cset = (size_t)((1.0 * capacity / 100 * ShenandoahEvacReserve) / ShenandoahEvacWaste); + + log_info(gc, ergo)("Adaptive CSet Selection. Target Free: " SIZE_FORMAT "%s, Actual Free: " + SIZE_FORMAT "%s, Max CSet: " SIZE_FORMAT "%s, Min Garbage: " SIZE_FORMAT "%s", + byte_size_in_proper_unit(free_target), proper_unit_for_byte_size(free_target), + byte_size_in_proper_unit(actual_free), proper_unit_for_byte_size(actual_free), + byte_size_in_proper_unit(max_cset), proper_unit_for_byte_size(max_cset), + byte_size_in_proper_unit(min_garbage), proper_unit_for_byte_size(min_garbage)); + + // Better select garbage-first regions, and then older ones + QuickSort::sort(data, (int) cnt, compare_by_garbage_then_alloc_seq_ascending, false); + + size_t cur_cset = 0; + size_t cur_garbage = 0; + + size_t garbage_threshold = ShenandoahHeapRegion::region_size_bytes() / 100 * ShenandoahGarbageThreshold; + + // Step 1. Add trustworthy regions to collection set. + // + // We can trust live/garbage data from regions that were fully traversed during + // previous cycle. Even if actual liveness is different now, we can only have _less_ + // live objects, because dead objects are not resurrected. Which means we can undershoot + // the collection set, but not overshoot it. + + for (size_t i = 0; i < cnt; i++) { + if (data[i]._seqnum_last_alloc > _last_cset_select) continue; + + ShenandoahHeapRegion* r = data[i]._region; + assert (r->is_regular(), "should have been filtered before"); + + size_t new_garbage = cur_garbage + r->garbage(); + size_t new_cset = cur_cset + r->get_live_data_bytes(); + + if (new_cset > max_cset) { + break; + } + + if ((new_garbage < min_garbage) || (r->garbage() > garbage_threshold)) { + assert(!collection_set->is_in(r), "must not yet be in cset"); + collection_set->add_region(r); + cur_cset = new_cset; + cur_garbage = new_garbage; + } + } + + // Step 2. Try to catch some recently allocated regions for evacuation ride. + // + // Pessimistically assume we are going to evacuate the entire region. While this + // is very pessimistic and in most cases undershoots the collection set when regions + // are mostly dead, it also provides more safety against running into allocation + // failure when newly allocated regions are fully live. + + for (size_t i = 0; i < cnt; i++) { + if (data[i]._seqnum_last_alloc <= _last_cset_select) continue; + + ShenandoahHeapRegion* r = data[i]._region; + assert (r->is_regular(), "should have been filtered before"); + + // size_t new_garbage = cur_garbage + 0; (implied) + size_t new_cset = cur_cset + r->used(); + + if (new_cset > max_cset) { + break; + } + + assert(!collection_set->is_in(r), "must not yet be in cset"); + collection_set->add_region(r); + cur_cset = new_cset; + } + + // Step 3. Clear liveness data + // TODO: Merge it with step 0, but save live data in RegionData before. + for (size_t i = 0; i < heap->num_regions(); i++) { + ShenandoahHeapRegion* r = heap->get_region(i); + if (r->used() > 0) { + r->clear_live_data(); + } + } + + collection_set->update_region_status(); + + _last_cset_select = ShenandoahHeapRegion::seqnum_current_alloc(); +} + +bool ShenandoahTraversalHeuristics::should_start_gc() const { + ShenandoahHeap* heap = ShenandoahHeap::heap(); + assert(!heap->has_forwarded_objects(), "no forwarded objects here"); + + size_t capacity = heap->max_capacity(); + size_t available = heap->free_set()->available(); + + // Check if we are falling below the worst limit, time to trigger the GC, regardless of + // anything else. + size_t min_threshold = capacity / 100 * ShenandoahMinFreeThreshold; + if (available < min_threshold) { + log_info(gc)("Trigger: Free (" SIZE_FORMAT "%s) is below minimum threshold (" SIZE_FORMAT "%s)", + byte_size_in_proper_unit(available), proper_unit_for_byte_size(available), + byte_size_in_proper_unit(min_threshold), proper_unit_for_byte_size(min_threshold)); + return true; + } + + // Check if are need to learn a bit about the application + const size_t max_learn = ShenandoahLearningSteps; + if (_gc_times_learned < max_learn) { + size_t init_threshold = capacity / 100 * ShenandoahInitFreeThreshold; + if (available < init_threshold) { + log_info(gc)("Trigger: Learning " SIZE_FORMAT " of " SIZE_FORMAT ". Free (" SIZE_FORMAT "%s) is below initial threshold (" SIZE_FORMAT "%s)", + _gc_times_learned + 1, max_learn, + byte_size_in_proper_unit(available), proper_unit_for_byte_size(available), + byte_size_in_proper_unit(init_threshold), proper_unit_for_byte_size(init_threshold)); + return true; + } + } + + // Check if allocation headroom is still okay. This also factors in: + // 1. Some space to absorb allocation spikes + // 2. Accumulated penalties from Degenerated and Full GC + + size_t allocation_headroom = available; + + size_t spike_headroom = capacity / 100 * ShenandoahAllocSpikeFactor; + size_t penalties = capacity / 100 * _gc_time_penalties; + + allocation_headroom -= MIN2(allocation_headroom, spike_headroom); + allocation_headroom -= MIN2(allocation_headroom, penalties); + + double average_gc = _gc_time_history->avg(); + double time_since_last = time_since_last_gc(); + double allocation_rate = heap->bytes_allocated_since_gc_start() / time_since_last; + + if (average_gc > allocation_headroom / allocation_rate) { + log_info(gc)("Trigger: Average GC time (%.2f ms) is above the time for allocation rate (%.0f %sB/s) to deplete free headroom (" SIZE_FORMAT "%s)", + average_gc * 1000, + byte_size_in_proper_unit(allocation_rate), proper_unit_for_byte_size(allocation_rate), + byte_size_in_proper_unit(allocation_headroom), proper_unit_for_byte_size(allocation_headroom)); + log_info(gc, ergo)("Free headroom: " SIZE_FORMAT "%s (free) - " SIZE_FORMAT "%s (spike) - " SIZE_FORMAT "%s (penalties) = " SIZE_FORMAT "%s", + byte_size_in_proper_unit(available), proper_unit_for_byte_size(available), + byte_size_in_proper_unit(spike_headroom), proper_unit_for_byte_size(spike_headroom), + byte_size_in_proper_unit(penalties), proper_unit_for_byte_size(penalties), + byte_size_in_proper_unit(allocation_headroom), proper_unit_for_byte_size(allocation_headroom)); + return true; + } else if (ShenandoahHeuristics::should_start_gc()) { + return true; + } + + return false; +} + +void ShenandoahTraversalHeuristics::choose_collection_set_from_regiondata(ShenandoahCollectionSet* set, + RegionData* data, size_t data_size, + size_t free) { + ShouldNotReachHere(); +} --- /dev/null 2019-11-11 08:42:03.603823722 +0100 +++ new/src/share/vm/gc_implementation/shenandoah/heuristics/shenandoahTraversalHeuristics.hpp 2019-11-11 16:10:31.313340993 +0100 @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2018, Red Hat, Inc. All rights reserved. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_SHENANDOAH_HEURISTICS_SHENANDOAHTRAVERSALHEURISTICS_HPP +#define SHARE_VM_GC_SHENANDOAH_HEURISTICS_SHENANDOAHTRAVERSALHEURISTICS_HPP + +#include "gc_implementation/shenandoah/shenandoahHeuristics.hpp" + +class ShenandoahTraversalHeuristics : public ShenandoahHeuristics { +private: + uint64_t _last_cset_select; + +protected: + virtual void choose_collection_set_from_regiondata(ShenandoahCollectionSet* set, + RegionData* data, size_t data_size, + size_t free); + +public: + ShenandoahTraversalHeuristics(); + + virtual bool is_experimental(); + + virtual bool is_diagnostic(); + + virtual const char* name(); + + virtual void choose_collection_set(ShenandoahCollectionSet* collection_set); + + virtual bool should_start_gc() const; +}; + +#endif // SHARE_VM_GC_SHENANDOAH_HEURISTICS_SHENANDOAHTRAVERSALHEURISTICS_HPP --- /dev/null 2019-11-11 08:42:03.603823722 +0100 +++ new/src/share/vm/gc_implementation/shenandoah/shenandoahOopClosures.cpp 2019-11-11 16:10:31.800340966 +0100 @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2019, Red Hat, Inc. All rights reserved. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "gc_implementation/shenandoah/shenandoahHeap.inline.hpp" +#include "gc_implementation/shenandoah/shenandoahOopClosures.hpp" +#include "runtime/thread.hpp" + +ShenandoahTraversalSuperClosure::ShenandoahTraversalSuperClosure(ShenandoahObjToScanQueue* q, ReferenceProcessor* rp, ShenandoahStrDedupQueue* dq) : + MetadataAwareOopClosure(rp), + _traversal_gc(ShenandoahHeap::heap()->traversal_gc()), + _dedup_queue(dq), + _thread(Thread::current()), + _queue(q), + _mark_context(ShenandoahHeap::heap()->marking_context()) { +} --- /dev/null 2019-11-11 08:42:03.603823722 +0100 +++ new/src/share/vm/gc_implementation/shenandoah/shenandoahTraversalGC.cpp 2019-11-11 16:10:32.285340939 +0100 @@ -0,0 +1,1198 @@ +/* + * Copyright (c) 2018, 2019, Red Hat, Inc. All rights reserved. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" + +#include "memory/referenceProcessor.hpp" +#include "utilities/workgroup.hpp" +#include "gc_implementation/shenandoah/shenandoahBarrierSet.hpp" +#include "gc_implementation/shenandoah/shenandoahClosures.inline.hpp" +#include "gc_implementation/shenandoah/shenandoahCodeRoots.hpp" +#include "gc_implementation/shenandoah/shenandoahCollectionSet.hpp" +#include "gc_implementation/shenandoah/shenandoahCollectorPolicy.hpp" +#include "gc_implementation/shenandoah/shenandoahFreeSet.hpp" +#include "gc_implementation/shenandoah/shenandoahPhaseTimings.hpp" +#include "gc_implementation/shenandoah/shenandoahHeap.inline.hpp" +#include "gc_implementation/shenandoah/shenandoahHeapRegionSet.inline.hpp" +#include "gc_implementation/shenandoah/shenandoahHeuristics.hpp" +#include "gc_implementation/shenandoah/shenandoahMarkingContext.inline.hpp" +#include "gc_implementation/shenandoah/shenandoahOopClosures.inline.hpp" +#include "gc_implementation/shenandoah/shenandoahRootProcessor.hpp" +#include "gc_implementation/shenandoah/shenandoahStringDedup.hpp" +#include "gc_implementation/shenandoah/shenandoahTaskqueue.inline.hpp" +#include "gc_implementation/shenandoah/shenandoahTimingTracker.hpp" +#include "gc_implementation/shenandoah/shenandoahTraversalGC.hpp" +#include "gc_implementation/shenandoah/shenandoahUtils.hpp" +#include "gc_implementation/shenandoah/shenandoahVerifier.hpp" +#include "gc_implementation/shenandoah/shenandoahWorkGroup.hpp" + +#include "memory/iterator.hpp" +#include "memory/metaspace.hpp" +#include "memory/resourceArea.hpp" + +/** + * NOTE: We are using the SATB buffer in thread.hpp and satbMarkQueue.hpp, however, it is not an SATB algorithm. + * We're using the buffer as generic oop buffer to enqueue new values in concurrent oop stores, IOW, the algorithm + * is incremental-update-based. + * + * NOTE on interaction with TAMS: we want to avoid traversing new objects for + * several reasons: + * - We will not reclaim them in this cycle anyway, because they are not in the + * cset + * - It makes up for the bulk of work during final-pause + * - It also shortens the concurrent cycle because we don't need to + * pointlessly traverse through newly allocated objects. + * - As a nice side-effect, it solves the I-U termination problem (mutators + * cannot outrun the GC by allocating like crazy) + * - It is an easy way to achieve MWF. What MWF does is to also enqueue the + * target object of stores if it's new. Treating new objects live implicitely + * achieves the same, but without extra barriers. I think the effect of + * shortened final-pause (mentioned above) is the main advantage of MWF. In + * particular, we will not see the head of a completely new long linked list + * in final-pause and end up traversing huge chunks of the heap there. + * - We don't need to see/update the fields of new objects either, because they + * are either still null, or anything that's been stored into them has been + * evacuated+enqueued before (and will thus be treated later). + * + * We achieve this by setting TAMS for each region, and everything allocated + * beyond TAMS will be 'implicitely marked'. + * + * Gotchas: + * - While we want new objects to be implicitely marked, we don't want to count + * them alive. Otherwise the next cycle wouldn't pick them up and consider + * them for cset. This means that we need to protect such regions from + * getting accidentally thrashed at the end of traversal cycle. This is why I + * keep track of alloc-regions and check is_alloc_region() in the trashing + * code. + * - We *need* to traverse through evacuated objects. Those objects are + * pre-existing, and any references in them point to interesting objects that + * we need to see. We also want to count them as live, because we just + * determined that they are alive :-) I achieve this by upping TAMS + * concurrently for every gclab/gc-shared alloc before publishing the + * evacuated object. This way, the GC threads will not consider such objects + * implictely marked, and traverse through them as normal. + */ +class ShenandoahTraversalSATBBufferClosure : public SATBBufferClosure { +private: + ShenandoahObjToScanQueue* _queue; + ShenandoahTraversalGC* _traversal_gc; + ShenandoahHeap* const _heap; + +public: + ShenandoahTraversalSATBBufferClosure(ShenandoahObjToScanQueue* q) : + _queue(q), + _heap(ShenandoahHeap::heap()) + { } + + void do_buffer(void** buffer, size_t size) { + for (size_t i = 0; i < size; ++i) { + oop* p = (oop*) &buffer[i]; + oop obj = oopDesc::load_heap_oop(p); + shenandoah_assert_not_forwarded(p, obj); + if (_heap->marking_context()->mark(obj)) { + _queue->push(ShenandoahMarkTask(obj)); + } + } + } +}; + +class ShenandoahTraversalSATBThreadsClosure : public ThreadClosure { +private: + ShenandoahTraversalSATBBufferClosure* _satb_cl; + int _thread_parity; + +public: + ShenandoahTraversalSATBThreadsClosure(ShenandoahTraversalSATBBufferClosure* satb_cl) : + _satb_cl(satb_cl), + _thread_parity(SharedHeap::heap()->strong_roots_parity()) {} + + void do_thread(Thread* thread) { + if (thread->is_Java_thread()) { + if (thread->claim_oops_do(true, _thread_parity)) { + JavaThread* jt = (JavaThread*)thread; + jt->satb_mark_queue().apply_closure_and_empty(_satb_cl); + } + } else if (thread->is_VM_thread()) { + if (thread->claim_oops_do(true, _thread_parity)) { + JavaThread::satb_mark_queue_set().shared_satb_queue()->apply_closure_and_empty(_satb_cl); + } + } + } +}; + +// Like CLDToOopClosure, but clears has_modified_oops, so that we can record modified CLDs during traversal +// and remark them later during final-traversal. +class ShenandoahMarkCLDClosure : public CLDClosure { +private: + OopClosure* _cl; +public: + ShenandoahMarkCLDClosure(OopClosure* cl) : _cl(cl) {} + void do_cld(ClassLoaderData* cld) { + KlassToOopClosure klasscl(_cl); + cld->oops_do(_cl, &klasscl, true); + } +}; + +// Like CLDToOopClosure, but only process modified CLDs +class ShenandoahRemarkCLDClosure : public CLDClosure { +private: + OopClosure* _cl; +public: + ShenandoahRemarkCLDClosure(OopClosure* cl) : _cl(cl) {} + void do_cld(ClassLoaderData* cld) { + KlassToOopClosure klasscl(_cl); + cld->oops_do(_cl, &klasscl, true); + } +}; + +class ShenandoahInitTraversalCollectionTask : public AbstractGangTask { +private: + ShenandoahRootProcessor* _rp; + ShenandoahHeap* _heap; + ShenandoahCsetCodeRootsIterator _cset_coderoots; +public: + ShenandoahInitTraversalCollectionTask(ShenandoahRootProcessor* rp) : + AbstractGangTask("Shenandoah Init Traversal Collection"), + _rp(rp), + _heap(ShenandoahHeap::heap()), + _cset_coderoots(ShenandoahCodeRoots::cset_iterator()) {} + + void work(uint worker_id) { + ShenandoahParallelWorkerSession worker_session(worker_id); + + ShenandoahEvacOOMScope oom_evac_scope; + ShenandoahObjToScanQueueSet* queues = _heap->traversal_gc()->task_queues(); + ShenandoahObjToScanQueue* q = queues->queue(worker_id); + + bool process_refs = _heap->process_references(); + bool unload_classes = _heap->unload_classes(); + ReferenceProcessor* rp = NULL; + if (process_refs) { + rp = _heap->ref_processor(); + } + + // Step 1: Process ordinary GC roots. + { + ShenandoahTraversalRootsClosure roots_cl(q, rp); + ShenandoahMarkCLDClosure cld_cl(&roots_cl); + MarkingCodeBlobClosure code_cl(&roots_cl, CodeBlobToOopClosure::FixRelocations); + if (unload_classes) { + _rp->process_strong_roots(&roots_cl, NULL, &cld_cl, NULL, NULL, NULL, worker_id); + // Need to pre-evac code roots here. Otherwise we might see from-space constants. + ShenandoahWorkerTimings* worker_times = _heap->phase_timings()->worker_times(); + ShenandoahWorkerTimingsTracker timer(worker_times, ShenandoahPhaseTimings::CodeCacheRoots, worker_id); + _cset_coderoots.possibly_parallel_blobs_do(&code_cl); + } else { + _rp->process_all_roots(&roots_cl, process_refs ? NULL : &roots_cl, &cld_cl, &code_cl, NULL, worker_id); + } + } + } +}; + +class ShenandoahTraversalWeakRootsClosure : public OopClosure { +private: + ShenandoahHeap* const _heap; + + template + void do_oop_work(T* p) { + T o = oopDesc::load_heap_oop(p); + if (!oopDesc::is_null(o)) { + oop obj = oopDesc::decode_heap_oop_not_null(o); + if (_heap->in_collection_set(obj)) { + oop forw = ShenandoahBarrierSet::resolve_forwarded_not_null(obj); + if (obj == forw) { + forw = _heap->evacuate_object(obj, Thread::current()); + } + shenandoah_assert_forwarded_except(p, obj, _heap->cancelled_gc()); + // Update reference. + oopDesc::encode_store_heap_oop_not_null(p, forw); + } + } + } +public: + ShenandoahTraversalWeakRootsClosure() : + _heap(ShenandoahHeap::heap()) {} + + virtual void do_oop(oop* p) { do_oop_work(p); } + virtual void do_oop(narrowOop* p) { do_oop_work(p); } +}; + +class ShenandoahWeakInitTraversalCollectionTask : public AbstractGangTask { +private: + ShenandoahRootProcessor* _rp; + ShenandoahHeap* _heap; +public: + ShenandoahWeakInitTraversalCollectionTask(ShenandoahRootProcessor* rp) : + AbstractGangTask("Shenandoah Weak Init Traversal Collection"), + _rp(rp), + _heap(ShenandoahHeap::heap()) {} + + void work(uint worker_id) { + ShenandoahParallelWorkerSession worker_session(worker_id); + + ShenandoahEvacOOMScope oom_evac_scope; + ShenandoahTraversalWeakRootsClosure roots_cl; + CLDToOopClosure cld_cl(&roots_cl); + MarkingCodeBlobClosure code_cl(&roots_cl, CodeBlobToOopClosure::FixRelocations); + _rp->process_all_roots(&roots_cl, &roots_cl, &cld_cl, &code_cl, NULL, worker_id); + } +}; + +class ShenandoahConcurrentTraversalCollectionTask : public AbstractGangTask { +private: + ShenandoahTaskTerminator* _terminator; + ShenandoahHeap* _heap; +public: + ShenandoahConcurrentTraversalCollectionTask(ShenandoahTaskTerminator* terminator) : + AbstractGangTask("Shenandoah Concurrent Traversal Collection"), + _terminator(terminator), + _heap(ShenandoahHeap::heap()) {} + + void work(uint worker_id) { + ShenandoahConcurrentWorkerSession worker_session(worker_id); + ShenandoahEvacOOMScope oom_evac_scope; + ShenandoahTraversalGC* traversal_gc = _heap->traversal_gc(); + + // Drain all outstanding work in queues. + traversal_gc->main_loop(worker_id, _terminator); + } +}; + +class ShenandoahPreFinalTraversalCollectionTask : public AbstractGangTask { +private: + ShenandoahHeap* const _heap; +public: + ShenandoahPreFinalTraversalCollectionTask() : + AbstractGangTask("Shenandoah Pre Final Traversal Collection"), + _heap(ShenandoahHeap::heap()) {} + + void work(uint worker_id) { + ShenandoahParallelWorkerSession worker_session(worker_id); + ShenandoahTraversalGC* traversal_gc = _heap->traversal_gc(); + ShenandoahObjToScanQueueSet* queues = traversal_gc->task_queues(); + ShenandoahObjToScanQueue* q = queues->queue(worker_id); + + // Step 0: Drain outstanding SATB queues. + ShenandoahTraversalSATBBufferClosure satb_cl(q); + // Process remaining finished SATB buffers. + SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set(); + while (satb_mq_set.apply_closure_to_completed_buffer(&satb_cl)); + + ShenandoahTraversalSATBThreadsClosure tc(&satb_cl); + Threads::threads_do(&tc); + } +}; + +class ShenandoahFinalTraversalCollectionTask : public AbstractGangTask { +private: + ShenandoahRootProcessor* _rp; + ShenandoahTaskTerminator* _terminator; + ShenandoahHeap* _heap; +public: + ShenandoahFinalTraversalCollectionTask(ShenandoahRootProcessor* rp, ShenandoahTaskTerminator* terminator) : + AbstractGangTask("Shenandoah Final Traversal Collection"), + _rp(rp), + _terminator(terminator), + _heap(ShenandoahHeap::heap()) {} + + void work(uint worker_id) { + ShenandoahParallelWorkerSession worker_session(worker_id); + + ShenandoahEvacOOMScope oom_evac_scope; + ShenandoahTraversalGC* traversal_gc = _heap->traversal_gc(); + + ShenandoahObjToScanQueueSet* queues = traversal_gc->task_queues(); + ShenandoahObjToScanQueue* q = queues->queue(worker_id); + + bool process_refs = _heap->process_references(); + bool unload_classes = _heap->unload_classes(); + ReferenceProcessor* rp = NULL; + if (process_refs) { + rp = _heap->ref_processor(); + } + + // Step 1: Process GC roots. + // For oops in code roots, they are marked, evacuated, enqueued for further traversal, + // and the references to the oops are updated during init pause. New nmethods are handled + // in similar way during nmethod-register process. Therefore, we don't need to rescan code + // roots here. + if (!_heap->is_degenerated_gc_in_progress()) { + ShenandoahTraversalRootsClosure roots_cl(q, rp); + CLDToOopClosure cld_cl(&roots_cl); + if (unload_classes) { + ShenandoahRemarkCLDClosure weak_cld_cl(&roots_cl); + _rp->process_strong_roots(&roots_cl, process_refs ? NULL : &roots_cl, &cld_cl, &weak_cld_cl, NULL, NULL, worker_id); + } else { + _rp->process_all_roots(&roots_cl, process_refs ? NULL : &roots_cl, &cld_cl, NULL, NULL, worker_id); + } + } else { + ShenandoahTraversalDegenClosure roots_cl(q, rp); + CLDToOopClosure cld_cl(&roots_cl); + if (unload_classes) { + ShenandoahRemarkCLDClosure weak_cld_cl(&roots_cl); + _rp->process_strong_roots(&roots_cl, process_refs ? NULL : &roots_cl, &cld_cl, &weak_cld_cl, NULL, NULL, worker_id); + } else { + _rp->process_all_roots(&roots_cl, process_refs ? NULL : &roots_cl, &cld_cl, NULL, NULL, worker_id); + } + } + + { + ShenandoahWorkerTimings *worker_times = _heap->phase_timings()->worker_times(); + ShenandoahWorkerTimingsTracker timer(worker_times, ShenandoahPhaseTimings::FinishQueues, worker_id); + + // Step 3: Finally drain all outstanding work in queues. + traversal_gc->main_loop(worker_id, _terminator); + } + + } +}; + +ShenandoahTraversalGC::ShenandoahTraversalGC(ShenandoahHeap* heap, size_t num_regions) : + _heap(heap), + _task_queues(new ShenandoahObjToScanQueueSet(heap->max_workers())), + _traversal_set(ShenandoahHeapRegionSet()) { + + // Traversal does not support concurrent code root scanning + FLAG_SET_DEFAULT(ShenandoahConcurrentScanCodeRoots, false); + + uint num_queues = heap->max_workers(); + for (uint i = 0; i < num_queues; ++i) { + ShenandoahObjToScanQueue* task_queue = new ShenandoahObjToScanQueue(); + task_queue->initialize(); + _task_queues->register_queue(i, task_queue); + } +} + +ShenandoahTraversalGC::~ShenandoahTraversalGC() { +} + +void ShenandoahTraversalGC::prepare_regions() { + size_t num_regions = _heap->num_regions(); + ShenandoahMarkingContext* const ctx = _heap->marking_context(); + for (size_t i = 0; i < num_regions; i++) { + ShenandoahHeapRegion* region = _heap->get_region(i); + if (_heap->is_bitmap_slice_committed(region)) { + if (_traversal_set.is_in(i)) { + ctx->capture_top_at_mark_start(region); + region->clear_live_data(); + assert(ctx->is_bitmap_clear_range(region->bottom(), region->end()), "bitmap for traversal regions must be cleared"); + } else { + // Everything outside the traversal set is always considered live. + ctx->reset_top_at_mark_start(region); + } + } else { + // FreeSet may contain uncommitted empty regions, once they are recommitted, + // their TAMS may have old values, so reset them here. + ctx->reset_top_at_mark_start(region); + } + } +} + +void ShenandoahTraversalGC::prepare() { + if (UseTLAB) { + ShenandoahGCPhase phase(ShenandoahPhaseTimings::traversal_gc_accumulate_stats); + _heap->accumulate_statistics_tlabs(); + } + + { + ShenandoahGCPhase phase(ShenandoahPhaseTimings::traversal_gc_make_parsable); + _heap->make_parsable(true); + } + + if (UseTLAB) { + ShenandoahGCPhase phase(ShenandoahPhaseTimings::traversal_gc_resize_tlabs); + _heap->resize_tlabs(); + } + + assert(_heap->marking_context()->is_bitmap_clear(), "need clean mark bitmap"); + assert(!_heap->marking_context()->is_complete(), "should not be complete"); + + // About to choose the collection set, make sure we know which regions are pinned. + { + ShenandoahGCPhase phase_cleanup(ShenandoahPhaseTimings::traversal_gc_prepare_sync_pinned); + _heap->sync_pinned_region_status(); + } + + ShenandoahCollectionSet* collection_set = _heap->collection_set(); + { + ShenandoahHeapLocker lock(_heap->lock()); + + collection_set->clear(); + assert(collection_set->count() == 0, "collection set not clear"); + + // Find collection set + _heap->heuristics()->choose_collection_set(collection_set); + prepare_regions(); + + // Rebuild free set + _heap->free_set()->rebuild(); + } + + log_info(gc, ergo)("Collectable Garbage: " SIZE_FORMAT "%s, " SIZE_FORMAT "%s CSet, " SIZE_FORMAT " CSet regions", + byte_size_in_proper_unit(collection_set->garbage()), proper_unit_for_byte_size(collection_set->garbage()), + byte_size_in_proper_unit(collection_set->live_data()), proper_unit_for_byte_size(collection_set->live_data()), + collection_set->count()); +} + +void ShenandoahTraversalGC::init_traversal_collection() { + assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "STW traversal GC"); + + if (ShenandoahVerify) { + _heap->verifier()->verify_before_traversal(); + } + + if (VerifyBeforeGC) { + Universe::verify(); + } + + { + ShenandoahGCPhase phase_prepare(ShenandoahPhaseTimings::traversal_gc_prepare); + prepare(); + } + + _heap->set_concurrent_traversal_in_progress(true); + + bool process_refs = _heap->process_references(); + if (process_refs) { + ReferenceProcessor* rp = _heap->ref_processor(); + rp->enable_discovery(true /*verify_no_refs*/, true); + rp->setup_policy(_heap->collector_policy()->should_clear_all_soft_refs()); + } + + { + ShenandoahGCPhase phase_work(ShenandoahPhaseTimings::init_traversal_gc_work); + assert(_task_queues->is_empty(), "queues must be empty before traversal GC"); + TASKQUEUE_STATS_ONLY(_task_queues->reset_taskqueue_stats()); + + + { + COMPILER2_PRESENT(DerivedPointerTable::clear()); + uint nworkers = _heap->workers()->active_workers(); + task_queues()->reserve(nworkers); + ShenandoahRootProcessor rp(_heap, nworkers, ShenandoahPhaseTimings::init_traversal_gc_work); + ShenandoahInitTraversalCollectionTask traversal_task(&rp); + _heap->workers()->run_task(&traversal_task); + COMPILER2_PRESENT(DerivedPointerTable::update_pointers()); + } + + + if (_heap->unload_classes()) { + COMPILER2_PRESENT(DerivedPointerTable::clear()); + uint nworkers = _heap->workers()->active_workers(); + task_queues()->reserve(nworkers); + ShenandoahRootProcessor rp(_heap, nworkers, ShenandoahPhaseTimings::init_weak_traversal_gc_work); + ShenandoahWeakInitTraversalCollectionTask task(&rp); + _heap->workers()->run_task(&task); + COMPILER2_PRESENT(DerivedPointerTable::update_pointers()); + } + } + + if (ShenandoahPacing) { + _heap->pacer()->setup_for_traversal(); + } +} + +void ShenandoahTraversalGC::main_loop(uint w, ShenandoahTaskTerminator* t) { + ShenandoahObjToScanQueue* q = task_queues()->queue(w); + + // Initialize live data. + jushort* ld = _heap->get_liveness_cache(w); + + ReferenceProcessor* rp = NULL; + if (_heap->process_references()) { + rp = _heap->ref_processor(); + } + { + if (!_heap->is_degenerated_gc_in_progress()) { + if (_heap->unload_classes()) { + if (ShenandoahStringDedup::is_enabled()) { + ShenandoahStrDedupQueue* dq = ShenandoahStringDedup::queue(w); + ShenandoahTraversalMetadataDedupClosure cl(q, rp, dq); + main_loop_work(&cl, ld, w, t); + } else { + ShenandoahTraversalMetadataClosure cl(q, rp); + main_loop_work(&cl, ld, w, t); + } + } else { + if (ShenandoahStringDedup::is_enabled()) { + ShenandoahStrDedupQueue* dq = ShenandoahStringDedup::queue(w); + ShenandoahTraversalDedupClosure cl(q, rp, dq); + main_loop_work(&cl, ld, w, t); + } else { + ShenandoahTraversalClosure cl(q, rp); + main_loop_work(&cl, ld, w, t); + } + } + } else { + if (_heap->unload_classes()) { + if (ShenandoahStringDedup::is_enabled()) { + ShenandoahStrDedupQueue* dq = ShenandoahStringDedup::queue(w); + ShenandoahTraversalMetadataDedupDegenClosure cl(q, rp, dq); + main_loop_work(&cl, ld, w, t); + } else { + ShenandoahTraversalMetadataDegenClosure cl(q, rp); + main_loop_work(&cl, ld, w, t); + } + } else { + if (ShenandoahStringDedup::is_enabled()) { + ShenandoahStrDedupQueue* dq = ShenandoahStringDedup::queue(w); + ShenandoahTraversalDedupDegenClosure cl(q, rp, dq); + main_loop_work(&cl, ld, w, t); + } else { + ShenandoahTraversalDegenClosure cl(q, rp); + main_loop_work(&cl, ld, w, t); + } + } + } + } + + _heap->flush_liveness_cache(w); +} + +template +void ShenandoahTraversalGC::main_loop_work(T* cl, jushort* live_data, uint worker_id, ShenandoahTaskTerminator* terminator) { + ShenandoahObjToScanQueueSet* queues = task_queues(); + ShenandoahObjToScanQueue* q = queues->queue(worker_id); + ShenandoahConcurrentMark* conc_mark = _heap->concurrent_mark(); + + uintx stride = ShenandoahMarkLoopStride; + + ShenandoahMarkTask task; + + // Process outstanding queues, if any. + q = queues->claim_next(); + while (q != NULL) { + if (_heap->cancelled_gc()) { + return; + } + + for (uint i = 0; i < stride; i++) { + if (q->pop(task)) { + conc_mark->do_task(q, cl, live_data, &task); + } else { + assert(q->is_empty(), "Must be empty"); + q = queues->claim_next(); + break; + } + } + } + + if (_heap->cancelled_gc()) return; + + // Normal loop. + q = queues->queue(worker_id); + + ShenandoahTraversalSATBBufferClosure drain_satb(q); + SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set(); + + int seed = 17; + + while (true) { + if (_heap->cancelled_gc()) return; + + while (satb_mq_set.completed_buffers_num() > 0) { + satb_mq_set.apply_closure_to_completed_buffer(&drain_satb); + } + + uint work = 0; + for (uint i = 0; i < stride; i++) { + if (q->pop(task) || + queues->steal(worker_id, &seed, task)) { + conc_mark->do_task(q, cl, live_data, &task); + work++; + } else { + break; + } + } + + if (work == 0) { + // No more work, try to terminate + ShenandoahEvacOOMScopeLeaver oom_scope_leaver; + ShenandoahTerminationTimingsTracker term_tracker(worker_id); + ShenandoahTerminatorTerminator tt(_heap); + + if (terminator->offer_termination(&tt)) return; + } + } +} + +void ShenandoahTraversalGC::concurrent_traversal_collection() { + ShenandoahGCPhase phase_work(ShenandoahPhaseTimings::conc_traversal); + if (!_heap->cancelled_gc()) { + uint nworkers = _heap->workers()->active_workers(); + task_queues()->reserve(nworkers); + ShenandoahTerminationTracker tracker(ShenandoahPhaseTimings::conc_traversal_termination); + + ShenandoahTaskTerminator terminator(nworkers, task_queues()); + ShenandoahConcurrentTraversalCollectionTask task(&terminator); + _heap->workers()->run_task(&task); + } + + if (!_heap->cancelled_gc() && ShenandoahPreclean && _heap->process_references()) { + preclean_weak_refs(); + } +} + +void ShenandoahTraversalGC::final_traversal_collection() { + _heap->make_parsable(true); + + if (!_heap->cancelled_gc()) { + COMPILER2_PRESENT(DerivedPointerTable::clear()); + ShenandoahGCPhase phase_work(ShenandoahPhaseTimings::final_traversal_gc_work); + uint nworkers = _heap->workers()->active_workers(); + task_queues()->reserve(nworkers); + + // Finish traversal + { + SharedHeap::StrongRootsScope scope(_heap, true); + ShenandoahPreFinalTraversalCollectionTask task; + _heap->workers()->run_task(&task); + } + + ShenandoahRootProcessor rp(_heap, nworkers, ShenandoahPhaseTimings::final_traversal_gc_work); + ShenandoahTerminationTracker term(ShenandoahPhaseTimings::final_traversal_gc_termination); + ShenandoahTaskTerminator terminator(nworkers, task_queues()); + ShenandoahFinalTraversalCollectionTask task(&rp, &terminator); + _heap->workers()->run_task(&task); + COMPILER2_PRESENT(DerivedPointerTable::update_pointers()); + } + + if (!_heap->cancelled_gc() && _heap->process_references()) { + weak_refs_work(); + } + + if (!_heap->cancelled_gc()) { + fixup_roots(); + if (_heap->unload_classes()) { + _heap->unload_classes_and_cleanup_tables(false); + } else { + ShenandoahIsAliveSelector alive; + StringTable::unlink(alive.is_alive_closure()); + } + } + + if (!_heap->cancelled_gc()) { + assert(_task_queues->is_empty(), "queues must be empty after traversal GC"); + TASKQUEUE_STATS_ONLY(_task_queues->print_taskqueue_stats()); + TASKQUEUE_STATS_ONLY(_task_queues->reset_taskqueue_stats()); + + // No more marking expected + _heap->mark_complete_marking_context(); + + // Resize metaspace + MetaspaceGC::compute_new_size(); + + // Need to see that pinned region status is updated: newly pinned regions must not + // be trashed. New unpinned regions should be trashed. + { + ShenandoahGCPhase phase_cleanup(ShenandoahPhaseTimings::traversal_gc_sync_pinned); + _heap->sync_pinned_region_status(); + } + + // Still good? We can now trash the cset, and make final verification + { + ShenandoahGCPhase phase_cleanup(ShenandoahPhaseTimings::traversal_gc_cleanup); + ShenandoahHeapLocker lock(_heap->lock()); + + // Trash everything + // Clear immediate garbage regions. + size_t num_regions = _heap->num_regions(); + + ShenandoahHeapRegionSet* traversal_regions = traversal_set(); + ShenandoahFreeSet* free_regions = _heap->free_set(); + ShenandoahMarkingContext* const ctx = _heap->marking_context(); + free_regions->clear(); + for (size_t i = 0; i < num_regions; i++) { + ShenandoahHeapRegion* r = _heap->get_region(i); + bool not_allocated = ctx->top_at_mark_start(r) == r->top(); + + bool candidate = traversal_regions->is_in(r) && !r->has_live() && not_allocated; + if (r->is_humongous_start() && candidate) { + // Trash humongous. + HeapWord* humongous_obj = r->bottom(); + assert(!ctx->is_marked(oop(humongous_obj)), "must not be marked"); + r->make_trash_immediate(); + while (i + 1 < num_regions && _heap->get_region(i + 1)->is_humongous_continuation()) { + i++; + r = _heap->get_region(i); + assert(r->is_humongous_continuation(), "must be humongous continuation"); + r->make_trash_immediate(); + } + } else if (!r->is_empty() && candidate) { + // Trash regular. + assert(!r->is_humongous(), "handled above"); + assert(!r->is_trash(), "must not already be trashed"); + r->make_trash_immediate(); + } + } + _heap->collection_set()->clear(); + _heap->free_set()->rebuild(); + reset(); + } + + assert(_task_queues->is_empty(), "queues must be empty after traversal GC"); + _heap->set_concurrent_traversal_in_progress(false); + assert(!_heap->cancelled_gc(), "must not be cancelled when getting out here"); + + if (ShenandoahVerify) { + _heap->verifier()->verify_after_traversal(); + } + + if (VerifyAfterGC) { + Universe::verify(); + } + } +} + +class ShenandoahTraversalFixRootsClosure : public OopClosure { +private: + template + inline void do_oop_work(T* p) { + T o = oopDesc::load_heap_oop(p); + if (!oopDesc::is_null(o)) { + oop obj = oopDesc::decode_heap_oop_not_null(o); + oop forw = ShenandoahBarrierSet::resolve_forwarded_not_null(obj); + if (obj != forw) { + oopDesc::encode_store_heap_oop_not_null(p, forw); + } + } + } + +public: + inline void do_oop(oop* p) { do_oop_work(p); } + inline void do_oop(narrowOop* p) { do_oop_work(p); } +}; + +class ShenandoahTraversalFixRootsTask : public AbstractGangTask { +private: + ShenandoahRootProcessor* _rp; + +public: + ShenandoahTraversalFixRootsTask(ShenandoahRootProcessor* rp) : + AbstractGangTask("Shenandoah traversal fix roots"), + _rp(rp) { + assert(ShenandoahHeap::heap()->has_forwarded_objects(), "Must be"); + } + + void work(uint worker_id) { + ShenandoahParallelWorkerSession worker_session(worker_id); + ShenandoahTraversalFixRootsClosure cl; + MarkingCodeBlobClosure blobsCl(&cl, CodeBlobToOopClosure::FixRelocations); + CLDToOopClosure cldCl(&cl); + _rp->process_all_roots(&cl, &cl, &cldCl, &blobsCl, NULL, worker_id); + } +}; + +void ShenandoahTraversalGC::fixup_roots() { + COMPILER2_PRESENT(DerivedPointerTable::clear()); + ShenandoahRootProcessor rp(_heap, _heap->workers()->active_workers(), ShenandoahPhaseTimings::final_traversal_update_roots); + ShenandoahTraversalFixRootsTask update_roots_task(&rp); + _heap->workers()->run_task(&update_roots_task); + COMPILER2_PRESENT(DerivedPointerTable::update_pointers()); +} + +void ShenandoahTraversalGC::reset() { + _task_queues->clear(); +} + +ShenandoahObjToScanQueueSet* ShenandoahTraversalGC::task_queues() { + return _task_queues; +} + +class ShenandoahTraversalCancelledGCYieldClosure : public YieldClosure { +private: + ShenandoahHeap* const _heap; +public: + ShenandoahTraversalCancelledGCYieldClosure() : _heap(ShenandoahHeap::heap()) {}; + virtual bool should_return() { return _heap->cancelled_gc(); } +}; + +class ShenandoahTraversalPrecleanCompleteGCClosure : public VoidClosure { +public: + void do_void() { + ShenandoahHeap* sh = ShenandoahHeap::heap(); + ShenandoahTraversalGC* traversal_gc = sh->traversal_gc(); + assert(sh->process_references(), "why else would we be here?"); + ShenandoahTaskTerminator terminator(1, traversal_gc->task_queues()); + shenandoah_assert_rp_isalive_installed(); + traversal_gc->main_loop((uint) 0, &terminator); + } +}; + +class ShenandoahTraversalKeepAliveUpdateClosure : public OopClosure { +private: + ShenandoahObjToScanQueue* _queue; + Thread* _thread; + ShenandoahTraversalGC* _traversal_gc; + ShenandoahMarkingContext* const _mark_context; + + template + inline void do_oop_work(T* p) { + _traversal_gc->process_oop(p, _thread, _queue, _mark_context); + } + +public: + ShenandoahTraversalKeepAliveUpdateClosure(ShenandoahObjToScanQueue* q) : + _queue(q), _thread(Thread::current()), + _traversal_gc(ShenandoahHeap::heap()->traversal_gc()), + _mark_context(ShenandoahHeap::heap()->marking_context()) {} + + void do_oop(narrowOop* p) { do_oop_work(p); } + void do_oop(oop* p) { do_oop_work(p); } +}; + +class ShenandoahTraversalKeepAliveUpdateDegenClosure : public OopClosure { +private: + ShenandoahObjToScanQueue* _queue; + Thread* _thread; + ShenandoahTraversalGC* _traversal_gc; + ShenandoahMarkingContext* const _mark_context; + + template + inline void do_oop_work(T* p) { + _traversal_gc->process_oop(p, _thread, _queue, _mark_context); + } + +public: + ShenandoahTraversalKeepAliveUpdateDegenClosure(ShenandoahObjToScanQueue* q) : + _queue(q), _thread(Thread::current()), + _traversal_gc(ShenandoahHeap::heap()->traversal_gc()), + _mark_context(ShenandoahHeap::heap()->marking_context()) {} + + void do_oop(narrowOop* p) { do_oop_work(p); } + void do_oop(oop* p) { do_oop_work(p); } +}; + +class ShenandoahTraversalSingleThreadKeepAliveUpdateClosure : public OopClosure { +private: + ShenandoahObjToScanQueue* _queue; + Thread* _thread; + ShenandoahTraversalGC* _traversal_gc; + ShenandoahMarkingContext* const _mark_context; + + template + inline void do_oop_work(T* p) { + ShenandoahEvacOOMScope evac_scope; + _traversal_gc->process_oop(p, _thread, _queue, _mark_context); + } + +public: + ShenandoahTraversalSingleThreadKeepAliveUpdateClosure(ShenandoahObjToScanQueue* q) : + _queue(q), _thread(Thread::current()), + _traversal_gc(ShenandoahHeap::heap()->traversal_gc()), + _mark_context(ShenandoahHeap::heap()->marking_context()) {} + + void do_oop(narrowOop* p) { do_oop_work(p); } + void do_oop(oop* p) { do_oop_work(p); } +}; + +class ShenandoahTraversalSingleThreadKeepAliveUpdateDegenClosure : public OopClosure { +private: + ShenandoahObjToScanQueue* _queue; + Thread* _thread; + ShenandoahTraversalGC* _traversal_gc; + ShenandoahMarkingContext* const _mark_context; + + template + inline void do_oop_work(T* p) { + ShenandoahEvacOOMScope evac_scope; + _traversal_gc->process_oop(p, _thread, _queue, _mark_context); + } + +public: + ShenandoahTraversalSingleThreadKeepAliveUpdateDegenClosure(ShenandoahObjToScanQueue* q) : + _queue(q), _thread(Thread::current()), + _traversal_gc(ShenandoahHeap::heap()->traversal_gc()), + _mark_context(ShenandoahHeap::heap()->marking_context()) {} + + void do_oop(narrowOop* p) { do_oop_work(p); } + void do_oop(oop* p) { do_oop_work(p); } +}; + +class ShenandoahTraversalPrecleanTask : public AbstractGangTask { +private: + ReferenceProcessor* _rp; + +public: + ShenandoahTraversalPrecleanTask(ReferenceProcessor* rp) : + AbstractGangTask("Precleaning task"), + _rp(rp) {} + + void work(uint worker_id) { + assert(worker_id == 0, "The code below is single-threaded, only one worker is expected"); + ShenandoahParallelWorkerSession worker_session(worker_id); + ShenandoahEvacOOMScope oom_evac_scope; + + ShenandoahHeap* sh = ShenandoahHeap::heap(); + + ShenandoahObjToScanQueue* q = sh->traversal_gc()->task_queues()->queue(worker_id); + + ShenandoahForwardedIsAliveClosure is_alive; + ShenandoahTraversalCancelledGCYieldClosure yield; + ShenandoahTraversalPrecleanCompleteGCClosure complete_gc; + ShenandoahTraversalKeepAliveUpdateClosure keep_alive(q); + ResourceMark rm; + _rp->preclean_discovered_references(&is_alive, &keep_alive, + &complete_gc, &yield, + NULL, sh->shenandoah_policy()->tracer()->gc_id()); + } +}; + +void ShenandoahTraversalGC::preclean_weak_refs() { + // Pre-cleaning weak references before diving into STW makes sense at the + // end of concurrent mark. This will filter out the references which referents + // are alive. Note that ReferenceProcessor already filters out these on reference + // discovery, and the bulk of work is done here. This phase processes leftovers + // that missed the initial filtering, i.e. when referent was marked alive after + // reference was discovered by RP. + + assert(_heap->process_references(), "sanity"); + assert(!_heap->is_degenerated_gc_in_progress(), "must be in concurrent non-degenerated phase"); + + // Shortcut if no references were discovered to avoid winding up threads. + ReferenceProcessor* rp = _heap->ref_processor(); + ReferenceProcessorMTDiscoveryMutator fix_mt_discovery(rp, false); + + shenandoah_assert_rp_isalive_not_installed(); + ShenandoahForwardedIsAliveClosure is_alive; + ReferenceProcessorIsAliveMutator fix_isalive(rp, &is_alive); + + assert(task_queues()->is_empty(), "Should be empty"); + + // Execute precleaning in the worker thread: it will give us GCLABs, String dedup + // queues and other goodies. When upstream ReferenceProcessor starts supporting + // parallel precleans, we can extend this to more threads. + ShenandoahPushWorkerScope scope(_heap->workers(), 1, /* check_workers = */ false); + + WorkGang* workers = _heap->workers(); + uint nworkers = workers->active_workers(); + assert(nworkers == 1, "This code uses only a single worker"); + task_queues()->reserve(nworkers); + + ShenandoahTraversalPrecleanTask task(rp); + workers->run_task(&task); + + assert(_heap->cancelled_gc() || task_queues()->is_empty(), "Should be empty"); +} + +// Weak Reference Closures +class ShenandoahTraversalDrainMarkingStackClosure: public VoidClosure { + uint _worker_id; + ShenandoahTaskTerminator* _terminator; + bool _reset_terminator; + +public: + ShenandoahTraversalDrainMarkingStackClosure(uint worker_id, ShenandoahTaskTerminator* t, bool reset_terminator = false): + _worker_id(worker_id), + _terminator(t), + _reset_terminator(reset_terminator) { + } + + void do_void() { + assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint"); + + ShenandoahHeap* sh = ShenandoahHeap::heap(); + ShenandoahTraversalGC* traversal_gc = sh->traversal_gc(); + assert(sh->process_references(), "why else would we be here?"); + shenandoah_assert_rp_isalive_installed(); + + traversal_gc->main_loop(_worker_id, _terminator); + + if (_reset_terminator) { + _terminator->reset_for_reuse(); + } + } +}; + +class ShenandoahTraversalSingleThreadedDrainMarkingStackClosure: public VoidClosure { + uint _worker_id; + ShenandoahTaskTerminator* _terminator; + bool _reset_terminator; + +public: + ShenandoahTraversalSingleThreadedDrainMarkingStackClosure(uint worker_id, ShenandoahTaskTerminator* t, bool reset_terminator = false): + _worker_id(worker_id), + _terminator(t), + _reset_terminator(reset_terminator) { + } + + void do_void() { + assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint"); + + ShenandoahHeap* sh = ShenandoahHeap::heap(); + ShenandoahTraversalGC* traversal_gc = sh->traversal_gc(); + assert(sh->process_references(), "why else would we be here?"); + shenandoah_assert_rp_isalive_installed(); + + ShenandoahEvacOOMScope evac_scope; + traversal_gc->main_loop(_worker_id, _terminator); + + if (_reset_terminator) { + _terminator->reset_for_reuse(); + } + } +}; + +void ShenandoahTraversalGC::weak_refs_work() { + assert(_heap->process_references(), "sanity"); + + ShenandoahPhaseTimings::Phase phase_root = ShenandoahPhaseTimings::weakrefs; + + ShenandoahGCPhase phase(phase_root); + + ReferenceProcessor* rp = _heap->ref_processor(); + + // NOTE: We cannot shortcut on has_discovered_references() here, because + // we will miss marking JNI Weak refs then, see implementation in + // ReferenceProcessor::process_discovered_references. + weak_refs_work_doit(); + + rp->verify_no_references_recorded(); + assert(!rp->discovery_enabled(), "Post condition"); + +} + +class ShenandoahTraversalRefProcTaskProxy : public AbstractGangTask { +private: + AbstractRefProcTaskExecutor::ProcessTask& _proc_task; + ShenandoahTaskTerminator* _terminator; + +public: + ShenandoahTraversalRefProcTaskProxy(AbstractRefProcTaskExecutor::ProcessTask& proc_task, + ShenandoahTaskTerminator* t) : + AbstractGangTask("Process reference objects in parallel"), + _proc_task(proc_task), + _terminator(t) { + } + + void work(uint worker_id) { + ShenandoahEvacOOMScope oom_evac_scope; + assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint"); + ShenandoahHeap* heap = ShenandoahHeap::heap(); + ShenandoahTraversalDrainMarkingStackClosure complete_gc(worker_id, _terminator); + + ShenandoahForwardedIsAliveClosure is_alive; + if (!heap->is_degenerated_gc_in_progress()) { + ShenandoahTraversalKeepAliveUpdateClosure keep_alive(heap->traversal_gc()->task_queues()->queue(worker_id)); + _proc_task.work(worker_id, is_alive, keep_alive, complete_gc); + } else { + ShenandoahTraversalKeepAliveUpdateDegenClosure keep_alive(heap->traversal_gc()->task_queues()->queue(worker_id)); + _proc_task.work(worker_id, is_alive, keep_alive, complete_gc); + } + } +}; + +class ShenandoahTraversalRefEnqueueTaskProxy : public AbstractGangTask { +private: + AbstractRefProcTaskExecutor::EnqueueTask& _enqueue_task; + +public: + ShenandoahTraversalRefEnqueueTaskProxy(AbstractRefProcTaskExecutor::EnqueueTask& enqueue_task) : + AbstractGangTask("Enqueue reference objects in parallel"), + _enqueue_task(enqueue_task) { + } + + void work(uint worker_id) { + _enqueue_task.work(worker_id); + } +}; + +class ShenandoahTraversalRefProcTaskExecutor : public AbstractRefProcTaskExecutor { +private: + WorkGang* _workers; + +public: + ShenandoahTraversalRefProcTaskExecutor(WorkGang* workers) : _workers(workers) {} + + // Executes a task using worker threads. + void execute(ProcessTask& task) { + assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint"); + + // Shortcut execution if task is empty. + // This should be replaced with the generic ReferenceProcessor shortcut, + // see JDK-8181214, JDK-8043575, JDK-6938732. + if (task.is_empty()) { + return; + } + + ShenandoahHeap* heap = ShenandoahHeap::heap(); + ShenandoahTraversalGC* traversal_gc = heap->traversal_gc(); + uint nworkers = _workers->active_workers(); + traversal_gc->task_queues()->reserve(nworkers); + + ShenandoahTaskTerminator terminator(nworkers, traversal_gc->task_queues()); + ShenandoahTraversalRefProcTaskProxy proc_task_proxy(task, &terminator); + _workers->run_task(&proc_task_proxy); + } + + void execute(EnqueueTask& task) { + ShenandoahTraversalRefEnqueueTaskProxy enqueue_task_proxy(task); + _workers->run_task(&enqueue_task_proxy); + } +}; + +void ShenandoahTraversalGC::weak_refs_work_doit() { + ReferenceProcessor* rp = _heap->ref_processor(); + + ShenandoahPhaseTimings::Phase phase_process = ShenandoahPhaseTimings::weakrefs_process; + ShenandoahPhaseTimings::Phase phase_enqueue = ShenandoahPhaseTimings::weakrefs_enqueue; + + shenandoah_assert_rp_isalive_not_installed(); + ShenandoahForwardedIsAliveClosure is_alive; + ReferenceProcessorIsAliveMutator fix_isalive(rp, &is_alive); + + WorkGang* workers = _heap->workers(); + uint nworkers = workers->active_workers(); + + rp->setup_policy(_heap->collector_policy()->should_clear_all_soft_refs()); + rp->set_active_mt_degree(nworkers); + + assert(task_queues()->is_empty(), "Should be empty"); + + // complete_gc and keep_alive closures instantiated here are only needed for + // single-threaded path in RP. They share the queue 0 for tracking work, which + // simplifies implementation. Since RP may decide to call complete_gc several + // times, we need to be able to reuse the terminator. + uint serial_worker_id = 0; + ShenandoahTaskTerminator terminator(1, task_queues()); + ShenandoahTraversalSingleThreadedDrainMarkingStackClosure complete_gc(serial_worker_id, &terminator, /* reset_terminator = */ true); + ShenandoahTraversalRefProcTaskExecutor executor(workers); + + if (!_heap->is_degenerated_gc_in_progress()) { + ShenandoahTraversalSingleThreadKeepAliveUpdateClosure keep_alive(task_queues()->queue(serial_worker_id)); + rp->process_discovered_references(&is_alive, &keep_alive, + &complete_gc, &executor, + NULL, _heap->shenandoah_policy()->tracer()->gc_id()); + } else { + ShenandoahTraversalSingleThreadKeepAliveUpdateDegenClosure keep_alive(task_queues()->queue(serial_worker_id)); + rp->process_discovered_references(&is_alive, &keep_alive, + &complete_gc, &executor, + NULL, _heap->shenandoah_policy()->tracer()->gc_id()); + } + assert(task_queues()->is_empty() || _heap->cancelled_gc(), "Should be empty"); + + { + ShenandoahGCPhase phase(phase_enqueue); + rp->enqueue_discovered_references(&executor); + } +} --- /dev/null 2019-11-11 08:42:03.603823722 +0100 +++ new/src/share/vm/gc_implementation/shenandoah/shenandoahTraversalGC.hpp 2019-11-11 16:10:32.776340912 +0100 @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2018, Red Hat, Inc. All rights reserved. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_SHENANDOAH_SHENANDOAHTRAVERSALGC_HPP +#define SHARE_VM_GC_SHENANDOAH_SHENANDOAHTRAVERSALGC_HPP + +#include "memory/allocation.hpp" +#include "gc_implementation/shenandoah/shenandoahHeap.hpp" +#include "gc_implementation/shenandoah/shenandoahHeapRegionSet.hpp" +#include "gc_implementation/shenandoah/shenandoahTaskqueue.hpp" +#include "runtime/thread.hpp" + +class ShenandoahTraversalGC : public CHeapObj { +private: + ShenandoahHeap* const _heap; + ShenandoahObjToScanQueueSet* const _task_queues; + ShenandoahHeapRegionSet _traversal_set; + +public: + ShenandoahTraversalGC(ShenandoahHeap* heap, size_t num_regions); + ~ShenandoahTraversalGC(); + + ShenandoahHeapRegionSet* traversal_set() { return &_traversal_set; } + + void reset(); + void prepare(); + void init_traversal_collection(); + void concurrent_traversal_collection(); + void final_traversal_collection(); + + template + inline void process_oop(T* p, Thread* thread, ShenandoahObjToScanQueue* queue, ShenandoahMarkingContext* const mark_context, ShenandoahStrDedupQueue* dq = NULL); + + ShenandoahObjToScanQueueSet* task_queues(); + + void main_loop(uint worker_id, ShenandoahTaskTerminator* terminator); + +private: + void prepare_regions(); + + template + void main_loop_work(T* cl, jushort* live_data, uint worker_id, ShenandoahTaskTerminator* terminator); + + void preclean_weak_refs(); + void weak_refs_work(); + void weak_refs_work_doit(); + + void fixup_roots(); +}; + +#endif // SHARE_VM_GC_SHENANDOAH_SHENANDOAHTRAVERSALGC_HPP --- /dev/null 2019-11-11 08:42:03.603823722 +0100 +++ new/src/share/vm/gc_implementation/shenandoah/shenandoahTraversalGC.inline.hpp 2019-11-11 16:10:33.262340885 +0100 @@ -0,0 +1,83 @@ +/* + * Copyright (c) 2018, Red Hat, Inc. All rights reserved. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_SHENANDOAH_SHENANDOAHTRAVERSALGC_INLINE_HPP +#define SHARE_VM_GC_SHENANDOAH_SHENANDOAHTRAVERSALGC_INLINE_HPP + +#include "gc_implementation/shenandoah/shenandoahAsserts.hpp" +#include "gc_implementation/shenandoah/shenandoahBarrierSet.inline.hpp" +#include "gc_implementation/shenandoah/shenandoahHeap.inline.hpp" +#include "gc_implementation/shenandoah/shenandoahMarkingContext.inline.hpp" +#include "gc_implementation/shenandoah/shenandoahStringDedup.hpp" +#include "gc_implementation/shenandoah/shenandoahTraversalGC.hpp" +#include "gc_implementation/shenandoah/shenandoahTaskqueue.hpp" +#include "gc_implementation/shenandoah/shenandoahTaskqueue.inline.hpp" +#include "oops/oop.inline.hpp" + +template +void ShenandoahTraversalGC::process_oop(T* p, Thread* thread, ShenandoahObjToScanQueue* queue, ShenandoahMarkingContext* const mark_context, ShenandoahStrDedupQueue* dq) { + T o = oopDesc::load_heap_oop(p); + if (!oopDesc::is_null(o)) { + oop obj = oopDesc::decode_heap_oop_not_null(o); + if (DEGEN) { + assert(!ATOMIC_UPDATE, "Degen path assumes non-atomic updates"); + oop forw = ShenandoahBarrierSet::resolve_forwarded_not_null(obj); + if (obj != forw) { + // Update reference. + oopDesc::encode_store_heap_oop_not_null(p, forw); + } + obj = forw; + } else if (_heap->in_collection_set(obj)) { + oop forw = ShenandoahBarrierSet::resolve_forwarded_not_null(obj); + if (obj == forw) { + forw = _heap->evacuate_object(obj, thread); + } + shenandoah_assert_forwarded_except(p, obj, _heap->cancelled_gc()); + // Update reference. + if (ATOMIC_UPDATE) { + _heap->cas_oop(forw, p, obj); + } else { + oopDesc::encode_store_heap_oop_not_null(p, forw); + } + obj = forw; + } + + shenandoah_assert_not_forwarded(p, obj); + shenandoah_assert_not_in_cset_except(p, obj, _heap->cancelled_gc()); + + if (mark_context->mark(obj)) { + bool succeeded = queue->push(ShenandoahMarkTask(obj)); + assert(succeeded, "must succeed to push to task queue"); + + if (STRING_DEDUP && ShenandoahStringDedup::is_candidate(obj) && !_heap->cancelled_gc()) { + assert(ShenandoahStringDedup::is_enabled(), "Must be enabled"); + // Only dealing with to-space string, so that we can avoid evac-oom protocol, which is costly here. + shenandoah_assert_not_in_cset(p, obj); + assert(dq != NULL, "Dedup queue not set"); + ShenandoahStringDedup::enqueue_candidate(obj, dq); + } + } + } +} + +#endif // SHARE_VM_GC_SHENANDOAH_SHENANDOAHTRAVERSALGC_INLINE_HPP --- /dev/null 2019-11-11 08:42:03.603823722 +0100 +++ new/src/share/vm/gc_implementation/shenandoah/shenandoahTraversalMode.cpp 2019-11-11 16:10:33.747340859 +0100 @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2019, Red Hat, Inc. All rights reserved. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc_implementation/shenandoah/shenandoahTraversalMode.hpp" +#include "gc_implementation/shenandoah/heuristics/shenandoahTraversalAggressiveHeuristics.hpp" +#include "gc_implementation/shenandoah/heuristics/shenandoahTraversalHeuristics.hpp" +#include "gc_implementation/shenandoah/shenandoahLogging.hpp" + +void ShenandoahTraversalMode::initialize_flags() const { + FLAG_SET_DEFAULT(ShenandoahSATBBarrier, false); + FLAG_SET_DEFAULT(ShenandoahStoreValEnqueueBarrier, true); + FLAG_SET_DEFAULT(ShenandoahKeepAliveBarrier, false); + FLAG_SET_DEFAULT(ShenandoahAllowMixedAllocs, false); + + SHENANDOAH_ERGO_ENABLE_FLAG(ExplicitGCInvokesConcurrent); + SHENANDOAH_ERGO_ENABLE_FLAG(ShenandoahImplicitGCInvokesConcurrent); + + // Final configuration checks + SHENANDOAH_CHECK_FLAG_SET(ShenandoahLoadRefBarrier); + SHENANDOAH_CHECK_FLAG_SET(ShenandoahStoreValEnqueueBarrier); + SHENANDOAH_CHECK_FLAG_SET(ShenandoahCASBarrier); + SHENANDOAH_CHECK_FLAG_SET(ShenandoahCloneBarrier); +} + +ShenandoahHeuristics* ShenandoahTraversalMode::initialize_heuristics() const { + if (ShenandoahGCHeuristics != NULL) { + if (strcmp(ShenandoahGCHeuristics, "adaptive") == 0) { + return new ShenandoahTraversalHeuristics(); + } else if (strcmp(ShenandoahGCHeuristics, "aggressive") == 0) { + return new ShenandoahTraversalAggressiveHeuristics(); + } else { + vm_exit_during_initialization("Unknown -XX:ShenandoahGCHeuristics option"); + } + } + ShouldNotReachHere(); + return NULL; +} --- /dev/null 2019-11-11 08:42:03.603823722 +0100 +++ new/src/share/vm/gc_implementation/shenandoah/shenandoahTraversalMode.hpp 2019-11-11 16:10:34.244340831 +0100 @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2019, Red Hat, Inc. All rights reserved. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_GC_SHENANDOAH_SHENANDOAHTRAVERSALMODE_HPP +#define SHARE_GC_SHENANDOAH_SHENANDOAHTRAVERSALMODE_HPP + +#include "gc_implementation/shenandoah/shenandoahMode.hpp" + +class ShenandoahHeuristics; + +class ShenandoahTraversalMode : public ShenandoahMode { +public: + virtual void initialize_flags() const; + virtual ShenandoahHeuristics* initialize_heuristics() const; +}; + +#endif // SHARE_GC_SHENANDOAH_SHENANDOAHTRAVERSALMODE_HPP