# HG changeset patch # User rkennke # Date 1554238822 -7200 # Tue Apr 02 23:00:22 2019 +0200 # Node ID b24687c4ae7f5e0353428a07af3ff504f902ffa0 # Parent 7039f95a2ae2ccc77a2a598ef825e76e3cb12d4f 8221766: Load-reference barriers for Shenandoah Reviewed-by: kvn, erikj, aph, shade diff --git a/make/hotspot/lib/JvmFeatures.gmk b/make/hotspot/lib/JvmFeatures.gmk --- a/make/hotspot/lib/JvmFeatures.gmk +++ b/make/hotspot/lib/JvmFeatures.gmk @@ -168,8 +168,6 @@ ifneq ($(call check-jvm-feature, shenandoahgc), true) JVM_CFLAGS_FEATURES += -DINCLUDE_SHENANDOAHGC=0 JVM_EXCLUDE_PATTERNS += gc/shenandoah -else - JVM_CFLAGS_FEATURES += -DSUPPORT_BARRIER_ON_PRIMITIVES -DSUPPORT_NOT_TO_SPACE_INVARIANT endif ifneq ($(call check-jvm-feature, jfr), true) diff --git a/src/hotspot/cpu/aarch64/aarch64.ad b/src/hotspot/cpu/aarch64/aarch64.ad --- a/src/hotspot/cpu/aarch64/aarch64.ad +++ b/src/hotspot/cpu/aarch64/aarch64.ad @@ -9345,21 +9345,6 @@ ins_pipe(ialu_reg); %} -instruct shenandoahRB(iRegPNoSp dst, iRegP src, rFlagsReg cr) %{ - match(Set dst (ShenandoahReadBarrier src)); - format %{ "shenandoah_rb $dst,$src" %} - ins_encode %{ -#if INCLUDE_SHENANDOAHGC - Register s = $src$$Register; - Register d = $dst$$Register; - __ ldr(d, Address(s, ShenandoahBrooksPointer::byte_offset())); -#else - ShouldNotReachHere(); -#endif - %} - ins_pipe(pipe_class_memory); -%} - // Convert oop pointer into compressed form instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{ diff --git a/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp --- a/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp @@ -40,7 +40,7 @@ #define __ masm-> -address ShenandoahBarrierSetAssembler::_shenandoah_wb = NULL; +address ShenandoahBarrierSetAssembler::_shenandoah_lrb = NULL; void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop, Register addr, Register count, RegSet saved_regs) { @@ -214,60 +214,31 @@ __ bind(done); } -void ShenandoahBarrierSetAssembler::read_barrier(MacroAssembler* masm, Register dst) { - if (ShenandoahReadBarrier) { - read_barrier_impl(masm, dst); - } -} - -void ShenandoahBarrierSetAssembler::read_barrier_impl(MacroAssembler* masm, Register dst) { - assert(UseShenandoahGC && (ShenandoahReadBarrier || ShenandoahStoreValReadBarrier || ShenandoahCASBarrier), "should be enabled"); +void ShenandoahBarrierSetAssembler::resolve_forward_pointer(MacroAssembler* masm, Register dst) { + assert(ShenandoahLoadRefBarrier || ShenandoahCASBarrier, "Should be enabled"); Label is_null; __ cbz(dst, is_null); - read_barrier_not_null_impl(masm, dst); + resolve_forward_pointer_not_null(masm, dst); __ bind(is_null); } -void ShenandoahBarrierSetAssembler::read_barrier_not_null(MacroAssembler* masm, Register dst) { - if (ShenandoahReadBarrier) { - read_barrier_not_null_impl(masm, dst); - } -} - - -void ShenandoahBarrierSetAssembler::read_barrier_not_null_impl(MacroAssembler* masm, Register dst) { - assert(UseShenandoahGC && (ShenandoahReadBarrier || ShenandoahStoreValReadBarrier || ShenandoahCASBarrier), "should be enabled"); +// IMPORTANT: This must preserve all registers, even rscratch1 and rscratch2. +void ShenandoahBarrierSetAssembler::resolve_forward_pointer_not_null(MacroAssembler* masm, Register dst) { + assert(ShenandoahLoadRefBarrier || ShenandoahCASBarrier, "Should be enabled"); __ ldr(dst, Address(dst, ShenandoahBrooksPointer::byte_offset())); } -void ShenandoahBarrierSetAssembler::write_barrier(MacroAssembler* masm, Register dst) { - if (ShenandoahWriteBarrier) { - write_barrier_impl(masm, dst); - } -} - -void ShenandoahBarrierSetAssembler::write_barrier_impl(MacroAssembler* masm, Register dst) { - assert(UseShenandoahGC && (ShenandoahWriteBarrier || ShenandoahStoreValEnqueueBarrier), "Should be enabled"); - assert(dst != rscratch1, "need rscratch1"); +void ShenandoahBarrierSetAssembler::load_reference_barrier_not_null(MacroAssembler* masm, Register dst, Register tmp) { + assert(ShenandoahLoadRefBarrier, "Should be enabled"); assert(dst != rscratch2, "need rscratch2"); Label done; - + __ enter(); Address gc_state(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset())); - __ ldrb(rscratch1, gc_state); + __ ldrb(rscratch2, gc_state); // Check for heap stability - __ mov(rscratch2, ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::EVACUATION | ShenandoahHeap::TRAVERSAL); - __ tst(rscratch1, rscratch2); - __ br(Assembler::EQ, done); - - // Heap is unstable, need to perform the read-barrier even if WB is inactive - __ ldr(dst, Address(dst, ShenandoahBrooksPointer::byte_offset())); - - // Check for evacuation-in-progress and jump to WB slow-path if needed - __ mov(rscratch2, ShenandoahHeap::EVACUATION | ShenandoahHeap::TRAVERSAL); - __ tst(rscratch1, rscratch2); - __ br(Assembler::EQ, done); + __ tbz(rscratch2, ShenandoahHeap::HAS_FORWARDED_BITPOS, done); RegSet to_save = RegSet::of(r0); if (dst != r0) { @@ -275,7 +246,7 @@ __ mov(r0, dst); } - __ far_call(RuntimeAddress(CAST_FROM_FN_PTR(address, ShenandoahBarrierSetAssembler::shenandoah_wb()))); + __ far_call(RuntimeAddress(CAST_FROM_FN_PTR(address, ShenandoahBarrierSetAssembler::shenandoah_lrb()))); if (dst != r0) { __ mov(dst, r0); @@ -283,14 +254,11 @@ } __ bind(done); + __ leave(); } void ShenandoahBarrierSetAssembler::storeval_barrier(MacroAssembler* masm, Register dst, Register tmp) { if (ShenandoahStoreValEnqueueBarrier) { - Label is_null; - __ cbz(dst, is_null); - write_barrier_impl(masm, dst); - __ bind(is_null); // Save possibly live regs. RegSet live_regs = RegSet::range(r0, r4) - dst; __ push(live_regs, sp); @@ -302,44 +270,45 @@ __ ldrd(v0, __ post(sp, 2 * wordSize)); __ pop(live_regs, sp); } - if (ShenandoahStoreValReadBarrier) { - read_barrier_impl(masm, dst); +} + +void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler* masm, Register dst, Register tmp) { + if (ShenandoahLoadRefBarrier) { + Label is_null; + __ cbz(dst, is_null); + load_reference_barrier_not_null(masm, dst, tmp); + __ bind(is_null); } } void ShenandoahBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Register dst, Address src, Register tmp1, Register tmp_thread) { bool on_oop = type == T_OBJECT || type == T_ARRAY; - bool in_heap = (decorators & IN_HEAP) != 0; bool on_weak = (decorators & ON_WEAK_OOP_REF) != 0; bool on_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0; bool on_reference = on_weak || on_phantom; - if (in_heap) { - read_barrier_not_null(masm, src.base()); - } + BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread); + if (on_oop) { + load_reference_barrier(masm, dst, tmp1); - BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread); - if (ShenandoahKeepAliveBarrier && on_oop && on_reference) { - __ enter(); - satb_write_barrier_pre(masm /* masm */, - noreg /* obj */, - dst /* pre_val */, - rthread /* thread */, - tmp1 /* tmp */, - true /* tosca_live */, - true /* expand_call */); - __ leave(); + if (ShenandoahKeepAliveBarrier && on_reference) { + __ enter(); + satb_write_barrier_pre(masm /* masm */, + noreg /* obj */, + dst /* pre_val */, + rthread /* thread */, + tmp1 /* tmp */, + true /* tosca_live */, + true /* expand_call */); + __ leave(); + } } } void ShenandoahBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Address dst, Register val, Register tmp1, Register tmp2) { bool on_oop = type == T_OBJECT || type == T_ARRAY; - bool in_heap = (decorators & IN_HEAP) != 0; - if (in_heap) { - write_barrier(masm, dst.base()); - } if (!on_oop) { BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2); return; @@ -377,21 +346,6 @@ } -void ShenandoahBarrierSetAssembler::obj_equals(MacroAssembler* masm, Register op1, Register op2) { - __ cmp(op1, op2); - if (ShenandoahAcmpBarrier) { - Label done; - __ br(Assembler::EQ, done); - // The object may have been evacuated, but we won't see it without a - // membar here. - __ membar(Assembler::LoadStore| Assembler::LoadLoad); - read_barrier(masm, op1); - read_barrier(masm, op2); - __ cmp(op1, op2); - __ bind(done); - } -} - void ShenandoahBarrierSetAssembler::tlab_allocate(MacroAssembler* masm, Register obj, Register var_size_in_bytes, int con_size_in_bytes, @@ -428,27 +382,6 @@ } } -void ShenandoahBarrierSetAssembler::resolve_for_read(MacroAssembler* masm, DecoratorSet decorators, Register obj) { - bool oop_not_null = (decorators & IS_NOT_NULL) != 0; - if (oop_not_null) { - read_barrier_not_null(masm, obj); - } else { - read_barrier(masm, obj); - } -} - -void ShenandoahBarrierSetAssembler::resolve_for_write(MacroAssembler* masm, DecoratorSet decorators, Register obj) { - bool oop_not_null = (decorators & IS_NOT_NULL) != 0; - if (oop_not_null) { - write_barrier(masm, obj); - } else { - Label is_null; - __ cbz(obj, is_null); - write_barrier(masm, obj); - __ bind(is_null); - } -} - void ShenandoahBarrierSetAssembler::cmpxchg_oop(MacroAssembler* masm, Register addr, Register expected, Register new_val, bool acquire, bool release, bool weak, bool is_cae, Register result) { @@ -487,8 +420,8 @@ __ decode_heap_oop(tmp1, tmp1); __ decode_heap_oop(tmp2, tmp2); } - read_barrier_impl(masm, tmp1); - read_barrier_impl(masm, tmp2); + resolve_forward_pointer(masm, tmp1); + resolve_forward_pointer(masm, tmp2); __ cmp(tmp1, tmp2); // Retry with expected now being the value we just loaded from addr. __ br(Assembler::EQ, retry); @@ -533,7 +466,7 @@ __ b(*stub->continuation()); } -void ShenandoahBarrierSetAssembler::gen_write_barrier_stub(LIR_Assembler* ce, ShenandoahWriteBarrierStub* stub) { +void ShenandoahBarrierSetAssembler::gen_load_reference_barrier_stub(LIR_Assembler* ce, ShenandoahLoadReferenceBarrierStub* stub) { Register obj = stub->obj()->as_register(); Register res = stub->result()->as_register(); @@ -550,7 +483,7 @@ __ cbz(res, done); } - write_barrier(ce->masm(), res); + load_reference_barrier_not_null(ce->masm(), res, rscratch1); __ bind(done); __ b(*stub->continuation()); @@ -610,14 +543,14 @@ #endif // COMPILER1 -address ShenandoahBarrierSetAssembler::shenandoah_wb() { - assert(_shenandoah_wb != NULL, "need write barrier stub"); - return _shenandoah_wb; +address ShenandoahBarrierSetAssembler::shenandoah_lrb() { + assert(_shenandoah_lrb != NULL, "need load reference barrier stub"); + return _shenandoah_lrb; } #define __ cgen->assembler()-> -// Shenandoah write barrier. +// Shenandoah load reference barrier. // // Input: // r0: OOP to evacuate. Not null. @@ -626,13 +559,13 @@ // r0: Pointer to evacuated OOP. // // Trash rscratch1, rscratch2. Preserve everything else. -address ShenandoahBarrierSetAssembler::generate_shenandoah_wb(StubCodeGenerator* cgen) { +address ShenandoahBarrierSetAssembler::generate_shenandoah_lrb(StubCodeGenerator* cgen) { __ align(6); - StubCodeMark mark(cgen, "StubRoutines", "shenandoah_wb"); + StubCodeMark mark(cgen, "StubRoutines", "shenandoah_lrb"); address start = __ pc(); - Label work; + Label work, done; __ mov(rscratch2, ShenandoahHeap::in_cset_fast_test_addr()); __ lsr(rscratch1, r0, ShenandoahHeapRegion::region_size_bytes_shift_jint()); __ ldrb(rscratch2, Address(rscratch2, rscratch1)); @@ -640,19 +573,23 @@ __ ret(lr); __ bind(work); - Register obj = r0; + __ mov(rscratch2, r0); + resolve_forward_pointer_not_null(cgen->assembler(), r0); + __ cmp(rscratch2, r0); + __ br(Assembler::NE, done); __ enter(); // required for proper stackwalking of RuntimeStub frame __ push_call_clobbered_registers(); - __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_JRT)); + __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_JRT)); __ blrt(lr, 1, 0, MacroAssembler::ret_type_integral); - __ mov(rscratch1, obj); + __ mov(rscratch1, r0); __ pop_call_clobbered_registers(); - __ mov(obj, rscratch1); + __ mov(r0, rscratch1); __ leave(); // required for proper stackwalking of RuntimeStub frame + __ bind(done); __ ret(lr); return start; @@ -661,12 +598,12 @@ #undef __ void ShenandoahBarrierSetAssembler::barrier_stubs_init() { - if (ShenandoahWriteBarrier || ShenandoahStoreValEnqueueBarrier) { + if (ShenandoahLoadRefBarrier) { int stub_code_size = 2048; ResourceMark rm; BufferBlob* bb = BufferBlob::create("shenandoah_barrier_stubs", stub_code_size); CodeBuffer buf(bb); StubCodeGenerator cgen(&buf); - _shenandoah_wb = generate_shenandoah_wb(&cgen); + _shenandoah_lrb = generate_shenandoah_lrb(&cgen); } } diff --git a/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.hpp b/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.hpp --- a/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.hpp +++ b/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.hpp @@ -29,7 +29,7 @@ #ifdef COMPILER1 class LIR_Assembler; class ShenandoahPreBarrierStub; -class ShenandoahWriteBarrierStub; +class ShenandoahLoadReferenceBarrierStub; class StubAssembler; class StubCodeGenerator; #endif @@ -37,7 +37,7 @@ class ShenandoahBarrierSetAssembler: public BarrierSetAssembler { private: - static address _shenandoah_wb; + static address _shenandoah_lrb; void satb_write_barrier_pre(MacroAssembler* masm, Register obj, @@ -54,24 +54,21 @@ bool tosca_live, bool expand_call); - void read_barrier(MacroAssembler* masm, Register dst); - void read_barrier_impl(MacroAssembler* masm, Register dst); - void read_barrier_not_null(MacroAssembler* masm, Register dst); - void read_barrier_not_null_impl(MacroAssembler* masm, Register dst); - void write_barrier(MacroAssembler* masm, Register dst); - void write_barrier_impl(MacroAssembler* masm, Register dst); - void asm_acmp_barrier(MacroAssembler* masm, Register op1, Register op2); + void resolve_forward_pointer(MacroAssembler* masm, Register dst); + void resolve_forward_pointer_not_null(MacroAssembler* masm, Register dst); + void load_reference_barrier(MacroAssembler* masm, Register dst, Register tmp); + void load_reference_barrier_not_null(MacroAssembler* masm, Register dst, Register tmp); - address generate_shenandoah_wb(StubCodeGenerator* cgen); + address generate_shenandoah_lrb(StubCodeGenerator* cgen); public: - static address shenandoah_wb(); + static address shenandoah_lrb(); void storeval_barrier(MacroAssembler* masm, Register dst, Register tmp); #ifdef COMPILER1 void gen_pre_barrier_stub(LIR_Assembler* ce, ShenandoahPreBarrierStub* stub); - void gen_write_barrier_stub(LIR_Assembler* ce, ShenandoahWriteBarrierStub* stub); + void gen_load_reference_barrier_stub(LIR_Assembler* ce, ShenandoahLoadReferenceBarrierStub* stub); void generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm); #endif @@ -83,7 +80,6 @@ Register dst, Address src, Register tmp1, Register tmp_thread); virtual void store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Address dst, Register val, Register tmp1, Register tmp2); - virtual void obj_equals(MacroAssembler* masm, Register src1, Register src2); virtual void tlab_allocate(MacroAssembler* masm, Register obj, Register var_size_in_bytes, int con_size_in_bytes, @@ -91,9 +87,6 @@ Register t2, Label& slow_case); - virtual void resolve_for_read(MacroAssembler* masm, DecoratorSet decorators, Register obj); - virtual void resolve_for_write(MacroAssembler* masm, DecoratorSet decorators, Register obj); - virtual void cmpxchg_oop(MacroAssembler* masm, Register addr, Register expected, Register new_val, bool acquire, bool release, bool weak, bool is_cae, Register result); diff --git a/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetC1_aarch64.cpp b/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetC1_aarch64.cpp --- a/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetC1_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetC1_aarch64.cpp @@ -62,6 +62,10 @@ BasicType bt = access.type(); if (access.is_oop()) { LIRGenerator *gen = access.gen(); + if (ShenandoahSATBBarrier) { + pre_barrier(gen, access.access_emit_info(), access.decorators(), access.resolved_addr(), + LIR_OprFact::illegalOpr /* pre_val */); + } if (ShenandoahCASBarrier) { cmp_value.load_item(); new_value.load_item(); @@ -87,12 +91,20 @@ LIR_Opr value_opr = value.result(); if (access.is_oop()) { - value_opr = storeval_barrier(access, value_opr, access.access_emit_info(), true); + value_opr = storeval_barrier(access.gen(), value_opr, access.access_emit_info(), access.decorators()); } assert(type == T_INT || type == T_OBJECT || type == T_ARRAY LP64_ONLY( || type == T_LONG ), "unexpected type"); LIR_Opr tmp = gen->new_register(T_INT); __ xchg(access.resolved_addr(), value_opr, result, tmp); + if (access.is_oop()) { + result = load_reference_barrier(access.gen(), result, access.access_emit_info(), true); + if (ShenandoahSATBBarrier) { + pre_barrier(access.gen(), access.access_emit_info(), access.decorators(), LIR_OprFact::illegalOpr, + result /* pre_val */); + } + } + return result; } diff --git a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp --- a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp +++ b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp @@ -41,7 +41,7 @@ #define __ masm-> -address ShenandoahBarrierSetAssembler::_shenandoah_wb = NULL; +address ShenandoahBarrierSetAssembler::_shenandoah_lrb = NULL; void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Register src, Register dst, Register count) { @@ -312,41 +312,23 @@ __ bind(done); } -void ShenandoahBarrierSetAssembler::read_barrier(MacroAssembler* masm, Register dst) { - if (ShenandoahReadBarrier) { - read_barrier_impl(masm, dst); - } -} - -void ShenandoahBarrierSetAssembler::read_barrier_impl(MacroAssembler* masm, Register dst) { - assert(UseShenandoahGC && (ShenandoahReadBarrier || ShenandoahStoreValReadBarrier || ShenandoahCASBarrier), "should be enabled"); +void ShenandoahBarrierSetAssembler::resolve_forward_pointer(MacroAssembler* masm, Register dst) { + assert(ShenandoahCASBarrier, "should be enabled"); Label is_null; __ testptr(dst, dst); __ jcc(Assembler::zero, is_null); - read_barrier_not_null_impl(masm, dst); + resolve_forward_pointer_not_null(masm, dst); __ bind(is_null); } -void ShenandoahBarrierSetAssembler::read_barrier_not_null(MacroAssembler* masm, Register dst) { - if (ShenandoahReadBarrier) { - read_barrier_not_null_impl(masm, dst); - } -} - -void ShenandoahBarrierSetAssembler::read_barrier_not_null_impl(MacroAssembler* masm, Register dst) { - assert(UseShenandoahGC && (ShenandoahReadBarrier || ShenandoahStoreValReadBarrier || ShenandoahCASBarrier), "should be enabled"); +void ShenandoahBarrierSetAssembler::resolve_forward_pointer_not_null(MacroAssembler* masm, Register dst) { + assert(ShenandoahCASBarrier || ShenandoahLoadRefBarrier, "should be enabled"); __ movptr(dst, Address(dst, ShenandoahBrooksPointer::byte_offset())); } -void ShenandoahBarrierSetAssembler::write_barrier(MacroAssembler* masm, Register dst) { - if (ShenandoahWriteBarrier) { - write_barrier_impl(masm, dst); - } -} - -void ShenandoahBarrierSetAssembler::write_barrier_impl(MacroAssembler* masm, Register dst) { - assert(UseShenandoahGC && (ShenandoahWriteBarrier || ShenandoahStoreValEnqueueBarrier), "Should be enabled"); +void ShenandoahBarrierSetAssembler::load_reference_barrier_not_null(MacroAssembler* masm, Register dst) { + assert(ShenandoahLoadRefBarrier, "Should be enabled"); #ifdef _LP64 Label done; @@ -354,8 +336,8 @@ __ testb(gc_state, ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::EVACUATION | ShenandoahHeap::TRAVERSAL); __ jccb(Assembler::zero, done); - // Heap is unstable, need to perform the read-barrier even if WB is inactive - read_barrier_not_null(masm, dst); + // Heap is unstable, need to perform the resolve even if LRB is inactive + resolve_forward_pointer_not_null(masm, dst); __ testb(gc_state, ShenandoahHeap::EVACUATION | ShenandoahHeap::TRAVERSAL); __ jccb(Assembler::zero, done); @@ -364,7 +346,7 @@ __ xchgptr(dst, rax); // Move obj into rax and save rax into obj. } - __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, ShenandoahBarrierSetAssembler::shenandoah_wb()))); + __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, ShenandoahBarrierSetAssembler::shenandoah_lrb()))); if (dst != rax) { __ xchgptr(rax, dst); // Swap back obj with rax. @@ -377,24 +359,18 @@ } void ShenandoahBarrierSetAssembler::storeval_barrier(MacroAssembler* masm, Register dst, Register tmp) { - if (ShenandoahStoreValReadBarrier || ShenandoahStoreValEnqueueBarrier) { + if (ShenandoahStoreValEnqueueBarrier) { storeval_barrier_impl(masm, dst, tmp); } } void ShenandoahBarrierSetAssembler::storeval_barrier_impl(MacroAssembler* masm, Register dst, Register tmp) { - assert(UseShenandoahGC && (ShenandoahStoreValReadBarrier || ShenandoahStoreValEnqueueBarrier), "should be enabled"); + assert(ShenandoahStoreValEnqueueBarrier, "should be enabled"); if (dst == noreg) return; #ifdef _LP64 if (ShenandoahStoreValEnqueueBarrier) { - Label is_null; - __ testptr(dst, dst); - __ jcc(Assembler::zero, is_null); - write_barrier_impl(masm, dst); - __ bind(is_null); - // The set of registers to be saved+restored is the same as in the write-barrier above. // Those are the commonly used registers in the interpreter. __ pusha(); @@ -408,50 +384,54 @@ //__ pop_callee_saved_registers(); __ popa(); } - if (ShenandoahStoreValReadBarrier) { - read_barrier_impl(masm, dst); - } #else Unimplemented(); #endif } +void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler* masm, Register dst) { + if (ShenandoahLoadRefBarrier) { + Label done; + __ testptr(dst, dst); + __ jcc(Assembler::zero, done); + load_reference_barrier_not_null(masm, dst); + __ bind(done); + } +} + void ShenandoahBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Register dst, Address src, Register tmp1, Register tmp_thread) { bool on_oop = type == T_OBJECT || type == T_ARRAY; - bool in_heap = (decorators & IN_HEAP) != 0; bool on_weak = (decorators & ON_WEAK_OOP_REF) != 0; bool on_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0; bool on_reference = on_weak || on_phantom; - if (in_heap) { - read_barrier_not_null(masm, src.base()); - } - BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread); - if (ShenandoahKeepAliveBarrier && on_oop && on_reference) { - const Register thread = NOT_LP64(tmp_thread) LP64_ONLY(r15_thread); - NOT_LP64(__ get_thread(thread)); + BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread); + if (on_oop) { + load_reference_barrier(masm, dst); - // Generate the SATB pre-barrier code to log the value of - // the referent field in an SATB buffer. - shenandoah_write_barrier_pre(masm /* masm */, - noreg /* obj */, - dst /* pre_val */, - thread /* thread */, - tmp1 /* tmp */, - true /* tosca_live */, - true /* expand_call */); + if (ShenandoahKeepAliveBarrier && on_reference) { + const Register thread = NOT_LP64(tmp_thread) LP64_ONLY(r15_thread); + NOT_LP64(__ get_thread(thread)); + // Generate the SATB pre-barrier code to log the value of + // the referent field in an SATB buffer. + shenandoah_write_barrier_pre(masm /* masm */, + noreg /* obj */, + dst /* pre_val */, + thread /* thread */, + tmp1 /* tmp */, + true /* tosca_live */, + true /* expand_call */); + } } } void ShenandoahBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Address dst, Register val, Register tmp1, Register tmp2) { + bool on_oop = type == T_OBJECT || type == T_ARRAY; bool in_heap = (decorators & IN_HEAP) != 0; bool as_normal = (decorators & AS_NORMAL) != 0; - if (in_heap) { - write_barrier(masm, dst.base()); - } - if (type == T_OBJECT || type == T_ARRAY) { + if (on_oop && in_heap) { bool needs_pre_barrier = as_normal; Register tmp3 = LP64_ONLY(r8) NOT_LP64(rsi); @@ -494,44 +474,6 @@ } } -#ifndef _LP64 -void ShenandoahBarrierSetAssembler::obj_equals(MacroAssembler* masm, - Address obj1, jobject obj2) { - Unimplemented(); -} - -void ShenandoahBarrierSetAssembler::obj_equals(MacroAssembler* masm, - Register obj1, jobject obj2) { - Unimplemented(); -} -#endif - - -void ShenandoahBarrierSetAssembler::obj_equals(MacroAssembler* masm, Register op1, Register op2) { - __ cmpptr(op1, op2); - if (ShenandoahAcmpBarrier) { - Label done; - __ jccb(Assembler::equal, done); - read_barrier(masm, op1); - read_barrier(masm, op2); - __ cmpptr(op1, op2); - __ bind(done); - } -} - -void ShenandoahBarrierSetAssembler::obj_equals(MacroAssembler* masm, Register src1, Address src2) { - __ cmpptr(src1, src2); - if (ShenandoahAcmpBarrier) { - Label done; - __ jccb(Assembler::equal, done); - __ movptr(rscratch2, src2); - read_barrier(masm, src1); - read_barrier(masm, rscratch2); - __ cmpptr(src1, rscratch2); - __ bind(done); - } -} - void ShenandoahBarrierSetAssembler::tlab_allocate(MacroAssembler* masm, Register thread, Register obj, Register var_size_in_bytes, @@ -581,28 +523,6 @@ __ verify_tlab(); } -void ShenandoahBarrierSetAssembler::resolve_for_read(MacroAssembler* masm, DecoratorSet decorators, Register obj) { - bool oop_not_null = (decorators & IS_NOT_NULL) != 0; - if (oop_not_null) { - read_barrier_not_null(masm, obj); - } else { - read_barrier(masm, obj); - } -} - -void ShenandoahBarrierSetAssembler::resolve_for_write(MacroAssembler* masm, DecoratorSet decorators, Register obj) { - bool oop_not_null = (decorators & IS_NOT_NULL) != 0; - if (oop_not_null) { - write_barrier(masm, obj); - } else { - Label done; - __ testptr(obj, obj); - __ jcc(Assembler::zero, done); - write_barrier(masm, obj); - __ bind(done); - } -} - // Special Shenandoah CAS implementation that handles false negatives // due to concurrent evacuation. #ifndef _LP64 @@ -641,14 +561,14 @@ // Step 2. CAS had failed. This may be a false negative. // // The trouble comes when we compare the to-space pointer with the from-space - // pointer to the same object. To resolve this, it will suffice to read both - // oldval and the value from memory through the read barriers -- this will give - // both to-space pointers. If they mismatch, then it was a legitimate failure. + // pointer to the same object. To resolve this, it will suffice to resolve both + // oldval and the value from memory -- this will give both to-space pointers. + // If they mismatch, then it was a legitimate failure. // if (UseCompressedOops) { __ decode_heap_oop(tmp1); } - read_barrier_impl(masm, tmp1); + resolve_forward_pointer(masm, tmp1); if (UseCompressedOops) { __ movl(tmp2, oldval); @@ -656,7 +576,7 @@ } else { __ movptr(tmp2, oldval); } - read_barrier_impl(masm, tmp2); + resolve_forward_pointer(masm, tmp2); __ cmpptr(tmp1, tmp2); __ jcc(Assembler::notEqual, done, true); @@ -665,8 +585,8 @@ // // Corner case: it may happen that somebody stored the from-space pointer // to memory while we were preparing for retry. Therefore, we can fail again - // on retry, and so need to do this in loop, always re-reading the failure - // witness through the read barrier. + // on retry, and so need to do this in loop, always resolving the failure + // witness. __ bind(retry); if (os::is_MP()) __ lock(); if (UseCompressedOops) { @@ -682,7 +602,7 @@ } else { __ movptr(tmp2, oldval); } - read_barrier_impl(masm, tmp2); + resolve_forward_pointer(masm, tmp2); __ cmpptr(tmp1, tmp2); __ jcc(Assembler::equal, retry, true); @@ -830,7 +750,7 @@ } -void ShenandoahBarrierSetAssembler::gen_write_barrier_stub(LIR_Assembler* ce, ShenandoahWriteBarrierStub* stub) { +void ShenandoahBarrierSetAssembler::gen_load_reference_barrier_stub(LIR_Assembler* ce, ShenandoahLoadReferenceBarrierStub* stub) { __ bind(*stub->entry()); Label done; @@ -847,7 +767,7 @@ __ jcc(Assembler::zero, done); } - write_barrier(ce->masm(), res); + load_reference_barrier_not_null(ce->masm(), res); __ bind(done); __ jmp(*stub->continuation()); @@ -917,16 +837,16 @@ #endif // COMPILER1 -address ShenandoahBarrierSetAssembler::shenandoah_wb() { - assert(_shenandoah_wb != NULL, "need write barrier stub"); - return _shenandoah_wb; +address ShenandoahBarrierSetAssembler::shenandoah_lrb() { + assert(_shenandoah_lrb != NULL, "need load reference barrier stub"); + return _shenandoah_lrb; } #define __ cgen->assembler()-> -address ShenandoahBarrierSetAssembler::generate_shenandoah_wb(StubCodeGenerator* cgen) { +address ShenandoahBarrierSetAssembler::generate_shenandoah_lrb(StubCodeGenerator* cgen) { __ align(CodeEntryAlignment); - StubCodeMark mark(cgen, "StubRoutines", "shenandoah_wb"); + StubCodeMark mark(cgen, "StubRoutines", "shenandoah_lrb"); address start = __ pc(); #ifdef _LP64 @@ -974,7 +894,7 @@ __ push(r15); save_vector_registers(cgen->assembler()); __ movptr(rdi, rax); - __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_JRT), rdi); + __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_JRT), rdi); restore_vector_registers(cgen->assembler()); __ pop(r15); __ pop(r14); @@ -1001,12 +921,12 @@ #undef __ void ShenandoahBarrierSetAssembler::barrier_stubs_init() { - if (ShenandoahWriteBarrier || ShenandoahStoreValEnqueueBarrier) { + if (ShenandoahLoadRefBarrier) { int stub_code_size = 4096; ResourceMark rm; BufferBlob* bb = BufferBlob::create("shenandoah_barrier_stubs", stub_code_size); CodeBuffer buf(bb); StubCodeGenerator cgen(&buf); - _shenandoah_wb = generate_shenandoah_wb(&cgen); + _shenandoah_lrb = generate_shenandoah_lrb(&cgen); } } diff --git a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.hpp b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.hpp --- a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.hpp +++ b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.hpp @@ -29,7 +29,7 @@ #ifdef COMPILER1 class LIR_Assembler; class ShenandoahPreBarrierStub; -class ShenandoahWriteBarrierStub; +class ShenandoahLoadReferenceBarrierStub; class StubAssembler; class StubCodeGenerator; #endif @@ -37,7 +37,7 @@ class ShenandoahBarrierSetAssembler: public BarrierSetAssembler { private: - static address _shenandoah_wb; + static address _shenandoah_lrb; void satb_write_barrier_pre(MacroAssembler* masm, Register obj, @@ -55,32 +55,30 @@ bool tosca_live, bool expand_call); - void read_barrier(MacroAssembler* masm, Register dst); - void read_barrier_impl(MacroAssembler* masm, Register dst); + void resolve_forward_pointer(MacroAssembler* masm, Register dst); + void resolve_forward_pointer_not_null(MacroAssembler* masm, Register dst); - void read_barrier_not_null(MacroAssembler* masm, Register dst); - void read_barrier_not_null_impl(MacroAssembler* masm, Register dst); - - void write_barrier(MacroAssembler* masm, Register dst); - void write_barrier_impl(MacroAssembler* masm, Register dst); + void load_reference_barrier_not_null(MacroAssembler* masm, Register dst); void storeval_barrier_impl(MacroAssembler* masm, Register dst, Register tmp); - address generate_shenandoah_wb(StubCodeGenerator* cgen); + address generate_shenandoah_lrb(StubCodeGenerator* cgen); void save_vector_registers(MacroAssembler* masm); void restore_vector_registers(MacroAssembler* masm); public: - static address shenandoah_wb(); + static address shenandoah_lrb(); void storeval_barrier(MacroAssembler* masm, Register dst, Register tmp); #ifdef COMPILER1 void gen_pre_barrier_stub(LIR_Assembler* ce, ShenandoahPreBarrierStub* stub); - void gen_write_barrier_stub(LIR_Assembler* ce, ShenandoahWriteBarrierStub* stub); + void gen_load_reference_barrier_stub(LIR_Assembler* ce, ShenandoahLoadReferenceBarrierStub* stub); void generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm); #endif + void load_reference_barrier(MacroAssembler* masm, Register dst); + virtual void cmpxchg_oop(MacroAssembler* masm, Register res, Address addr, Register oldval, Register newval, bool exchange, Register tmp1, Register tmp2); @@ -93,16 +91,6 @@ virtual void store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Address dst, Register val, Register tmp1, Register tmp2); -#ifndef _LP64 - virtual void obj_equals(MacroAssembler* masm, - Address obj1, jobject obj2); - virtual void obj_equals(MacroAssembler* masm, - Register obj1, jobject obj2); -#endif - - virtual void obj_equals(MacroAssembler* masm, Register src1, Register src2); - virtual void obj_equals(MacroAssembler* masm, Register src1, Address src2); - virtual void tlab_allocate(MacroAssembler* masm, Register thread, Register obj, Register var_size_in_bytes, @@ -110,9 +98,6 @@ Register t1, Register t2, Label& slow_case); - virtual void resolve_for_read(MacroAssembler* masm, DecoratorSet decorators, Register obj); - virtual void resolve_for_write(MacroAssembler* masm, DecoratorSet decorators, Register obj); - virtual void barrier_stubs_init(); }; diff --git a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetC1_x86.cpp b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetC1_x86.cpp --- a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetC1_x86.cpp +++ b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetC1_x86.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019, Red Hat, Inc. All rights reserved. + * Copyright (c) 2018, Red Hat, Inc. All rights reserved. * * This code is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License version 2 only, as @@ -57,6 +57,7 @@ } #undef __ + #ifdef ASSERT #define __ gen->lir(__FILE__, __LINE__)-> #else @@ -67,6 +68,10 @@ if (access.is_oop()) { LIRGenerator* gen = access.gen(); + if (ShenandoahSATBBarrier) { + pre_barrier(gen, access.access_emit_info(), access.decorators(), access.resolved_addr(), + LIR_OprFact::illegalOpr /* pre_val */); + } if (ShenandoahCASBarrier) { cmp_value.load_item_force(FrameMap::rax_oop_opr); new_value.load_item(); @@ -92,7 +97,7 @@ LIR_Opr value_opr = value.result(); if (access.is_oop()) { - value_opr = storeval_barrier(access, value_opr, access.access_emit_info(), true); + value_opr = storeval_barrier(access.gen(), value_opr, access.access_emit_info(), access.decorators()); } // Because we want a 2-arg form of xchg and xadd @@ -101,5 +106,13 @@ assert(type == T_INT || type == T_OBJECT || type == T_ARRAY LP64_ONLY( || type == T_LONG ), "unexpected type"); __ xchg(access.resolved_addr(), result, result, LIR_OprFact::illegalOpr); + if (access.is_oop()) { + result = load_reference_barrier(access.gen(), result, access.access_emit_info(), true); + if (ShenandoahSATBBarrier) { + pre_barrier(access.gen(), access.access_emit_info(), access.decorators(), LIR_OprFact::illegalOpr, + result /* pre_val */); + } + } + return result; } diff --git a/src/hotspot/cpu/x86/x86_64.ad b/src/hotspot/cpu/x86/x86_64.ad --- a/src/hotspot/cpu/x86/x86_64.ad +++ b/src/hotspot/cpu/x86/x86_64.ad @@ -6721,59 +6721,6 @@ ins_pipe(ialu_reg_reg); // XXX %} -instruct shenandoahRB(rRegP dst, rRegP src, rFlagsReg cr) %{ - match(Set dst (ShenandoahReadBarrier src)); - effect(DEF dst, USE src); - ins_cost(125); // XXX - format %{ "shenandoah_rb $dst, $src" %} - ins_encode %{ -#if INCLUDE_SHENANDOAHGC - Register d = $dst$$Register; - Register s = $src$$Register; - __ movptr(d, Address(s, ShenandoahBrooksPointer::byte_offset())); -#else - ShouldNotReachHere(); -#endif - %} - ins_pipe(ialu_reg_mem); -%} - -instruct shenandoahRBNarrow(rRegP dst, rRegN src) %{ - predicate(UseCompressedOops && (Universe::narrow_oop_shift() == 0)); - match(Set dst (ShenandoahReadBarrier (DecodeN src))); - effect(DEF dst, USE src); - ins_cost(125); // XXX - format %{ "shenandoah_rb $dst, $src" %} - ins_encode %{ -#if INCLUDE_SHENANDOAHGC - Register d = $dst$$Register; - Register s = $src$$Register; - __ movptr(d, Address(r12, s, Address::times_1, ShenandoahBrooksPointer::byte_offset())); -#else - ShouldNotReachHere(); -#endif - %} - ins_pipe(ialu_reg_mem); -%} - -instruct shenandoahRBNarrowShift(rRegP dst, rRegN src) %{ - predicate(UseCompressedOops && (Universe::narrow_oop_shift() == Address::times_8)); - match(Set dst (ShenandoahReadBarrier (DecodeN src))); - effect(DEF dst, USE src); - ins_cost(125); // XXX - format %{ "shenandoah_rb $dst, $src" %} - ins_encode %{ -#if INCLUDE_SHENANDOAHGC - Register d = $dst$$Register; - Register s = $src$$Register; - __ movptr(d, Address(r12, s, Address::times_8, ShenandoahBrooksPointer::byte_offset())); -#else - ShouldNotReachHere(); -#endif - %} - ins_pipe(ialu_reg_mem); -%} - // Convert oop pointer into compressed form instruct encodeHeapOop(rRegN dst, rRegP src, rFlagsReg cr) %{ predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull); diff --git a/src/hotspot/share/adlc/formssel.cpp b/src/hotspot/share/adlc/formssel.cpp --- a/src/hotspot/share/adlc/formssel.cpp +++ b/src/hotspot/share/adlc/formssel.cpp @@ -777,7 +777,6 @@ #if INCLUDE_SHENANDOAHGC !strcmp(_matrule->_rChild->_opType,"ShenandoahCompareAndExchangeP") || !strcmp(_matrule->_rChild->_opType,"ShenandoahCompareAndExchangeN") || - !strcmp(_matrule->_rChild->_opType,"ShenandoahReadBarrier") || #endif !strcmp(_matrule->_rChild->_opType,"CompareAndExchangeP") || !strcmp(_matrule->_rChild->_opType,"CompareAndExchangeN"))) return true; @@ -3507,7 +3506,6 @@ "ClearArray", "GetAndSetB", "GetAndSetS", "GetAndAddI", "GetAndSetI", "GetAndSetP", "GetAndAddB", "GetAndAddS", "GetAndAddL", "GetAndSetL", "GetAndSetN", - "ShenandoahReadBarrier", "LoadBarrierSlowReg", "LoadBarrierWeakSlowReg" }; int cnt = sizeof(needs_ideal_memory_list)/sizeof(char*); diff --git a/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.cpp b/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.cpp --- a/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.cpp +++ b/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.cpp @@ -23,7 +23,7 @@ #include "precompiled.hpp" #include "c1/c1_IR.hpp" -#include "gc/g1/satbMarkQueue.hpp" +#include "gc/shared/satbMarkQueue.hpp" #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp" #include "gc/shenandoah/shenandoahBrooksPointer.hpp" #include "gc/shenandoah/shenandoahHeap.hpp" @@ -31,6 +31,10 @@ #include "gc/shenandoah/shenandoahThreadLocalData.hpp" #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp" +#ifndef PATCHED_ADDR +#define PATCHED_ADDR (max_jint) +#endif + #ifdef ASSERT #define __ gen->lir(__FILE__, __LINE__)-> #else @@ -42,16 +46,12 @@ bs->gen_pre_barrier_stub(ce, this); } -void ShenandoahWriteBarrierStub::emit_code(LIR_Assembler* ce) { +void ShenandoahLoadReferenceBarrierStub::emit_code(LIR_Assembler* ce) { ShenandoahBarrierSetAssembler* bs = (ShenandoahBarrierSetAssembler*)BarrierSet::barrier_set()->barrier_set_assembler(); - bs->gen_write_barrier_stub(ce, this); + bs->gen_load_reference_barrier_stub(ce, this); } -void ShenandoahBarrierSetC1::pre_barrier(LIRAccess& access, LIR_Opr addr_opr, LIR_Opr pre_val) { - LIRGenerator* gen = access.gen(); - CodeEmitInfo* info = access.access_emit_info(); - DecoratorSet decorators = access.decorators(); - +void ShenandoahBarrierSetC1::pre_barrier(LIRGenerator* gen, CodeEmitInfo* info, DecoratorSet decorators, LIR_Opr addr_opr, LIR_Opr pre_val) { // First we test whether marking is in progress. BasicType flag_type; bool patch = (decorators & C1_NEEDS_PATCHING) != 0; @@ -105,44 +105,18 @@ __ branch_destination(slow->continuation()); } -LIR_Opr ShenandoahBarrierSetC1::read_barrier(LIRAccess& access, LIR_Opr obj, CodeEmitInfo* info, bool need_null_check) { - if (UseShenandoahGC && ShenandoahReadBarrier) { - return read_barrier_impl(access, obj, info, need_null_check); +LIR_Opr ShenandoahBarrierSetC1::load_reference_barrier(LIRGenerator* gen, LIR_Opr obj, CodeEmitInfo* info, bool need_null_check) { + if (ShenandoahLoadRefBarrier) { + return load_reference_barrier_impl(gen, obj, info, need_null_check); } else { return obj; } } -LIR_Opr ShenandoahBarrierSetC1::read_barrier_impl(LIRAccess& access, LIR_Opr obj, CodeEmitInfo* info, bool need_null_check) { - assert(UseShenandoahGC && (ShenandoahReadBarrier || ShenandoahStoreValReadBarrier), "Should be enabled"); - LIRGenerator* gen = access.gen(); - LabelObj* done = new LabelObj(); - LIR_Opr result = gen->new_register(T_OBJECT); - __ move(obj, result); - if (need_null_check) { - __ cmp(lir_cond_equal, result, LIR_OprFact::oopConst(NULL)); - __ branch(lir_cond_equal, T_LONG, done->label()); - } - LIR_Address* brooks_ptr_address = gen->generate_address(result, ShenandoahBrooksPointer::byte_offset(), T_ADDRESS); - __ load(brooks_ptr_address, result, info ? new CodeEmitInfo(info) : NULL, lir_patch_none); +LIR_Opr ShenandoahBarrierSetC1::load_reference_barrier_impl(LIRGenerator* gen, LIR_Opr obj, CodeEmitInfo* info, bool need_null_check) { + assert(ShenandoahLoadRefBarrier, "Should be enabled"); - __ branch_destination(done->label()); - return result; -} - -LIR_Opr ShenandoahBarrierSetC1::write_barrier(LIRAccess& access, LIR_Opr obj, CodeEmitInfo* info, bool need_null_check) { - if (UseShenandoahGC && ShenandoahWriteBarrier) { - return write_barrier_impl(access, obj, info, need_null_check); - } else { - return obj; - } -} - -LIR_Opr ShenandoahBarrierSetC1::write_barrier_impl(LIRAccess& access, LIR_Opr obj, CodeEmitInfo* info, bool need_null_check) { - assert(UseShenandoahGC && (ShenandoahWriteBarrier || ShenandoahStoreValEnqueueBarrier), "Should be enabled"); - LIRGenerator* gen = access.gen(); - - obj = ensure_in_register(access, obj); + obj = ensure_in_register(gen, obj); assert(obj->is_register(), "must be a register at this point"); LIR_Opr result = gen->new_register(T_OBJECT); __ move(obj, result); @@ -170,16 +144,15 @@ } __ cmp(lir_cond_notEqual, flag_val, LIR_OprFact::intConst(0)); - CodeStub* slow = new ShenandoahWriteBarrierStub(obj, result, info ? new CodeEmitInfo(info) : NULL, need_null_check); + CodeStub* slow = new ShenandoahLoadReferenceBarrierStub(obj, result, info ? new CodeEmitInfo(info) : NULL, need_null_check); __ branch(lir_cond_notEqual, T_INT, slow); __ branch_destination(slow->continuation()); return result; } -LIR_Opr ShenandoahBarrierSetC1::ensure_in_register(LIRAccess& access, LIR_Opr obj) { +LIR_Opr ShenandoahBarrierSetC1::ensure_in_register(LIRGenerator* gen, LIR_Opr obj) { if (!obj->is_register()) { - LIRGenerator* gen = access.gen(); LIR_Opr obj_reg = gen->new_register(T_OBJECT); if (obj->is_constant()) { __ move(obj, obj_reg); @@ -191,55 +164,54 @@ return obj; } -LIR_Opr ShenandoahBarrierSetC1::storeval_barrier(LIRAccess& access, LIR_Opr obj, CodeEmitInfo* info, bool need_null_check) { - LIRGenerator* gen = access.gen(); - if (UseShenandoahGC) { - if (ShenandoahStoreValEnqueueBarrier) { - obj = write_barrier_impl(access, obj, info, need_null_check); - pre_barrier(access, LIR_OprFact::illegalOpr, obj); - } - if (ShenandoahStoreValReadBarrier) { - obj = read_barrier_impl(access, obj, info, true /*need_null_check*/); - } +LIR_Opr ShenandoahBarrierSetC1::storeval_barrier(LIRGenerator* gen, LIR_Opr obj, CodeEmitInfo* info, DecoratorSet decorators) { + if (ShenandoahStoreValEnqueueBarrier) { + obj = ensure_in_register(gen, obj); + pre_barrier(gen, info, decorators, LIR_OprFact::illegalOpr, obj); } return obj; } -void ShenandoahBarrierSetC1::store_at(LIRAccess& access, LIR_Opr value) { - access.set_base(write_barrier(access, access.base().item().result(), access.access_emit_info(), access.needs_null_check())); - LIR_Opr resolved = resolve_address(access, false); - access.set_resolved_addr(resolved); +void ShenandoahBarrierSetC1::store_at_resolved(LIRAccess& access, LIR_Opr value) { if (access.is_oop()) { if (ShenandoahSATBBarrier) { - pre_barrier(access, access.resolved_addr(), LIR_OprFact::illegalOpr /* pre_val */); + pre_barrier(access.gen(), access.access_emit_info(), access.decorators(), access.resolved_addr(), LIR_OprFact::illegalOpr /* pre_val */); } - value = storeval_barrier(access, value, access.access_emit_info(), access.needs_null_check()); + value = storeval_barrier(access.gen(), value, access.access_emit_info(), access.decorators()); } BarrierSetC1::store_at_resolved(access, value); } -void ShenandoahBarrierSetC1::load_at(LIRAccess& access, LIR_Opr result) { - LIRItem& base_item = access.base().item(); - access.set_base(read_barrier(access, access.base().item().result(), access.access_emit_info(), access.needs_null_check())); - LIR_Opr resolved = resolve_address(access, false); - access.set_resolved_addr(resolved); - BarrierSetC1::load_at_resolved(access, result); - access.set_base(base_item); +void ShenandoahBarrierSetC1::load_at_resolved(LIRAccess& access, LIR_Opr result) { + if (!access.is_oop()) { + BarrierSetC1::load_at_resolved(access, result); + return; + } + + LIRGenerator *gen = access.gen(); + + if (ShenandoahLoadRefBarrier) { + LIR_Opr tmp = gen->new_register(T_OBJECT); + BarrierSetC1::load_at_resolved(access, tmp); + tmp = load_reference_barrier(access.gen(), tmp, access.access_emit_info(), true); + __ move(tmp, result); + } else { + BarrierSetC1::load_at_resolved(access, result); + } if (ShenandoahKeepAliveBarrier) { DecoratorSet decorators = access.decorators(); bool is_weak = (decorators & ON_WEAK_OOP_REF) != 0; bool is_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0; bool is_anonymous = (decorators & ON_UNKNOWN_OOP_REF) != 0; - LIRGenerator *gen = access.gen(); - if (access.is_oop() && (is_weak || is_phantom || is_anonymous)) { + if (is_weak || is_phantom || is_anonymous) { // Register the value in the referent field with the pre-barrier LabelObj *Lcont_anonymous; if (is_anonymous) { Lcont_anonymous = new LabelObj(); generate_referent_check(access, Lcont_anonymous); } - pre_barrier(access, LIR_OprFact::illegalOpr /* addr_opr */, + pre_barrier(access.gen(), access.access_emit_info(), access.decorators(), LIR_OprFact::illegalOpr /* addr_opr */, result /* pre_val */); if (is_anonymous) { __ branch_destination(Lcont_anonymous->label()); @@ -248,49 +220,6 @@ } } -LIR_Opr ShenandoahBarrierSetC1::atomic_cmpxchg_at(LIRAccess& access, LIRItem& cmp_value, LIRItem& new_value) { - access.load_address(); - access.set_base(write_barrier(access, access.base().item().result(), access.access_emit_info(), access.needs_null_check())); - LIR_Opr resolved = resolve_address(access, true); - access.set_resolved_addr(resolved); - if (access.is_oop()) { - if (ShenandoahSATBBarrier) { - pre_barrier(access, access.resolved_addr(), - LIR_OprFact::illegalOpr /* pre_val */); - } - } - return atomic_cmpxchg_at_resolved(access, cmp_value, new_value); -} - -LIR_Opr ShenandoahBarrierSetC1::atomic_xchg_at(LIRAccess& access, LIRItem& value) { - access.set_base(write_barrier(access, access.base().item().result(), access.access_emit_info(), access.needs_null_check())); - LIR_Opr resolved = resolve_address(access, true); - access.set_resolved_addr(resolved); - if (access.is_oop()) { - if (ShenandoahSATBBarrier) { - pre_barrier(access, access.resolved_addr(), - LIR_OprFact::illegalOpr /* pre_val */); - } - } - return BarrierSetC1::atomic_xchg_at_resolved(access, value); -} - -LIR_Opr ShenandoahBarrierSetC1::atomic_add_at(LIRAccess& access, LIRItem& value) { - access.load_address(); - access.set_base(write_barrier(access, access.base().item().result(), access.access_emit_info(), access.needs_null_check())); - LIR_Opr resolved = resolve_address(access, true); - access.set_resolved_addr(resolved); - return BarrierSetC1::atomic_add_at_resolved(access, value); -} - -LIR_Opr ShenandoahBarrierSetC1::resolve_for_read(LIRAccess& access) { - return read_barrier(access, access.base().opr(), access.access_emit_info(), access.needs_null_check()); -} - -LIR_Opr ShenandoahBarrierSetC1::resolve_for_write(LIRAccess& access) { - return write_barrier(access, access.base().opr(), access.access_emit_info(), access.needs_null_check()); -} - class C1ShenandoahPreBarrierCodeGenClosure : public StubAssemblerCodeGenClosure { virtual OopMapSet* generate_code(StubAssembler* sasm) { ShenandoahBarrierSetAssembler* bs = (ShenandoahBarrierSetAssembler*)BarrierSet::barrier_set()->barrier_set_assembler(); diff --git a/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.hpp b/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.hpp --- a/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.hpp +++ b/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.hpp @@ -21,8 +21,8 @@ * */ -#ifndef SHARE_VM_GC_SHENANDOAH_C1_SHENANDOAHBARRIERSETC1_HPP -#define SHARE_VM_GC_SHENANDOAH_C1_SHENANDOAHBARRIERSETC1_HPP +#ifndef SHARE_GC_SHENANDOAH_C1_SHENANDOAHBARRIERSETC1_HPP +#define SHARE_GC_SHENANDOAH_C1_SHENANDOAHBARRIERSETC1_HPP #include "c1/c1_CodeStubs.hpp" #include "gc/shared/c1/barrierSetC1.hpp" @@ -85,7 +85,7 @@ #endif // PRODUCT }; -class ShenandoahWriteBarrierStub: public CodeStub { +class ShenandoahLoadReferenceBarrierStub: public CodeStub { friend class ShenandoahBarrierSetC1; private: LIR_Opr _obj; @@ -94,7 +94,7 @@ bool _needs_null_check; public: - ShenandoahWriteBarrierStub(LIR_Opr obj, LIR_Opr result, CodeEmitInfo* info, bool needs_null_check) : + ShenandoahLoadReferenceBarrierStub(LIR_Opr obj, LIR_Opr result, CodeEmitInfo* info, bool needs_null_check) : _obj(obj), _result(result), _info(info), _needs_null_check(needs_null_check) { assert(_obj->is_register(), "should be register"); @@ -113,7 +113,7 @@ visitor->do_temp(_result); } #ifndef PRODUCT - virtual void print_name(outputStream* out) const { out->print("ShenandoahWritePreBarrierStub"); } + virtual void print_name(outputStream* out) const { out->print("ShenandoahLoadReferenceBarrierStub"); } #endif // PRODUCT }; @@ -179,33 +179,30 @@ private: CodeBlob* _pre_barrier_c1_runtime_code_blob; - void pre_barrier(LIRAccess& access, LIR_Opr addr_opr, LIR_Opr pre_val); + void pre_barrier(LIRGenerator* gen, CodeEmitInfo* info, DecoratorSet decorators, LIR_Opr addr_opr, LIR_Opr pre_val); + + LIR_Opr load_reference_barrier(LIRGenerator* gen, LIR_Opr obj, CodeEmitInfo* info, bool need_null_check); + LIR_Opr storeval_barrier(LIRGenerator* gen, LIR_Opr obj, CodeEmitInfo* info, DecoratorSet decorators); - LIR_Opr read_barrier(LIRAccess& access, LIR_Opr obj, CodeEmitInfo* info, bool need_null_check); - LIR_Opr write_barrier(LIRAccess& access, LIR_Opr obj, CodeEmitInfo* info, bool need_null_check); - LIR_Opr storeval_barrier(LIRAccess& access, LIR_Opr obj, CodeEmitInfo* info, bool need_null_check); + LIR_Opr load_reference_barrier_impl(LIRGenerator* gen, LIR_Opr obj, CodeEmitInfo* info, bool need_null_check); - LIR_Opr read_barrier_impl(LIRAccess& access, LIR_Opr obj, CodeEmitInfo* info, bool need_null_check); - LIR_Opr write_barrier_impl(LIRAccess& access, LIR_Opr obj, CodeEmitInfo* info, bool need_null_check); + LIR_Opr ensure_in_register(LIRGenerator* gen, LIR_Opr obj); - LIR_Opr ensure_in_register(LIRAccess& access, LIR_Opr obj); public: CodeBlob* pre_barrier_c1_runtime_code_blob() { return _pre_barrier_c1_runtime_code_blob; } - virtual void store_at(LIRAccess& access, LIR_Opr value); - virtual void load_at(LIRAccess& access, LIR_Opr result); +protected: + + virtual void store_at_resolved(LIRAccess& access, LIR_Opr value); + virtual void load_at_resolved(LIRAccess& access, LIR_Opr result); virtual LIR_Opr atomic_cmpxchg_at_resolved(LIRAccess& access, LIRItem& cmp_value, LIRItem& new_value); + virtual LIR_Opr atomic_xchg_at_resolved(LIRAccess& access, LIRItem& value); - virtual LIR_Opr atomic_cmpxchg_at(LIRAccess& access, LIRItem& cmp_value, LIRItem& new_value); - virtual LIR_Opr atomic_xchg_at(LIRAccess& access, LIRItem& value); - virtual LIR_Opr atomic_add_at(LIRAccess& access, LIRItem& value); - - virtual LIR_Opr resolve_for_read(LIRAccess& access); - virtual LIR_Opr resolve_for_write(LIRAccess& access); +public: virtual void generate_c1_runtime_stubs(BufferBlob* buffer_blob); }; -#endif // SHARE_VM_GC_SHENANDOAH_C1_SHENANDOAHBARRIERSETC1_HPP +#endif // SHARE_GC_SHENANDOAH_C1_SHENANDOAHBARRIERSETC1_HPP diff --git a/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp b/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp --- a/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp +++ b/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp @@ -29,130 +29,70 @@ #include "gc/shenandoah/shenandoahThreadLocalData.hpp" #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp" #include "gc/shenandoah/c2/shenandoahSupport.hpp" +#include "opto/arraycopynode.hpp" +#include "opto/escape.hpp" #include "opto/graphKit.hpp" #include "opto/idealKit.hpp" #include "opto/macro.hpp" +#include "opto/movenode.hpp" +#include "opto/narrowptrnode.hpp" +#include "opto/rootnode.hpp" ShenandoahBarrierSetC2* ShenandoahBarrierSetC2::bsc2() { return reinterpret_cast(BarrierSet::barrier_set()->barrier_set_c2()); } ShenandoahBarrierSetC2State::ShenandoahBarrierSetC2State(Arena* comp_arena) - : _shenandoah_barriers(new (comp_arena) GrowableArray(comp_arena, 8, 0, NULL)) { + : _enqueue_barriers(new (comp_arena) GrowableArray(comp_arena, 8, 0, NULL)), + _load_reference_barriers(new (comp_arena) GrowableArray(comp_arena, 8, 0, NULL)) { } -int ShenandoahBarrierSetC2State::shenandoah_barriers_count() const { - return _shenandoah_barriers->length(); +int ShenandoahBarrierSetC2State::enqueue_barriers_count() const { + return _enqueue_barriers->length(); } -ShenandoahWriteBarrierNode* ShenandoahBarrierSetC2State::shenandoah_barrier(int idx) const { - return _shenandoah_barriers->at(idx); +ShenandoahEnqueueBarrierNode* ShenandoahBarrierSetC2State::enqueue_barrier(int idx) const { + return _enqueue_barriers->at(idx); } -void ShenandoahBarrierSetC2State::add_shenandoah_barrier(ShenandoahWriteBarrierNode * n) { - assert(!_shenandoah_barriers->contains(n), "duplicate entry in barrier list"); - _shenandoah_barriers->append(n); +void ShenandoahBarrierSetC2State::add_enqueue_barrier(ShenandoahEnqueueBarrierNode * n) { + assert(!_enqueue_barriers->contains(n), "duplicate entry in barrier list"); + _enqueue_barriers->append(n); } -void ShenandoahBarrierSetC2State::remove_shenandoah_barrier(ShenandoahWriteBarrierNode * n) { - if (_shenandoah_barriers->contains(n)) { - _shenandoah_barriers->remove(n); +void ShenandoahBarrierSetC2State::remove_enqueue_barrier(ShenandoahEnqueueBarrierNode * n) { + if (_enqueue_barriers->contains(n)) { + _enqueue_barriers->remove(n); } } -#define __ kit-> +int ShenandoahBarrierSetC2State::load_reference_barriers_count() const { + return _load_reference_barriers->length(); +} + +ShenandoahLoadReferenceBarrierNode* ShenandoahBarrierSetC2State::load_reference_barrier(int idx) const { + return _load_reference_barriers->at(idx); +} -Node* ShenandoahBarrierSetC2::shenandoah_read_barrier(GraphKit* kit, Node* obj) const { - if (ShenandoahReadBarrier) { - obj = shenandoah_read_barrier_impl(kit, obj, false, true, true); +void ShenandoahBarrierSetC2State::add_load_reference_barrier(ShenandoahLoadReferenceBarrierNode * n) { + assert(!_load_reference_barriers->contains(n), "duplicate entry in barrier list"); + _load_reference_barriers->append(n); +} + +void ShenandoahBarrierSetC2State::remove_load_reference_barrier(ShenandoahLoadReferenceBarrierNode * n) { + if (_load_reference_barriers->contains(n)) { + _load_reference_barriers->remove(n); } - return obj; } Node* ShenandoahBarrierSetC2::shenandoah_storeval_barrier(GraphKit* kit, Node* obj) const { if (ShenandoahStoreValEnqueueBarrier) { - obj = shenandoah_write_barrier(kit, obj); obj = shenandoah_enqueue_barrier(kit, obj); } - if (ShenandoahStoreValReadBarrier) { - obj = shenandoah_read_barrier_impl(kit, obj, true, false, false); - } return obj; } -Node* ShenandoahBarrierSetC2::shenandoah_read_barrier_impl(GraphKit* kit, Node* obj, bool use_ctrl, bool use_mem, bool allow_fromspace) const { - const Type* obj_type = obj->bottom_type(); - if (obj_type->higher_equal(TypePtr::NULL_PTR)) { - return obj; - } - const TypePtr* adr_type = ShenandoahBarrierNode::brooks_pointer_type(obj_type); - Node* mem = use_mem ? __ memory(adr_type) : __ immutable_memory(); - - if (! ShenandoahBarrierNode::needs_barrier(&__ gvn(), NULL, obj, mem, allow_fromspace)) { - // We know it is null, no barrier needed. - return obj; - } - - if (obj_type->meet(TypePtr::NULL_PTR) == obj_type->remove_speculative()) { - - // We don't know if it's null or not. Need null-check. - enum { _not_null_path = 1, _null_path, PATH_LIMIT }; - RegionNode* region = new RegionNode(PATH_LIMIT); - Node* phi = new PhiNode(region, obj_type); - Node* null_ctrl = __ top(); - Node* not_null_obj = __ null_check_oop(obj, &null_ctrl); - - region->init_req(_null_path, null_ctrl); - phi ->init_req(_null_path, __ zerocon(T_OBJECT)); - - Node* ctrl = use_ctrl ? __ control() : NULL; - ShenandoahReadBarrierNode* rb = new ShenandoahReadBarrierNode(ctrl, mem, not_null_obj, allow_fromspace); - Node* n = __ gvn().transform(rb); - - region->init_req(_not_null_path, __ control()); - phi ->init_req(_not_null_path, n); - - __ set_control(__ gvn().transform(region)); - __ record_for_igvn(region); - return __ gvn().transform(phi); - - } else { - // We know it is not null. Simple barrier is sufficient. - Node* ctrl = use_ctrl ? __ control() : NULL; - ShenandoahReadBarrierNode* rb = new ShenandoahReadBarrierNode(ctrl, mem, obj, allow_fromspace); - Node* n = __ gvn().transform(rb); - __ record_for_igvn(n); - return n; - } -} - -Node* ShenandoahBarrierSetC2::shenandoah_write_barrier_helper(GraphKit* kit, Node* obj, const TypePtr* adr_type) const { - ShenandoahWriteBarrierNode* wb = new ShenandoahWriteBarrierNode(kit->C, kit->control(), kit->memory(adr_type), obj); - Node* n = __ gvn().transform(wb); - if (n == wb) { // New barrier needs memory projection. - Node* proj = __ gvn().transform(new ShenandoahWBMemProjNode(n)); - __ set_memory(proj, adr_type); - } - return n; -} - -Node* ShenandoahBarrierSetC2::shenandoah_write_barrier(GraphKit* kit, Node* obj) const { - if (ShenandoahWriteBarrier) { - obj = shenandoah_write_barrier_impl(kit, obj); - } - return obj; -} - -Node* ShenandoahBarrierSetC2::shenandoah_write_barrier_impl(GraphKit* kit, Node* obj) const { - if (! ShenandoahBarrierNode::needs_barrier(&__ gvn(), NULL, obj, NULL, true)) { - return obj; - } - const Type* obj_type = obj->bottom_type(); - const TypePtr* adr_type = ShenandoahBarrierNode::brooks_pointer_type(obj_type); - Node* n = shenandoah_write_barrier_helper(kit, obj, adr_type); - __ record_for_igvn(n); - return n; -} +#define __ kit-> bool ShenandoahBarrierSetC2::satb_can_remove_pre_barrier(GraphKit* kit, PhaseTransform* phase, Node* adr, BasicType bt, uint adr_idx) const { @@ -299,7 +239,7 @@ Node* gc_state = __ AddP(no_base, tls, __ ConX(in_bytes(ShenandoahThreadLocalData::gc_state_offset()))); Node* ld = __ load(__ ctrl(), gc_state, TypeInt::BYTE, T_BYTE, Compile::AliasIdxRaw); marking = __ AndI(ld, __ ConI(ShenandoahHeap::MARKING)); - assert(ShenandoahWriteBarrierNode::is_gc_state_load(ld), "Should match the shape"); + assert(ShenandoahBarrierC2Support::is_gc_state_load(ld), "Should match the shape"); // if (!marking) __ if_then(marking, BoolTest::ne, zero, unlikely); { @@ -356,7 +296,7 @@ bool ShenandoahBarrierSetC2::is_shenandoah_wb_call(Node* call) { return call->is_CallLeaf() && - call->as_CallLeaf()->entry_point() == CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_JRT); + call->as_CallLeaf()->entry_point() == CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_JRT); } bool ShenandoahBarrierSetC2::is_shenandoah_marking_if(PhaseTransform *phase, Node* n) { @@ -544,14 +484,8 @@ return TypeFunc::make(domain, range); } -Node* ShenandoahBarrierSetC2::store_at(C2Access& access, C2AccessValue& val) const { - // TODO: Implement using proper barriers. - return BarrierSetC2::store_at(access, val); -} - Node* ShenandoahBarrierSetC2::store_at_resolved(C2Access& access, C2AccessValue& val) const { DecoratorSet decorators = access.decorators(); - GraphKit* kit = access.kit(); const TypePtr* adr_type = access.addr().type(); Node* adr = access.addr().node(); @@ -563,24 +497,34 @@ return BarrierSetC2::store_at_resolved(access, val); } - uint adr_idx = kit->C->get_alias_index(adr_type); - assert(adr_idx != Compile::AliasIdxTop, "use other store_to_memory factory" ); - Node* value = val.node(); - value = shenandoah_storeval_barrier(kit, value); - val.set_node(value); - shenandoah_write_barrier_pre(kit, true /* do_load */, /*kit->control(),*/ access.base(), adr, adr_idx, val.node(), - static_cast(val.type()), NULL /* pre_val */, access.type()); + if (access.is_parse_access()) { + C2ParseAccess& parse_access = static_cast(access); + GraphKit* kit = parse_access.kit(); + + uint adr_idx = kit->C->get_alias_index(adr_type); + assert(adr_idx != Compile::AliasIdxTop, "use other store_to_memory factory" ); + Node* value = val.node(); + value = shenandoah_storeval_barrier(kit, value); + val.set_node(value); + shenandoah_write_barrier_pre(kit, true /* do_load */, /*kit->control(),*/ access.base(), adr, adr_idx, val.node(), + static_cast(val.type()), NULL /* pre_val */, access.type()); + } else { + assert(access.is_opt_access(), "only for optimization passes"); + assert(((decorators & C2_TIGHTLY_COUPLED_ALLOC) != 0 || !ShenandoahSATBBarrier) && (decorators & C2_ARRAY_COPY) != 0, "unexpected caller of this code"); + C2OptAccess& opt_access = static_cast(access); + PhaseGVN& gvn = opt_access.gvn(); + MergeMemNode* mm = opt_access.mem(); + + if (ShenandoahStoreValEnqueueBarrier) { + Node* enqueue = gvn.transform(new ShenandoahEnqueueBarrierNode(val.node())); + val.set_node(enqueue); + } + } return BarrierSetC2::store_at_resolved(access, val); } -Node* ShenandoahBarrierSetC2::load_at(C2Access& access, const Type* val_type) const { - // TODO: Implement using proper barriers. - return BarrierSetC2::load_at(access, val_type); -} - Node* ShenandoahBarrierSetC2::load_at_resolved(C2Access& access, const Type* val_type) const { DecoratorSet decorators = access.decorators(); - GraphKit* kit = access.kit(); Node* adr = access.addr().node(); Node* obj = access.base(); @@ -592,9 +536,22 @@ bool is_unordered = (decorators & MO_UNORDERED) != 0; bool need_cpu_mem_bar = !is_unordered || mismatched || !on_heap; - Node* offset = adr->is_AddP() ? adr->in(AddPNode::Offset) : kit->top(); + Node* top = Compile::current()->top(); + + Node* offset = adr->is_AddP() ? adr->in(AddPNode::Offset) : top; Node* load = BarrierSetC2::load_at_resolved(access, val_type); + if (access.is_oop()) { + if (ShenandoahLoadRefBarrier) { + load = new ShenandoahLoadReferenceBarrierNode(NULL, load); + if (access.is_parse_access()) { + load = static_cast(access).kit()->gvn().transform(load); + } else { + load = static_cast(access).gvn().transform(load); + } + } + } + // If we are reading the value of the referent field of a Reference // object (either by using Unsafe directly or through reflection) // then, if SATB is enabled, we need to record the referent in an @@ -602,12 +559,16 @@ // Also we need to add memory barrier to prevent commoning reads // from this field across safepoint since GC can change its value. bool need_read_barrier = ShenandoahKeepAliveBarrier && - (on_heap && (on_weak || (unknown && offset != kit->top() && obj != kit->top()))); + (on_heap && (on_weak || (unknown && offset != top && obj != top))); if (!access.is_oop() || !need_read_barrier) { return load; } + assert(access.is_parse_access(), "entry not supported at optimization time"); + C2ParseAccess& parse_access = static_cast(access); + GraphKit* kit = parse_access.kit(); + if (on_weak) { // Use the pre-barrier to record the value in the referent field satb_write_barrier_pre(kit, false /* do_load */, @@ -625,50 +586,117 @@ return load; } -Node* ShenandoahBarrierSetC2::atomic_cmpxchg_val_at_resolved(C2AtomicAccess& access, Node* expected_val, - Node* val, const Type* value_type) const { +Node* ShenandoahBarrierSetC2::atomic_cmpxchg_val_at_resolved(C2AtomicParseAccess& access, Node* expected_val, + Node* new_val, const Type* value_type) const { GraphKit* kit = access.kit(); if (access.is_oop()) { - val = shenandoah_storeval_barrier(kit, val); + new_val = shenandoah_storeval_barrier(kit, new_val); shenandoah_write_barrier_pre(kit, false /* do_load */, NULL, NULL, max_juint, NULL, NULL, expected_val /* pre_val */, T_OBJECT); + MemNode::MemOrd mo = access.mem_node_mo(); + Node* mem = access.memory(); + Node* adr = access.addr().node(); + const TypePtr* adr_type = access.addr().type(); + Node* load_store = NULL; + +#ifdef _LP64 + if (adr->bottom_type()->is_ptr_to_narrowoop()) { + Node *newval_enc = kit->gvn().transform(new EncodePNode(new_val, new_val->bottom_type()->make_narrowoop())); + Node *oldval_enc = kit->gvn().transform(new EncodePNode(expected_val, expected_val->bottom_type()->make_narrowoop())); + if (ShenandoahCASBarrier) { + load_store = kit->gvn().transform(new ShenandoahCompareAndExchangeNNode(kit->control(), mem, adr, newval_enc, oldval_enc, adr_type, value_type->make_narrowoop(), mo)); + } else { + load_store = kit->gvn().transform(new CompareAndExchangeNNode(kit->control(), mem, adr, newval_enc, oldval_enc, adr_type, value_type->make_narrowoop(), mo)); + } + } else +#endif + { + if (ShenandoahCASBarrier) { + load_store = kit->gvn().transform(new ShenandoahCompareAndExchangePNode(kit->control(), mem, adr, new_val, expected_val, adr_type, value_type->is_oopptr(), mo)); + } else { + load_store = kit->gvn().transform(new CompareAndExchangePNode(kit->control(), mem, adr, new_val, expected_val, adr_type, value_type->is_oopptr(), mo)); + } + } + + access.set_raw_access(load_store); + pin_atomic_op(access); + +#ifdef _LP64 + if (adr->bottom_type()->is_ptr_to_narrowoop()) { + load_store = kit->gvn().transform(new DecodeNNode(load_store, load_store->get_ptr_type())); + } +#endif + load_store = kit->gvn().transform(new ShenandoahLoadReferenceBarrierNode(NULL, load_store)); + return load_store; } - return BarrierSetC2::atomic_cmpxchg_val_at_resolved(access, expected_val, val, value_type); + return BarrierSetC2::atomic_cmpxchg_val_at_resolved(access, expected_val, new_val, value_type); } -Node* ShenandoahBarrierSetC2::atomic_cmpxchg_val_at(C2AtomicAccess& access, Node* expected_val, - Node* new_val, const Type* val_type) const { - // TODO: Implement using proper barriers. - return BarrierSetC2::atomic_cmpxchg_val_at(access, expected_val, new_val, val_type); -} - -Node* ShenandoahBarrierSetC2::atomic_cmpxchg_bool_at_resolved(C2AtomicAccess& access, Node* expected_val, - Node* val, const Type* value_type) const { +Node* ShenandoahBarrierSetC2::atomic_cmpxchg_bool_at_resolved(C2AtomicParseAccess& access, Node* expected_val, + Node* new_val, const Type* value_type) const { GraphKit* kit = access.kit(); if (access.is_oop()) { - val = shenandoah_storeval_barrier(kit, val); + new_val = shenandoah_storeval_barrier(kit, new_val); shenandoah_write_barrier_pre(kit, false /* do_load */, NULL, NULL, max_juint, NULL, NULL, expected_val /* pre_val */, T_OBJECT); + DecoratorSet decorators = access.decorators(); + MemNode::MemOrd mo = access.mem_node_mo(); + Node* mem = access.memory(); + bool is_weak_cas = (decorators & C2_WEAK_CMPXCHG) != 0; + Node* load_store = NULL; + Node* adr = access.addr().node(); +#ifdef _LP64 + if (adr->bottom_type()->is_ptr_to_narrowoop()) { + Node *newval_enc = kit->gvn().transform(new EncodePNode(new_val, new_val->bottom_type()->make_narrowoop())); + Node *oldval_enc = kit->gvn().transform(new EncodePNode(expected_val, expected_val->bottom_type()->make_narrowoop())); + if (ShenandoahCASBarrier) { + if (is_weak_cas) { + load_store = kit->gvn().transform(new ShenandoahWeakCompareAndSwapNNode(kit->control(), mem, adr, newval_enc, oldval_enc, mo)); + } else { + load_store = kit->gvn().transform(new ShenandoahCompareAndSwapNNode(kit->control(), mem, adr, newval_enc, oldval_enc, mo)); + } + } else { + if (is_weak_cas) { + load_store = kit->gvn().transform(new WeakCompareAndSwapNNode(kit->control(), mem, adr, newval_enc, oldval_enc, mo)); + } else { + load_store = kit->gvn().transform(new CompareAndSwapNNode(kit->control(), mem, adr, newval_enc, oldval_enc, mo)); + } + } + } else +#endif + { + if (ShenandoahCASBarrier) { + if (is_weak_cas) { + load_store = kit->gvn().transform(new ShenandoahWeakCompareAndSwapPNode(kit->control(), mem, adr, new_val, expected_val, mo)); + } else { + load_store = kit->gvn().transform(new ShenandoahCompareAndSwapPNode(kit->control(), mem, adr, new_val, expected_val, mo)); + } + } else { + if (is_weak_cas) { + load_store = kit->gvn().transform(new WeakCompareAndSwapPNode(kit->control(), mem, adr, new_val, expected_val, mo)); + } else { + load_store = kit->gvn().transform(new CompareAndSwapPNode(kit->control(), mem, adr, new_val, expected_val, mo)); + } + } + } + access.set_raw_access(load_store); + pin_atomic_op(access); + return load_store; } - return BarrierSetC2::atomic_cmpxchg_bool_at_resolved(access, expected_val, val, value_type); + return BarrierSetC2::atomic_cmpxchg_bool_at_resolved(access, expected_val, new_val, value_type); } -Node* ShenandoahBarrierSetC2::atomic_cmpxchg_bool_at(C2AtomicAccess& access, Node* expected_val, - Node* new_val, const Type* val_type) const { - // TODO: Implement using proper barriers. - return BarrierSetC2::atomic_cmpxchg_bool_at(access, expected_val, new_val, val_type); -} - -Node* ShenandoahBarrierSetC2::atomic_xchg_at_resolved(C2AtomicAccess& access, Node* val, const Type* value_type) const { +Node* ShenandoahBarrierSetC2::atomic_xchg_at_resolved(C2AtomicParseAccess& access, Node* val, const Type* value_type) const { GraphKit* kit = access.kit(); if (access.is_oop()) { val = shenandoah_storeval_barrier(kit, val); } Node* result = BarrierSetC2::atomic_xchg_at_resolved(access, val, value_type); if (access.is_oop()) { + result = kit->gvn().transform(new ShenandoahLoadReferenceBarrierNode(NULL, result)); shenandoah_write_barrier_pre(kit, false /* do_load */, NULL, NULL, max_juint, NULL, NULL, result /* pre_val */, T_OBJECT); @@ -676,31 +704,38 @@ return result; } -Node* ShenandoahBarrierSetC2::atomic_xchg_at(C2AtomicAccess& access, Node* new_val, const Type* value_type) const { - // TODO: Implement using proper barriers. - return BarrierSetC2::atomic_xchg_at(access, new_val, value_type); -} - -Node* ShenandoahBarrierSetC2::atomic_add_at(C2AtomicAccess& access, Node* new_val, const Type* value_type) const { - // TODO: Implement using proper barriers. - return BarrierSetC2::atomic_add_at(access, new_val, value_type); -} - void ShenandoahBarrierSetC2::clone(GraphKit* kit, Node* src, Node* dst, Node* size, bool is_array) const { - // TODO: Implement using proper barriers. + assert(!src->is_AddP(), "unexpected input"); BarrierSetC2::clone(kit, src, dst, size, is_array); } -Node* ShenandoahBarrierSetC2::resolve_for_read(GraphKit* kit, Node* n) const { - return shenandoah_read_barrier(kit, n); -} +Node* ShenandoahBarrierSetC2::obj_allocate(PhaseMacroExpand* macro, Node* ctrl, Node* mem, Node* toobig_false, Node* size_in_bytes, + Node*& i_o, Node*& needgc_ctrl, + Node*& fast_oop_ctrl, Node*& fast_oop_rawmem, + intx prefetch_lines) const { + PhaseIterGVN& igvn = macro->igvn(); + + // Allocate several words more for the Shenandoah brooks pointer. + size_in_bytes = new AddXNode(size_in_bytes, igvn.MakeConX(ShenandoahBrooksPointer::byte_size())); + macro->transform_later(size_in_bytes); -Node* ShenandoahBarrierSetC2::resolve_for_write(GraphKit* kit, Node* n) const { - return shenandoah_write_barrier(kit, n); + Node* fast_oop = BarrierSetC2::obj_allocate(macro, ctrl, mem, toobig_false, size_in_bytes, + i_o, needgc_ctrl, fast_oop_ctrl, fast_oop_rawmem, + prefetch_lines); + + // Bump up object for Shenandoah brooks pointer. + fast_oop = new AddPNode(macro->top(), fast_oop, igvn.MakeConX(ShenandoahBrooksPointer::byte_size())); + macro->transform_later(fast_oop); + + // Initialize Shenandoah brooks pointer to point to the object itself. + fast_oop_rawmem = macro->make_store(fast_oop_ctrl, fast_oop_rawmem, fast_oop, ShenandoahBrooksPointer::byte_offset(), fast_oop, T_OBJECT); + + return fast_oop; } // Support for GC barriers emitted during parsing bool ShenandoahBarrierSetC2::is_gc_barrier_node(Node* node) const { + if (node->Opcode() == Op_ShenandoahLoadReferenceBarrier) return true; if (node->Opcode() != Op_CallLeaf && node->Opcode() != Op_CallLeafNoFP) { return false; } @@ -715,23 +750,132 @@ } Node* ShenandoahBarrierSetC2::step_over_gc_barrier(Node* c) const { - return ShenandoahBarrierNode::skip_through_barrier(c); + if (c->Opcode() == Op_ShenandoahLoadReferenceBarrier) { + return c->in(ShenandoahLoadReferenceBarrierNode::ValueIn); + } + if (c->Opcode() == Op_ShenandoahEnqueueBarrier) { + c = c->in(1); + } + return c; } -bool ShenandoahBarrierSetC2::array_copy_requires_gc_barriers(BasicType type) const { +bool ShenandoahBarrierSetC2::expand_barriers(Compile* C, PhaseIterGVN& igvn) const { + return !ShenandoahBarrierC2Support::expand(C, igvn); +} + +bool ShenandoahBarrierSetC2::optimize_loops(PhaseIdealLoop* phase, LoopOptsMode mode, VectorSet& visited, Node_Stack& nstack, Node_List& worklist) const { + if (mode == LoopOptsShenandoahExpand) { + assert(UseShenandoahGC, "only for shenandoah"); + ShenandoahBarrierC2Support::pin_and_expand(phase); + return true; + } else if (mode == LoopOptsShenandoahPostExpand) { + assert(UseShenandoahGC, "only for shenandoah"); + visited.Clear(); + ShenandoahBarrierC2Support::optimize_after_expansion(visited, nstack, worklist, phase); + return true; + } return false; } +bool ShenandoahBarrierSetC2::array_copy_requires_gc_barriers(bool tightly_coupled_alloc, BasicType type, bool is_clone, ArrayCopyPhase phase) const { + bool is_oop = type == T_OBJECT || type == T_ARRAY; + if (!is_oop) { + return false; + } + if (tightly_coupled_alloc) { + if (phase == Optimization) { + return false; + } + return !is_clone; + } + if (phase == Optimization) { + return !ShenandoahStoreValEnqueueBarrier; + } + return true; +} + +bool ShenandoahBarrierSetC2::clone_needs_postbarrier(ArrayCopyNode *ac, PhaseIterGVN& igvn) { + Node* src = ac->in(ArrayCopyNode::Src); + const TypeOopPtr* src_type = igvn.type(src)->is_oopptr(); + if (src_type->isa_instptr() != NULL) { + ciInstanceKlass* ik = src_type->klass()->as_instance_klass(); + if ((src_type->klass_is_exact() || (!ik->is_interface() && !ik->has_subklass())) && !ik->has_injected_fields()) { + if (ik->has_object_fields()) { + return true; + } else { + if (!src_type->klass_is_exact()) { + igvn.C->dependencies()->assert_leaf_type(ik); + } + } + } else { + return true; + } + } else if (src_type->isa_aryptr()) { + BasicType src_elem = src_type->klass()->as_array_klass()->element_type()->basic_type(); + if (src_elem == T_OBJECT || src_elem == T_ARRAY) { + return true; + } + } else { + return true; + } + return false; +} + +void ShenandoahBarrierSetC2::clone_barrier_at_expansion(ArrayCopyNode* ac, Node* call, PhaseIterGVN& igvn) const { + assert(ac->is_clonebasic(), "no other kind of arraycopy here"); + + if (!clone_needs_postbarrier(ac, igvn)) { + BarrierSetC2::clone_barrier_at_expansion(ac, call, igvn); + return; + } + + const TypePtr* raw_adr_type = TypeRawPtr::BOTTOM; + Node* c = new ProjNode(call,TypeFunc::Control); + c = igvn.transform(c); + Node* m = new ProjNode(call, TypeFunc::Memory); + m = igvn.transform(m); + + Node* dest = ac->in(ArrayCopyNode::Dest); + assert(dest->is_AddP(), "bad input"); + Node* barrier_call = new CallLeafNode(ShenandoahBarrierSetC2::shenandoah_clone_barrier_Type(), + CAST_FROM_FN_PTR(address, ShenandoahRuntime::shenandoah_clone_barrier), + "shenandoah_clone_barrier", raw_adr_type); + barrier_call->init_req(TypeFunc::Control, c); + barrier_call->init_req(TypeFunc::I_O , igvn.C->top()); + barrier_call->init_req(TypeFunc::Memory , m); + barrier_call->init_req(TypeFunc::ReturnAdr, igvn.C->top()); + barrier_call->init_req(TypeFunc::FramePtr, igvn.C->top()); + barrier_call->init_req(TypeFunc::Parms+0, dest->in(AddPNode::Base)); + + barrier_call = igvn.transform(barrier_call); + c = new ProjNode(barrier_call,TypeFunc::Control); + c = igvn.transform(c); + m = new ProjNode(barrier_call, TypeFunc::Memory); + m = igvn.transform(m); + + Node* out_c = ac->proj_out(TypeFunc::Control); + Node* out_m = ac->proj_out(TypeFunc::Memory); + igvn.replace_node(out_c, c); + igvn.replace_node(out_m, m); +} + + // Support for macro expanded GC barriers void ShenandoahBarrierSetC2::register_potential_barrier_node(Node* node) const { - if (node->Opcode() == Op_ShenandoahWriteBarrier) { - state()->add_shenandoah_barrier((ShenandoahWriteBarrierNode*) node); + if (node->Opcode() == Op_ShenandoahEnqueueBarrier) { + state()->add_enqueue_barrier((ShenandoahEnqueueBarrierNode*) node); + } + if (node->Opcode() == Op_ShenandoahLoadReferenceBarrier) { + state()->add_load_reference_barrier((ShenandoahLoadReferenceBarrierNode*) node); } } void ShenandoahBarrierSetC2::unregister_potential_barrier_node(Node* node) const { - if (node->Opcode() == Op_ShenandoahWriteBarrier) { - state()->remove_shenandoah_barrier((ShenandoahWriteBarrierNode*) node); + if (node->Opcode() == Op_ShenandoahEnqueueBarrier) { + state()->remove_enqueue_barrier((ShenandoahEnqueueBarrierNode*) node); + } + if (node->Opcode() == Op_ShenandoahLoadReferenceBarrier) { + state()->remove_load_reference_barrier((ShenandoahLoadReferenceBarrierNode*) node); } } @@ -762,17 +906,33 @@ call->del_req(call->req()-1); } -void ShenandoahBarrierSetC2::enqueue_useful_gc_barrier(Unique_Node_List &worklist, Node* node) const { +void ShenandoahBarrierSetC2::enqueue_useful_gc_barrier(PhaseIterGVN* igvn, Node* node) const { + if (node->Opcode() == Op_AddP && ShenandoahBarrierSetC2::has_only_shenandoah_wb_pre_uses(node)) { + igvn->add_users_to_worklist(node); + } } -void ShenandoahBarrierSetC2::eliminate_useless_gc_barriers(Unique_Node_List &useful) const { - for (int i = state()->shenandoah_barriers_count()-1; i >= 0; i--) { - ShenandoahWriteBarrierNode* n = state()->shenandoah_barrier(i); - if (!useful.member(n)) { - state()->remove_shenandoah_barrier(n); +void ShenandoahBarrierSetC2::eliminate_useless_gc_barriers(Unique_Node_List &useful, Compile* C) const { + for (uint i = 0; i < useful.size(); i++) { + Node* n = useful.at(i); + if (n->Opcode() == Op_AddP && ShenandoahBarrierSetC2::has_only_shenandoah_wb_pre_uses(n)) { + for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) { + C->record_for_igvn(n->fast_out(i)); + } } } - + for (int i = state()->enqueue_barriers_count() - 1; i >= 0; i--) { + ShenandoahEnqueueBarrierNode* n = state()->enqueue_barrier(i); + if (!useful.member(n)) { + state()->remove_enqueue_barrier(n); + } + } + for (int i = state()->load_reference_barriers_count() - 1; i >= 0; i--) { + ShenandoahLoadReferenceBarrierNode* n = state()->load_reference_barrier(i); + if (!useful.member(n)) { + state()->remove_load_reference_barrier(n); + } + } } void ShenandoahBarrierSetC2::add_users_to_worklist(Unique_Node_List* worklist) const {} @@ -788,15 +948,74 @@ // If the BarrierSetC2 state has kept macro nodes in its compilation unit state to be // expanded later, then now is the time to do so. bool ShenandoahBarrierSetC2::expand_macro_nodes(PhaseMacroExpand* macro) const { return false; } -void ShenandoahBarrierSetC2::verify_gc_barriers(bool post_parse) const { + #ifdef ASSERT - if (ShenandoahVerifyOptoBarriers && !post_parse) { - ShenandoahBarrierNode::verify(Compile::current()->root()); +void ShenandoahBarrierSetC2::verify_gc_barriers(Compile* compile, CompilePhase phase) const { + if (ShenandoahVerifyOptoBarriers && phase == BarrierSetC2::BeforeExpand) { + ShenandoahBarrierC2Support::verify(Compile::current()->root()); + } else if (phase == BarrierSetC2::BeforeCodeGen) { + // Verify G1 pre-barriers + const int marking_offset = in_bytes(ShenandoahThreadLocalData::satb_mark_queue_active_offset()); + + ResourceArea *area = Thread::current()->resource_area(); + Unique_Node_List visited(area); + Node_List worklist(area); + // We're going to walk control flow backwards starting from the Root + worklist.push(compile->root()); + while (worklist.size() > 0) { + Node *x = worklist.pop(); + if (x == NULL || x == compile->top()) continue; + if (visited.member(x)) { + continue; + } else { + visited.push(x); + } + + if (x->is_Region()) { + for (uint i = 1; i < x->req(); i++) { + worklist.push(x->in(i)); + } + } else { + worklist.push(x->in(0)); + // We are looking for the pattern: + // /->ThreadLocal + // If->Bool->CmpI->LoadB->AddP->ConL(marking_offset) + // \->ConI(0) + // We want to verify that the If and the LoadB have the same control + // See GraphKit::g1_write_barrier_pre() + if (x->is_If()) { + IfNode *iff = x->as_If(); + if (iff->in(1)->is_Bool() && iff->in(1)->in(1)->is_Cmp()) { + CmpNode *cmp = iff->in(1)->in(1)->as_Cmp(); + if (cmp->Opcode() == Op_CmpI && cmp->in(2)->is_Con() && cmp->in(2)->bottom_type()->is_int()->get_con() == 0 + && cmp->in(1)->is_Load()) { + LoadNode *load = cmp->in(1)->as_Load(); + if (load->Opcode() == Op_LoadB && load->in(2)->is_AddP() && load->in(2)->in(2)->Opcode() == Op_ThreadLocal + && load->in(2)->in(3)->is_Con() + && load->in(2)->in(3)->bottom_type()->is_intptr_t()->get_con() == marking_offset) { + + Node *if_ctrl = iff->in(0); + Node *load_ctrl = load->in(0); + + if (if_ctrl != load_ctrl) { + // Skip possible CProj->NeverBranch in infinite loops + if ((if_ctrl->is_Proj() && if_ctrl->Opcode() == Op_CProj) + && (if_ctrl->in(0)->is_MultiBranch() && if_ctrl->in(0)->Opcode() == Op_NeverBranch)) { + if_ctrl = if_ctrl->in(0)->in(0); + } + } + assert(load_ctrl != NULL && if_ctrl == load_ctrl, "controls must match"); + } + } + } + } + } + } } +} #endif -} -Node* ShenandoahBarrierSetC2::ideal_node(PhaseGVN *phase, Node* n, bool can_reshape) const { +Node* ShenandoahBarrierSetC2::ideal_node(PhaseGVN* phase, Node* n, bool can_reshape) const { if (is_shenandoah_wb_pre_call(n)) { uint cnt = ShenandoahBarrierSetC2::write_ref_field_pre_entry_Type()->domain()->cnt(); if (n->req() > cnt) { @@ -810,13 +1029,65 @@ } } } + if (n->Opcode() == Op_CmpP) { + Node* in1 = n->in(1); + Node* in2 = n->in(2); + if (in1->bottom_type() == TypePtr::NULL_PTR) { + in2 = step_over_gc_barrier(in2); + } + if (in2->bottom_type() == TypePtr::NULL_PTR) { + in1 = step_over_gc_barrier(in1); + } + PhaseIterGVN* igvn = phase->is_IterGVN(); + if (in1 != n->in(1)) { + if (igvn != NULL) { + n->set_req_X(1, in1, igvn); + } else { + n->set_req(1, in1); + } + assert(in2 == n->in(2), "only one change"); + return n; + } + if (in2 != n->in(2)) { + if (igvn != NULL) { + n->set_req_X(2, in2, igvn); + } else { + n->set_req(2, in2); + } + return n; + } + } else if (can_reshape && + n->Opcode() == Op_If && + ShenandoahBarrierC2Support::is_heap_stable_test(n) && + n->in(0) != NULL) { + Node* dom = n->in(0); + Node* prev_dom = n; + int op = n->Opcode(); + int dist = 16; + // Search up the dominator tree for another heap stable test + while (dom->Opcode() != op || // Not same opcode? + !ShenandoahBarrierC2Support::is_heap_stable_test(dom) || // Not same input 1? + prev_dom->in(0) != dom) { // One path of test does not dominate? + if (dist < 0) return NULL; + + dist--; + prev_dom = dom; + dom = IfNode::up_one_dom(dom); + if (!dom) return NULL; + } + + // Check that we did not follow a loop back to ourselves + if (n == dom) { + return NULL; + } + + return n->as_If()->dominated_by(prev_dom, phase->is_IterGVN()); + } + return NULL; } bool ShenandoahBarrierSetC2::has_only_shenandoah_wb_pre_uses(Node* n) { - if (!UseShenandoahGC) { - return false; - } for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) { Node* u = n->fast_out(i); if (!is_shenandoah_wb_pre_call(u)) { @@ -825,3 +1096,117 @@ } return n->outcnt() > 0; } + +bool ShenandoahBarrierSetC2::escape_add_to_con_graph(ConnectionGraph* conn_graph, PhaseGVN* gvn, Unique_Node_List* delayed_worklist, Node* n, uint opcode) const { + switch (opcode) { + case Op_ShenandoahCompareAndExchangeP: + case Op_ShenandoahCompareAndExchangeN: + conn_graph->add_objload_to_connection_graph(n, delayed_worklist); + // fallthrough + case Op_ShenandoahWeakCompareAndSwapP: + case Op_ShenandoahWeakCompareAndSwapN: + case Op_ShenandoahCompareAndSwapP: + case Op_ShenandoahCompareAndSwapN: + conn_graph->add_to_congraph_unsafe_access(n, opcode, delayed_worklist); + return true; + case Op_StoreP: { + Node* adr = n->in(MemNode::Address); + const Type* adr_type = gvn->type(adr); + // Pointer stores in G1 barriers looks like unsafe access. + // Ignore such stores to be able scalar replace non-escaping + // allocations. + if (adr_type->isa_rawptr() && adr->is_AddP()) { + Node* base = conn_graph->get_addp_base(adr); + if (base->Opcode() == Op_LoadP && + base->in(MemNode::Address)->is_AddP()) { + adr = base->in(MemNode::Address); + Node* tls = conn_graph->get_addp_base(adr); + if (tls->Opcode() == Op_ThreadLocal) { + int offs = (int) gvn->find_intptr_t_con(adr->in(AddPNode::Offset), Type::OffsetBot); + const int buf_offset = in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()); + if (offs == buf_offset) { + return true; // Pre barrier previous oop value store. + } + } + } + } + return false; + } + case Op_ShenandoahEnqueueBarrier: + conn_graph->add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(1), delayed_worklist); + break; + case Op_ShenandoahLoadReferenceBarrier: + conn_graph->add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(ShenandoahLoadReferenceBarrierNode::ValueIn), delayed_worklist); + return true; + default: + // Nothing + break; + } + return false; +} + +bool ShenandoahBarrierSetC2::escape_add_final_edges(ConnectionGraph* conn_graph, PhaseGVN* gvn, Node* n, uint opcode) const { + switch (opcode) { + case Op_ShenandoahCompareAndExchangeP: + case Op_ShenandoahCompareAndExchangeN: { + Node *adr = n->in(MemNode::Address); + conn_graph->add_local_var_and_edge(n, PointsToNode::NoEscape, adr, NULL); + // fallthrough + } + case Op_ShenandoahCompareAndSwapP: + case Op_ShenandoahCompareAndSwapN: + case Op_ShenandoahWeakCompareAndSwapP: + case Op_ShenandoahWeakCompareAndSwapN: + return conn_graph->add_final_edges_unsafe_access(n, opcode); + case Op_ShenandoahEnqueueBarrier: + conn_graph->add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(1), NULL); + return true; + case Op_ShenandoahLoadReferenceBarrier: + conn_graph->add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(ShenandoahLoadReferenceBarrierNode::ValueIn), NULL); + return true; + default: + // Nothing + break; + } + return false; +} + +bool ShenandoahBarrierSetC2::escape_has_out_with_unsafe_object(Node* n) const { + return n->has_out_with(Op_ShenandoahCompareAndExchangeP) || n->has_out_with(Op_ShenandoahCompareAndExchangeN) || + n->has_out_with(Op_ShenandoahCompareAndSwapP, Op_ShenandoahCompareAndSwapN, Op_ShenandoahWeakCompareAndSwapP, Op_ShenandoahWeakCompareAndSwapN); + +} + +bool ShenandoahBarrierSetC2::escape_is_barrier_node(Node* n) const { + return n->Opcode() == Op_ShenandoahLoadReferenceBarrier; +} + +bool ShenandoahBarrierSetC2::matcher_find_shared_post_visit(Matcher* matcher, Node* n, uint opcode) const { + switch (opcode) { + case Op_ShenandoahCompareAndExchangeP: + case Op_ShenandoahCompareAndExchangeN: + case Op_ShenandoahWeakCompareAndSwapP: + case Op_ShenandoahWeakCompareAndSwapN: + case Op_ShenandoahCompareAndSwapP: + case Op_ShenandoahCompareAndSwapN: { // Convert trinary to binary-tree + Node* newval = n->in(MemNode::ValueIn); + Node* oldval = n->in(LoadStoreConditionalNode::ExpectedIn); + Node* pair = new BinaryNode(oldval, newval); + n->set_req(MemNode::ValueIn,pair); + n->del_req(LoadStoreConditionalNode::ExpectedIn); + return true; + } + default: + break; + } + return false; +} + +bool ShenandoahBarrierSetC2::matcher_is_store_load_barrier(Node* x, uint xop) const { + return xop == Op_ShenandoahCompareAndExchangeP || + xop == Op_ShenandoahCompareAndExchangeN || + xop == Op_ShenandoahWeakCompareAndSwapP || + xop == Op_ShenandoahWeakCompareAndSwapN || + xop == Op_ShenandoahCompareAndSwapN || + xop == Op_ShenandoahCompareAndSwapP; +} diff --git a/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.hpp b/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.hpp --- a/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.hpp +++ b/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.hpp @@ -21,8 +21,8 @@ * */ -#ifndef SHARE_VM_GC_SHENANDOAH_C2_SHENANDOAHBARRIERSETC2_HPP -#define SHARE_VM_GC_SHENANDOAH_C2_SHENANDOAHBARRIERSETC2_HPP +#ifndef SHARE_GC_SHENANDOAH_C2_SHENANDOAHBARRIERSETC2_HPP +#define SHARE_GC_SHENANDOAH_C2_SHENANDOAHBARRIERSETC2_HPP #include "gc/shared/c2/barrierSetC2.hpp" #include "gc/shenandoah/c2/shenandoahSupport.hpp" @@ -30,14 +30,21 @@ class ShenandoahBarrierSetC2State : public ResourceObj { private: - GrowableArray* _shenandoah_barriers; + GrowableArray* _enqueue_barriers; + GrowableArray* _load_reference_barriers; public: ShenandoahBarrierSetC2State(Arena* comp_arena); - int shenandoah_barriers_count() const; - ShenandoahWriteBarrierNode* shenandoah_barrier(int idx) const; - void add_shenandoah_barrier(ShenandoahWriteBarrierNode * n); - void remove_shenandoah_barrier(ShenandoahWriteBarrierNode * n); + + int enqueue_barriers_count() const; + ShenandoahEnqueueBarrierNode* enqueue_barrier(int idx) const; + void add_enqueue_barrier(ShenandoahEnqueueBarrierNode* n); + void remove_enqueue_barrier(ShenandoahEnqueueBarrierNode * n); + + int load_reference_barriers_count() const; + ShenandoahLoadReferenceBarrierNode* load_reference_barrier(int idx) const; + void add_load_reference_barrier(ShenandoahLoadReferenceBarrierNode* n); + void remove_load_reference_barrier(ShenandoahLoadReferenceBarrierNode * n); }; class ShenandoahBarrierSetC2 : public BarrierSetC2 { @@ -66,16 +73,13 @@ BasicType bt) const; Node* shenandoah_enqueue_barrier(GraphKit* kit, Node* val) const; - Node* shenandoah_read_barrier(GraphKit* kit, Node* obj) const; Node* shenandoah_storeval_barrier(GraphKit* kit, Node* obj) const; - Node* shenandoah_write_barrier(GraphKit* kit, Node* obj) const; - Node* shenandoah_read_barrier_impl(GraphKit* kit, Node* obj, bool use_ctrl, bool use_mem, bool allow_fromspace) const; - Node* shenandoah_write_barrier_impl(GraphKit* kit, Node* obj) const; - Node* shenandoah_write_barrier_helper(GraphKit* kit, Node* obj, const TypePtr* adr_type) const; void insert_pre_barrier(GraphKit* kit, Node* base_oop, Node* offset, Node* pre_val, bool need_mem_bar) const; + static bool clone_needs_postbarrier(ArrayCopyNode *ac, PhaseIterGVN& igvn); + protected: virtual Node* load_at_resolved(C2Access& access, const Type* val_type) const; virtual Node* store_at_resolved(C2Access& access, C2AccessValue& val) const; @@ -99,22 +103,15 @@ static const TypeFunc* write_ref_field_pre_entry_Type(); static const TypeFunc* shenandoah_clone_barrier_Type(); static const TypeFunc* shenandoah_write_barrier_Type(); + virtual bool has_load_barriers() const { return true; } // This is the entry-point for the backend to perform accesses through the Access API. - virtual Node* store_at(C2Access& access, C2AccessValue& val) const; - virtual Node* load_at(C2Access& access, const Type* val_type) const; - - virtual Node* atomic_cmpxchg_val_at(C2AtomicAccess& access, Node* expected_val, - Node* new_val, const Type* val_type) const; - virtual Node* atomic_cmpxchg_bool_at(C2AtomicAccess& access, Node* expected_val, - Node* new_val, const Type* val_type) const; - virtual Node* atomic_xchg_at(C2AtomicAccess& access, Node* new_val, const Type* value_type) const; - virtual Node* atomic_add_at(C2AtomicAccess& access, Node* new_val, const Type* value_type) const; - virtual void clone(GraphKit* kit, Node* src, Node* dst, Node* size, bool is_array) const; - virtual Node* resolve_for_read(GraphKit* kit, Node* n) const; - virtual Node* resolve_for_write(GraphKit* kit, Node* n) const; + virtual Node* obj_allocate(PhaseMacroExpand* macro, Node* ctrl, Node* mem, Node* toobig_false, Node* size_in_bytes, + Node*& i_o, Node*& needgc_ctrl, + Node*& fast_oop_ctrl, Node*& fast_oop_rawmem, + intx prefetch_lines) const; // These are general helper methods used by C2 virtual bool array_copy_requires_gc_barriers(BasicType type) const; @@ -122,13 +119,17 @@ // Support for GC barriers emitted during parsing virtual bool is_gc_barrier_node(Node* node) const; virtual Node* step_over_gc_barrier(Node* c) const; + virtual bool expand_barriers(Compile* C, PhaseIterGVN& igvn) const; + virtual bool optimize_loops(PhaseIdealLoop* phase, LoopOptsMode mode, VectorSet& visited, Node_Stack& nstack, Node_List& worklist) const; + virtual bool strip_mined_loops_expanded(LoopOptsMode mode) const { return mode == LoopOptsShenandoahExpand || mode == LoopOptsShenandoahPostExpand; } + virtual bool is_gc_specific_loop_opts_pass(LoopOptsMode mode) const { return mode == LoopOptsShenandoahExpand || mode == LoopOptsShenandoahPostExpand; } // Support for macro expanded GC barriers virtual void register_potential_barrier_node(Node* node) const; virtual void unregister_potential_barrier_node(Node* node) const; - virtual void eliminate_gc_barrier(PhaseMacroExpand* macro, Node* node) const; virtual void enqueue_useful_gc_barrier(Unique_Node_List &worklist, Node* node) const; virtual void eliminate_useless_gc_barriers(Unique_Node_List &useful) const; + virtual void eliminate_useless_gc_barriers(Unique_Node_List &useful, Compile* C) const; virtual void add_users_to_worklist(Unique_Node_List* worklist) const; // Allow barrier sets to have shared state that is preserved across a compilation unit. @@ -137,9 +138,20 @@ // If the BarrierSetC2 state has kept macro nodes in its compilation unit state to be // expanded later, then now is the time to do so. virtual bool expand_macro_nodes(PhaseMacroExpand* macro) const; - virtual void verify_gc_barriers(bool post_parse) const; + +#ifdef ASSERT + virtual void verify_gc_barriers(bool post_parse) const {} +#endif virtual Node* ideal_node(PhaseGVN* phase, Node* n, bool can_reshape) const; + + virtual bool escape_add_to_con_graph(ConnectionGraph* conn_graph, PhaseGVN* gvn, Unique_Node_List* delayed_worklist, Node* n, uint opcode) const; + virtual bool escape_add_final_edges(ConnectionGraph* conn_graph, PhaseGVN* gvn, Node* n, uint opcode) const; + virtual bool escape_has_out_with_unsafe_object(Node* n) const; + virtual bool escape_is_barrier_node(Node* n) const; + + virtual bool matcher_find_shared_post_visit(Matcher* matcher, Node* n, uint opcode) const; + virtual bool matcher_is_store_load_barrier(Node* x, uint xop) const; }; -#endif // SHARE_VM_GC_SHENANDOAH_C2_SHENANDOAHBARRIERSETC2_HPP +#endif // SHARE_GC_SHENANDOAH_C2_SHENANDOAHBARRIERSETC2_HPP diff --git a/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp b/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp --- a/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp +++ b/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp @@ -41,384 +41,28 @@ #include "opto/runtime.hpp" #include "opto/subnode.hpp" -Node* ShenandoahBarrierNode::skip_through_barrier(Node* n) { - if (!UseShenandoahGC) { - return n; - } - if (n == NULL) { - return NULL; - } - if (n->Opcode() == Op_ShenandoahEnqueueBarrier) { - n = n->in(1); - } - - if (n->is_ShenandoahBarrier()) { - return n->in(ValueIn); - } else if (n->is_Phi() && - n->req() == 3 && - n->in(1) != NULL && - n->in(1)->is_ShenandoahBarrier() && - n->in(2) != NULL && - n->in(2)->bottom_type() == TypePtr::NULL_PTR && - n->in(0) != NULL && - n->in(0)->in(1) != NULL && - n->in(0)->in(1)->is_IfProj() && - n->in(0)->in(2) != NULL && - n->in(0)->in(2)->is_IfProj() && - n->in(0)->in(1)->in(0) != NULL && - n->in(0)->in(1)->in(0) == n->in(0)->in(2)->in(0) && - n->in(1)->in(ValueIn)->Opcode() == Op_CastPP) { - Node* iff = n->in(0)->in(1)->in(0); - Node* res = n->in(1)->in(ValueIn)->in(1); - if (iff->is_If() && - iff->in(1) != NULL && - iff->in(1)->is_Bool() && - iff->in(1)->as_Bool()->_test._test == BoolTest::ne && - iff->in(1)->in(1) != NULL && - iff->in(1)->in(1)->Opcode() == Op_CmpP && - iff->in(1)->in(1)->in(1) != NULL && - iff->in(1)->in(1)->in(1) == res && - iff->in(1)->in(1)->in(2) != NULL && - iff->in(1)->in(1)->in(2)->bottom_type() == TypePtr::NULL_PTR) { - return res; - } - } - return n; -} - -bool ShenandoahBarrierNode::needs_barrier(PhaseGVN* phase, ShenandoahBarrierNode* orig, Node* n, Node* rb_mem, bool allow_fromspace) { - Unique_Node_List visited; - return needs_barrier_impl(phase, orig, n, rb_mem, allow_fromspace, visited); -} - -bool ShenandoahBarrierNode::needs_barrier_impl(PhaseGVN* phase, ShenandoahBarrierNode* orig, Node* n, Node* rb_mem, bool allow_fromspace, Unique_Node_List &visited) { - if (visited.member(n)) { - return false; // Been there. - } - visited.push(n); - - if (n->is_Allocate()) { - return false; - } - - if (n->is_Call()) { - return true; - } - - const Type* type = phase->type(n); - if (type == Type::TOP) { - return false; - } - if (type->make_ptr()->higher_equal(TypePtr::NULL_PTR)) { - return false; - } - if (type->make_oopptr() && type->make_oopptr()->const_oop() != NULL) { - return false; - } - - if (ShenandoahOptimizeStableFinals) { - const TypeAryPtr* ary = type->isa_aryptr(); - if (ary && ary->is_stable() && allow_fromspace) { - return false; - } - } - - if (n->is_CheckCastPP() || n->is_ConstraintCast() || n->Opcode() == Op_ShenandoahEnqueueBarrier) { - return needs_barrier_impl(phase, orig, n->in(1), rb_mem, allow_fromspace, visited); - } - if (n->is_Parm()) { - return true; - } - if (n->is_Proj()) { - return needs_barrier_impl(phase, orig, n->in(0), rb_mem, allow_fromspace, visited); - } - - if (n->Opcode() == Op_ShenandoahWBMemProj) { - return needs_barrier_impl(phase, orig, n->in(ShenandoahWBMemProjNode::WriteBarrier), rb_mem, allow_fromspace, visited); - } - if (n->is_Phi()) { - bool need_barrier = false; - for (uint i = 1; i < n->req() && ! need_barrier; i++) { - Node* input = n->in(i); - if (input == NULL) { - need_barrier = true; // Phi not complete yet? - } else if (needs_barrier_impl(phase, orig, input, rb_mem, allow_fromspace, visited)) { - need_barrier = true; - } - } - return need_barrier; - } - if (n->is_CMove()) { - return needs_barrier_impl(phase, orig, n->in(CMoveNode::IfFalse), rb_mem, allow_fromspace, visited) || - needs_barrier_impl(phase, orig, n->in(CMoveNode::IfTrue ), rb_mem, allow_fromspace, visited); - } - if (n->Opcode() == Op_CreateEx) { - return true; - } - if (n->Opcode() == Op_ShenandoahWriteBarrier) { - return false; - } - if (n->Opcode() == Op_ShenandoahReadBarrier) { - if (rb_mem == n->in(Memory)) { - return false; - } else { - return true; - } - } - - if (n->Opcode() == Op_LoadP || - n->Opcode() == Op_LoadN || - n->Opcode() == Op_GetAndSetP || - n->Opcode() == Op_CompareAndExchangeP || - n->Opcode() == Op_GetAndSetN || - n->Opcode() == Op_CompareAndExchangeN) { - return true; - } - if (n->Opcode() == Op_DecodeN || - n->Opcode() == Op_EncodeP) { - return needs_barrier_impl(phase, orig, n->in(1), rb_mem, allow_fromspace, visited); - } - -#ifdef ASSERT - tty->print("need barrier on?: "); n->dump(); - ShouldNotReachHere(); -#endif - return true; -} - -bool ShenandoahReadBarrierNode::dominates_memory_rb_impl(PhaseGVN* phase, - Node* b1, - Node* b2, - Node* current, - bool linear) { - ResourceMark rm; - VectorSet visited(Thread::current()->resource_area()); - Node_Stack phis(0); - - for(int i = 0; i < 10; i++) { - if (current == NULL) { - return false; - } else if (visited.test_set(current->_idx) || current->is_top() || current == b1) { - current = NULL; - while (phis.is_nonempty() && current == NULL) { - uint idx = phis.index(); - Node* phi = phis.node(); - if (idx >= phi->req()) { - phis.pop(); - } else { - current = phi->in(idx); - phis.set_index(idx+1); - } - } - if (current == NULL) { - return true; - } - } else if (current == phase->C->immutable_memory()) { - return false; - } else if (current->isa_Phi()) { - if (!linear) { +bool ShenandoahBarrierC2Support::expand(Compile* C, PhaseIterGVN& igvn) { + ShenandoahBarrierSetC2State* state = ShenandoahBarrierSetC2::bsc2()->state(); + if ((state->enqueue_barriers_count() + + state->load_reference_barriers_count()) > 0) { + bool attempt_more_loopopts = ShenandoahLoopOptsAfterExpansion; + C->clear_major_progress(); + PhaseIdealLoop ideal_loop(igvn, LoopOptsShenandoahExpand); + if (C->failing()) return false; + PhaseIdealLoop::verify(igvn); + DEBUG_ONLY(verify_raw_mem(C->root());) + if (attempt_more_loopopts) { + C->set_major_progress(); + if (!C->optimize_loops(igvn, LoopOptsShenandoahPostExpand)) { return false; } - phis.push(current, 2); - current = current->in(1); - } else if (current->Opcode() == Op_ShenandoahWriteBarrier) { - const Type* in_type = current->bottom_type(); - const Type* this_type = b2->bottom_type(); - if (is_independent(in_type, this_type)) { - current = current->in(Memory); - } else { - return false; - } - } else if (current->Opcode() == Op_ShenandoahWBMemProj) { - current = current->in(ShenandoahWBMemProjNode::WriteBarrier); - } else if (current->is_Proj()) { - current = current->in(0); - } else if (current->is_Call()) { - return false; // TODO: Maybe improve by looking at the call's memory effects? - } else if (current->is_MemBar()) { - return false; // TODO: Do we need to stop at *any* membar? - } else if (current->is_MergeMem()) { - const TypePtr* adr_type = brooks_pointer_type(phase->type(b2)); - uint alias_idx = phase->C->get_alias_index(adr_type); - current = current->as_MergeMem()->memory_at(alias_idx); - } else { -#ifdef ASSERT - current->dump(); -#endif - ShouldNotReachHere(); - return false; - } - } - return false; -} - -bool ShenandoahReadBarrierNode::is_independent(Node* mem) { - if (mem->is_Phi() || mem->is_Proj() || mem->is_MergeMem()) { - return true; - } else if (mem->Opcode() == Op_ShenandoahWBMemProj) { - return true; - } else if (mem->Opcode() == Op_ShenandoahWriteBarrier) { - const Type* mem_type = mem->bottom_type(); - const Type* this_type = bottom_type(); - if (is_independent(mem_type, this_type)) { - return true; - } else { - return false; - } - } else if (mem->is_Call() || mem->is_MemBar()) { - return false; - } -#ifdef ASSERT - mem->dump(); -#endif - ShouldNotReachHere(); - return true; -} - -bool ShenandoahReadBarrierNode::dominates_memory_rb(PhaseGVN* phase, Node* b1, Node* b2, bool linear) { - return dominates_memory_rb_impl(phase, b1->in(Memory), b2, b2->in(Memory), linear); -} - -bool ShenandoahReadBarrierNode::is_independent(const Type* in_type, const Type* this_type) { - assert(in_type->isa_oopptr(), "expect oop ptr"); - assert(this_type->isa_oopptr(), "expect oop ptr"); - - ciKlass* in_kls = in_type->is_oopptr()->klass(); - ciKlass* this_kls = this_type->is_oopptr()->klass(); - if (in_kls != NULL && this_kls != NULL && - in_kls->is_loaded() && this_kls->is_loaded() && - (!in_kls->is_subclass_of(this_kls)) && - (!this_kls->is_subclass_of(in_kls))) { - return true; - } - return false; -} - -Node* ShenandoahReadBarrierNode::Ideal(PhaseGVN *phase, bool can_reshape) { - if (! can_reshape) { - return NULL; - } - - if (in(Memory) == phase->C->immutable_memory()) return NULL; - - // If memory input is a MergeMem, take the appropriate slice out of it. - Node* mem_in = in(Memory); - if (mem_in->isa_MergeMem()) { - const TypePtr* adr_type = brooks_pointer_type(bottom_type()); - uint alias_idx = phase->C->get_alias_index(adr_type); - mem_in = mem_in->as_MergeMem()->memory_at(alias_idx); - set_req(Memory, mem_in); - return this; - } - - Node* input = in(Memory); - if (input->Opcode() == Op_ShenandoahWBMemProj) { - ResourceMark rm; - VectorSet seen(Thread::current()->resource_area()); - Node* n = in(Memory); - while (n->Opcode() == Op_ShenandoahWBMemProj && - n->in(ShenandoahWBMemProjNode::WriteBarrier) != NULL && - n->in(ShenandoahWBMemProjNode::WriteBarrier)->Opcode() == Op_ShenandoahWriteBarrier && - n->in(ShenandoahWBMemProjNode::WriteBarrier)->in(Memory) != NULL) { - if (seen.test_set(n->_idx)) { - return NULL; // loop - } - n = n->in(ShenandoahWBMemProjNode::WriteBarrier)->in(Memory); - } - - Node* wb = input->in(ShenandoahWBMemProjNode::WriteBarrier); - const Type* in_type = phase->type(wb); - // is_top() test not sufficient here: we can come here after CCP - // in a dead branch of the graph that has not yet been removed. - if (in_type == Type::TOP) return NULL; // Dead path. - assert(wb->Opcode() == Op_ShenandoahWriteBarrier, "expect write barrier"); - if (is_independent(in_type, _type)) { - phase->igvn_rehash_node_delayed(wb); - set_req(Memory, wb->in(Memory)); - if (can_reshape && input->outcnt() == 0) { - phase->is_IterGVN()->_worklist.push(input); - } - return this; - } - } - return NULL; -} - -ShenandoahWriteBarrierNode::ShenandoahWriteBarrierNode(Compile* C, Node* ctrl, Node* mem, Node* obj) - : ShenandoahBarrierNode(ctrl, mem, obj, false) { - assert(UseShenandoahGC && ShenandoahWriteBarrier, "should be enabled"); - ShenandoahBarrierSetC2::bsc2()->state()->add_shenandoah_barrier(this); -} - -Node* ShenandoahWriteBarrierNode::Identity(PhaseGVN* phase) { - assert(in(0) != NULL, "should have control"); - PhaseIterGVN* igvn = phase->is_IterGVN(); - Node* mem_in = in(Memory); - Node* mem_proj = NULL; - - if (igvn != NULL) { - mem_proj = find_out_with(Op_ShenandoahWBMemProj); - if (mem_in == mem_proj) { - return this; - } - } - - Node* replacement = Identity_impl(phase); - if (igvn != NULL) { - if (replacement != NULL && replacement != this && mem_proj != NULL) { - igvn->replace_node(mem_proj, mem_in); - } - } - return replacement; -} - -Node* ShenandoahWriteBarrierNode::Ideal(PhaseGVN *phase, bool can_reshape) { - assert(in(0) != NULL, "should have control"); - if (!can_reshape) { - return NULL; - } - - Node* mem_in = in(Memory); - - if (mem_in->isa_MergeMem()) { - const TypePtr* adr_type = brooks_pointer_type(bottom_type()); - uint alias_idx = phase->C->get_alias_index(adr_type); - mem_in = mem_in->as_MergeMem()->memory_at(alias_idx); - set_req(Memory, mem_in); - return this; - } - - Node* val = in(ValueIn); - if (val->is_ShenandoahBarrier()) { - set_req(ValueIn, val->in(ValueIn)); - return this; - } - - return NULL; -} - -bool ShenandoahWriteBarrierNode::expand(Compile* C, PhaseIterGVN& igvn, int& loop_opts_cnt) { - if (UseShenandoahGC) { - if (ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_barriers_count() > 0 || (!ShenandoahWriteBarrier && ShenandoahStoreValEnqueueBarrier)) { - bool attempt_more_loopopts = ShenandoahLoopOptsAfterExpansion; C->clear_major_progress(); - PhaseIdealLoop ideal_loop(igvn, LoopOptsShenandoahExpand); - if (C->failing()) return false; - PhaseIdealLoop::verify(igvn); - DEBUG_ONLY(ShenandoahBarrierNode::verify_raw_mem(C->root());) - if (attempt_more_loopopts) { - C->set_major_progress(); - if (!C->optimize_loops(loop_opts_cnt, igvn, LoopOptsShenandoahPostExpand)) { - return false; - } - C->clear_major_progress(); - } } } return true; } -bool ShenandoahWriteBarrierNode::is_heap_state_test(Node* iff, int mask) { +bool ShenandoahBarrierC2Support::is_heap_state_test(Node* iff, int mask) { if (!UseShenandoahGC) { return false; } @@ -451,11 +95,11 @@ return is_gc_state_load(in1); } -bool ShenandoahWriteBarrierNode::is_heap_stable_test(Node* iff) { +bool ShenandoahBarrierC2Support::is_heap_stable_test(Node* iff) { return is_heap_state_test(iff, ShenandoahHeap::HAS_FORWARDED); } -bool ShenandoahWriteBarrierNode::is_gc_state_load(Node *n) { +bool ShenandoahBarrierC2Support::is_gc_state_load(Node *n) { if (!UseShenandoahGC) { return false; } @@ -477,7 +121,7 @@ return true; } -bool ShenandoahWriteBarrierNode::has_safepoint_between(Node* start, Node* stop, PhaseIdealLoop *phase) { +bool ShenandoahBarrierC2Support::has_safepoint_between(Node* start, Node* stop, PhaseIdealLoop *phase) { assert(phase->is_dominator(stop, start), "bad inputs"); ResourceMark rm; Unique_Node_List wq; @@ -501,7 +145,7 @@ return false; } -bool ShenandoahWriteBarrierNode::try_common_gc_state_load(Node *n, PhaseIdealLoop *phase) { +bool ShenandoahBarrierC2Support::try_common_gc_state_load(Node *n, PhaseIdealLoop *phase) { assert(is_gc_state_load(n), "inconsistent"); Node* addp = n->in(MemNode::Address); Node* dominator = NULL; @@ -526,193 +170,8 @@ return true; } -bool ShenandoahBarrierNode::dominates_memory_impl(PhaseGVN* phase, - Node* b1, - Node* b2, - Node* current, - bool linear) { - ResourceMark rm; - VectorSet visited(Thread::current()->resource_area()); - Node_Stack phis(0); - - for(int i = 0; i < 10; i++) { - if (current == NULL) { - return false; - } else if (visited.test_set(current->_idx) || current->is_top() || current == b1) { - current = NULL; - while (phis.is_nonempty() && current == NULL) { - uint idx = phis.index(); - Node* phi = phis.node(); - if (idx >= phi->req()) { - phis.pop(); - } else { - current = phi->in(idx); - phis.set_index(idx+1); - } - } - if (current == NULL) { - return true; - } - } else if (current == b2) { - return false; - } else if (current == phase->C->immutable_memory()) { - return false; - } else if (current->isa_Phi()) { - if (!linear) { - return false; - } - phis.push(current, 2); - current = current->in(1); - } else if (current->Opcode() == Op_ShenandoahWriteBarrier) { - current = current->in(Memory); - } else if (current->Opcode() == Op_ShenandoahWBMemProj) { - current = current->in(ShenandoahWBMemProjNode::WriteBarrier); - } else if (current->is_Proj()) { - current = current->in(0); - } else if (current->is_Call()) { - current = current->in(TypeFunc::Memory); - } else if (current->is_MemBar()) { - current = current->in(TypeFunc::Memory); - } else if (current->is_MergeMem()) { - const TypePtr* adr_type = brooks_pointer_type(phase->type(b2)); - uint alias_idx = phase->C->get_alias_index(adr_type); - current = current->as_MergeMem()->memory_at(alias_idx); - } else { #ifdef ASSERT - current->dump(); -#endif - ShouldNotReachHere(); - return false; - } - } - return false; -} - -/** - * Determines if b1 dominates b2 through memory inputs. It returns true if: - * - b1 can be reached by following each branch in b2's memory input (through phis, etc) - * - or we get back to b2 (i.e. through a loop) without seeing b1 - * In all other cases, (in particular, if we reach immutable_memory without having seen b1) - * we return false. - */ -bool ShenandoahBarrierNode::dominates_memory(PhaseGVN* phase, Node* b1, Node* b2, bool linear) { - return dominates_memory_impl(phase, b1, b2, b2->in(Memory), linear); -} - -Node* ShenandoahBarrierNode::Identity_impl(PhaseGVN* phase) { - Node* n = in(ValueIn); - - Node* rb_mem = Opcode() == Op_ShenandoahReadBarrier ? in(Memory) : NULL; - if (! needs_barrier(phase, this, n, rb_mem, _allow_fromspace)) { - return n; - } - - // Try to find a write barrier sibling with identical inputs that we can fold into. - for (DUIterator i = n->outs(); n->has_out(i); i++) { - Node* sibling = n->out(i); - if (sibling == this) { - continue; - } - if (sibling->Opcode() != Op_ShenandoahWriteBarrier) { - continue; - } - - assert(sibling->in(ValueIn) == in(ValueIn), "sanity"); - assert(sibling->Opcode() == Op_ShenandoahWriteBarrier, "sanity"); - - if (dominates_memory(phase, sibling, this, phase->is_IterGVN() == NULL)) { - return sibling; - } - } - return this; -} - -#ifndef PRODUCT -void ShenandoahBarrierNode::dump_spec(outputStream *st) const { - const TypePtr* adr = adr_type(); - if (adr == NULL) { - return; - } - st->print(" @"); - adr->dump_on(st); - st->print(" ("); - Compile::current()->alias_type(adr)->adr_type()->dump_on(st); - st->print(") "); -} -#endif - -Node* ShenandoahReadBarrierNode::Identity(PhaseGVN* phase) { - Node* id = Identity_impl(phase); - - if (id == this && phase->is_IterGVN()) { - Node* n = in(ValueIn); - // No success in super call. Try to combine identical read barriers. - for (DUIterator i = n->outs(); n->has_out(i); i++) { - Node* sibling = n->out(i); - if (sibling == this || sibling->Opcode() != Op_ShenandoahReadBarrier) { - continue; - } - assert(sibling->in(ValueIn) == in(ValueIn), "sanity"); - if (phase->is_IterGVN()->hash_find(sibling) && - sibling->bottom_type() == bottom_type() && - sibling->in(Control) == in(Control) && - dominates_memory_rb(phase, sibling, this, phase->is_IterGVN() == NULL)) { - return sibling; - } - } - } - return id; -} - -const Type* ShenandoahBarrierNode::Value(PhaseGVN* phase) const { - // Either input is TOP ==> the result is TOP - const Type *t1 = phase->type(in(Memory)); - if (t1 == Type::TOP) return Type::TOP; - const Type *t2 = phase->type(in(ValueIn)); - if( t2 == Type::TOP ) return Type::TOP; - - if (t2 == TypePtr::NULL_PTR) { - return _type; - } - - const Type* type = t2->is_oopptr()->cast_to_nonconst(); - return type; -} - -uint ShenandoahBarrierNode::hash() const { - return TypeNode::hash() + _allow_fromspace; -} - -uint ShenandoahBarrierNode::cmp(const Node& n) const { - return _allow_fromspace == ((ShenandoahBarrierNode&) n)._allow_fromspace - && TypeNode::cmp(n); -} - -uint ShenandoahBarrierNode::size_of() const { - return sizeof(*this); -} - -Node* ShenandoahWBMemProjNode::Identity(PhaseGVN* phase) { - Node* wb = in(WriteBarrier); - if (wb->is_top()) return phase->C->top(); // Dead path. - - assert(wb->Opcode() == Op_ShenandoahWriteBarrier, "expect write barrier"); - PhaseIterGVN* igvn = phase->is_IterGVN(); - // We can't do the below unless the graph is fully constructed. - if (igvn == NULL) { - return this; - } - - // If the mem projection has no barrier users, it's not needed anymore. - if (wb->outcnt() == 1) { - return wb->in(ShenandoahBarrierNode::Memory); - } - - return this; -} - -#ifdef ASSERT -bool ShenandoahBarrierNode::verify_helper(Node* in, Node_Stack& phis, VectorSet& visited, verify_type t, bool trace, Unique_Node_List& barriers_used) { +bool ShenandoahBarrierC2Support::verify_helper(Node* in, Node_Stack& phis, VectorSet& visited, verify_type t, bool trace, Unique_Node_List& barriers_used) { assert(phis.size() == 0, ""); while (true) { @@ -733,12 +192,24 @@ in = in->in(AddPNode::Address); continue; } else if (in->is_Con()) { - if (trace) {tty->print("Found constant"); in->dump();} - } else if (in->is_ShenandoahBarrier()) { + if (trace) { + tty->print("Found constant"); + in->dump(); + } + } else if (in->Opcode() == Op_Parm) { + if (trace) { + tty->print("Found argument"); + } + } else if (in->Opcode() == Op_CreateEx) { + if (trace) { + tty->print("Found create-exception"); + } + } else if (in->Opcode() == Op_LoadP && in->adr_type() == TypeRawPtr::BOTTOM) { + if (trace) { + tty->print("Found raw LoadP (OSR argument?)"); + } + } else if (in->Opcode() == Op_ShenandoahLoadReferenceBarrier) { if (t == ShenandoahOopStore) { - if (in->Opcode() != Op_ShenandoahWriteBarrier) { - return false; - } uint i = 0; for (; i < phis.size(); i++) { Node* n = phis.node_at(i); @@ -749,21 +220,27 @@ if (i == phis.size()) { return false; } - } else if (t == ShenandoahStore && in->Opcode() != Op_ShenandoahWriteBarrier) { - return false; } barriers_used.push(in); if (trace) {tty->print("Found barrier"); in->dump();} } else if (in->Opcode() == Op_ShenandoahEnqueueBarrier) { if (t != ShenandoahOopStore) { - return false; + in = in->in(1); + continue; } if (trace) {tty->print("Found enqueue barrier"); in->dump();} phis.push(in, in->req()); in = in->in(1); continue; } else if (in->is_Proj() && in->in(0)->is_Allocate()) { - if (trace) {tty->print("Found alloc"); in->in(0)->dump();} + if (trace) { + tty->print("Found alloc"); + in->in(0)->dump(); + } + } else if (in->is_Proj() && (in->in(0)->Opcode() == Op_CallStaticJava || in->in(0)->Opcode() == Op_CallDynamicJava)) { + if (trace) { + tty->print("Found Java call"); + } } else if (in->is_Phi()) { if (!visited.test_set(in->_idx)) { if (trace) {tty->print("Pushed phi:"); in->dump();} @@ -809,7 +286,7 @@ return true; } -void ShenandoahBarrierNode::report_verify_failure(const char *msg, Node *n1, Node *n2) { +void ShenandoahBarrierC2Support::report_verify_failure(const char* msg, Node* n1, Node* n2) { if (n1 != NULL) { n1->dump(+10); } @@ -819,7 +296,7 @@ fatal("%s", msg); } -void ShenandoahBarrierNode::verify(RootNode* root) { +void ShenandoahBarrierC2Support::verify(RootNode* root) { ResourceMark rm; Unique_Node_List wq; GrowableArray barriers; @@ -871,7 +348,7 @@ } } - if (verify && !ShenandoahBarrierNode::verify_helper(n->in(MemNode::Address), phis, visited, ShenandoahLoad, trace, barriers_used)) { + if (verify && !verify_helper(n->in(MemNode::Address), phis, visited, ShenandoahLoad, trace, barriers_used)) { report_verify_failure("Shenandoah verification: Load should have barriers", n); } } @@ -899,11 +376,11 @@ } } - if (verify && !ShenandoahBarrierNode::verify_helper(n->in(MemNode::ValueIn), phis, visited, ShenandoahStoreValEnqueueBarrier ? ShenandoahOopStore : ShenandoahValue, trace, barriers_used)) { + if (verify && !verify_helper(n->in(MemNode::ValueIn), phis, visited, ShenandoahStoreValEnqueueBarrier ? ShenandoahOopStore : ShenandoahValue, trace, barriers_used)) { report_verify_failure("Shenandoah verification: Store should have barriers", n); } } - if (!ShenandoahBarrierNode::verify_helper(n->in(MemNode::Address), phis, visited, ShenandoahStore, trace, barriers_used)) { + if (!verify_helper(n->in(MemNode::Address), phis, visited, ShenandoahStore, trace, barriers_used)) { report_verify_failure("Shenandoah verification: Store (address) should have barriers", n); } } else if (n->Opcode() == Op_CmpP) { @@ -926,26 +403,26 @@ } else { assert(in2->bottom_type()->isa_oopptr(), ""); - if (!ShenandoahBarrierNode::verify_helper(in1, phis, visited, ShenandoahStore, trace, barriers_used) || - !ShenandoahBarrierNode::verify_helper(in2, phis, visited, ShenandoahStore, trace, barriers_used)) { + if (!verify_helper(in1, phis, visited, ShenandoahStore, trace, barriers_used) || + !verify_helper(in2, phis, visited, ShenandoahStore, trace, barriers_used)) { report_verify_failure("Shenandoah verification: Cmp should have barriers", n); } } if (verify_no_useless_barrier && mark_inputs && - (!ShenandoahBarrierNode::verify_helper(in1, phis, visited, ShenandoahValue, trace, barriers_used) || - !ShenandoahBarrierNode::verify_helper(in2, phis, visited, ShenandoahValue, trace, barriers_used))) { + (!verify_helper(in1, phis, visited, ShenandoahValue, trace, barriers_used) || + !verify_helper(in2, phis, visited, ShenandoahValue, trace, barriers_used))) { phis.clear(); visited.Reset(); } } } else if (n->is_LoadStore()) { if (n->in(MemNode::ValueIn)->bottom_type()->make_ptr() && - !ShenandoahBarrierNode::verify_helper(n->in(MemNode::ValueIn), phis, visited, ShenandoahStoreValEnqueueBarrier ? ShenandoahOopStore : ShenandoahValue, trace, barriers_used)) { + !verify_helper(n->in(MemNode::ValueIn), phis, visited, ShenandoahStoreValEnqueueBarrier ? ShenandoahOopStore : ShenandoahValue, trace, barriers_used)) { report_verify_failure("Shenandoah verification: LoadStore (value) should have barriers", n); } - if (n->in(MemNode::Address)->bottom_type()->make_oopptr() && !ShenandoahBarrierNode::verify_helper(n->in(MemNode::Address), phis, visited, ShenandoahStore, trace, barriers_used)) { + if (n->in(MemNode::Address)->bottom_type()->make_oopptr() && !verify_helper(n->in(MemNode::Address), phis, visited, ShenandoahStore, trace, barriers_used)) { report_verify_failure("Shenandoah verification: LoadStore (address) should have barriers", n); } } else if (n->Opcode() == Op_CallLeafNoFP || n->Opcode() == Op_CallLeaf) { @@ -1041,13 +518,13 @@ } } } - if (!ShenandoahBarrierNode::verify_helper(n->in(TypeFunc::Parms), phis, visited, ShenandoahLoad, trace, barriers_used) || - !ShenandoahBarrierNode::verify_helper(dest, phis, visited, ShenandoahStore, trace, barriers_used)) { + if (!verify_helper(n->in(TypeFunc::Parms), phis, visited, ShenandoahLoad, trace, barriers_used) || + !verify_helper(dest, phis, visited, ShenandoahStore, trace, barriers_used)) { report_verify_failure("Shenandoah verification: ArrayCopy should have barriers", n); } } else if (strlen(call->_name) > 5 && !strcmp(call->_name + strlen(call->_name) - 5, "_fill")) { - if (!ShenandoahBarrierNode::verify_helper(n->in(TypeFunc::Parms), phis, visited, ShenandoahStore, trace, barriers_used)) { + if (!verify_helper(n->in(TypeFunc::Parms), phis, visited, ShenandoahStore, trace, barriers_used)) { report_verify_failure("Shenandoah verification: _fill should have barriers", n); } } else if (!strcmp(call->_name, "shenandoah_wb_pre")) { @@ -1067,7 +544,7 @@ if (pos == -1) { break; } - if (!ShenandoahBarrierNode::verify_helper(call->in(pos), phis, visited, calls[i].args[j].t, trace, barriers_used)) { + if (!verify_helper(call->in(pos), phis, visited, calls[i].args[j].t, trace, barriers_used)) { report_verify_failure("Shenandoah verification: intrinsic calls should have barriers", n); } } @@ -1090,15 +567,8 @@ } } } - } else if (n->is_ShenandoahBarrier()) { - assert(!barriers.contains(n), ""); - assert(n->Opcode() != Op_ShenandoahWriteBarrier || n->find_out_with(Op_ShenandoahWBMemProj) != NULL, "bad shenandoah write barrier"); - assert(n->Opcode() != Op_ShenandoahWriteBarrier || n->outcnt() > 1, "bad shenandoah write barrier"); - barriers.push(n); - } else if (n->Opcode() == Op_ShenandoahEnqueueBarrier) { + } else if (n->Opcode() == Op_ShenandoahEnqueueBarrier || n->Opcode() == Op_ShenandoahLoadReferenceBarrier) { // skip - } else if (n->Opcode() == Op_ShenandoahWBMemProj) { - assert(n->in(0) == NULL && n->in(ShenandoahWBMemProjNode::WriteBarrier)->Opcode() == Op_ShenandoahWriteBarrier, "strange ShenandoahWBMemProj"); } else if (n->is_AddP() || n->is_Phi() || n->is_ConstraintCast() @@ -1165,7 +635,7 @@ if (pos == -1) { break; } - if (!ShenandoahBarrierNode::verify_helper(n->in(pos), phis, visited, others[i].inputs[j].t, trace, barriers_used)) { + if (!verify_helper(n->in(pos), phis, visited, others[i].inputs[j].t, trace, barriers_used)) { report_verify_failure("Shenandoah verification: intrinsic calls should have barriers", n); } } @@ -1193,7 +663,7 @@ SafePointNode* sfpt = n->as_SafePoint(); if (verify_no_useless_barrier && sfpt->jvms() != NULL) { for (uint i = sfpt->jvms()->scloff(); i < sfpt->jvms()->endoff(); i++) { - if (!ShenandoahBarrierNode::verify_helper(sfpt->in(i), phis, visited, ShenandoahLoad, trace, barriers_used)) { + if (!verify_helper(sfpt->in(i), phis, visited, ShenandoahLoad, trace, barriers_used)) { phis.clear(); visited.Reset(); } @@ -1227,9 +697,8 @@ n->Opcode() == Op_SCMemProj || n->Opcode() == Op_EncodeP || n->Opcode() == Op_DecodeN || - n->Opcode() == Op_ShenandoahWriteBarrier || - n->Opcode() == Op_ShenandoahWBMemProj || - n->Opcode() == Op_ShenandoahEnqueueBarrier)) { + n->Opcode() == Op_ShenandoahEnqueueBarrier || + n->Opcode() == Op_ShenandoahLoadReferenceBarrier)) { if (m->bottom_type()->make_oopptr() && m->bottom_type()->make_oopptr()->meet(TypePtr::NULL_PTR) == m->bottom_type()) { report_verify_failure("Shenandoah verification: null input", n, m); } @@ -1251,7 +720,7 @@ } #endif -bool ShenandoahBarrierNode::is_dominator_same_ctrl(Node*c, Node* d, Node* n, PhaseIdealLoop* phase) { +bool ShenandoahBarrierC2Support::is_dominator_same_ctrl(Node* c, Node* d, Node* n, PhaseIdealLoop* phase) { // That both nodes have the same control is not sufficient to prove // domination, verify that there's no path from d to n ResourceMark rm; @@ -1275,7 +744,7 @@ return true; } -bool ShenandoahBarrierNode::is_dominator(Node *d_c, Node *n_c, Node* d, Node* n, PhaseIdealLoop* phase) { +bool ShenandoahBarrierC2Support::is_dominator(Node* d_c, Node* n_c, Node* d, Node* n, PhaseIdealLoop* phase) { if (d_c != n_c) { return phase->is_dominator(d_c, n_c); } @@ -1290,15 +759,11 @@ res = mem->in(TypeFunc::Memory); } else if (mem->is_Phi()) { res = mem->in(1); - } else if (mem->is_ShenandoahBarrier()) { - res = mem->in(ShenandoahBarrierNode::Memory); } else if (mem->is_MergeMem()) { res = mem->as_MergeMem()->memory_at(alias); } else if (mem->is_Store() || mem->is_LoadStore() || mem->is_ClearArray()) { assert(alias = Compile::AliasIdxRaw, "following raw memory can't lead to a barrier"); res = mem->in(MemNode::Memory); - } else if (mem->Opcode() == Op_ShenandoahWBMemProj) { - res = mem->in(ShenandoahWBMemProjNode::WriteBarrier); } else { #ifdef ASSERT mem->dump(); @@ -1308,7 +773,7 @@ return res; } -Node* ShenandoahBarrierNode::no_branches(Node* c, Node* dom, bool allow_one_proj, PhaseIdealLoop* phase) { +Node* ShenandoahBarrierC2Support::no_branches(Node* c, Node* dom, bool allow_one_proj, PhaseIdealLoop* phase) { Node* iffproj = NULL; while (c != dom) { Node* next = phase->idom(c); @@ -1373,241 +838,7 @@ return iffproj; } -#ifdef ASSERT -void ShenandoahWriteBarrierNode::memory_dominates_all_paths_helper(Node* c, Node* rep_ctrl, Unique_Node_List& controls, PhaseIdealLoop* phase) { - const bool trace = false; - if (trace) { tty->print("X control is"); c->dump(); } - - uint start = controls.size(); - controls.push(c); - for (uint i = start; i < controls.size(); i++) { - Node *n = controls.at(i); - - if (trace) { tty->print("X from"); n->dump(); } - - if (n == rep_ctrl) { - continue; - } - - if (n->is_Proj()) { - Node* n_dom = n->in(0); - IdealLoopTree* n_dom_loop = phase->get_loop(n_dom); - if (n->is_IfProj() && n_dom->outcnt() == 2) { - n_dom_loop = phase->get_loop(n_dom->as_If()->proj_out(n->as_Proj()->_con == 0 ? 1 : 0)); - } - if (n_dom_loop != phase->ltree_root()) { - Node* tail = n_dom_loop->tail(); - if (tail->is_Region()) { - for (uint j = 1; j < tail->req(); j++) { - if (phase->is_dominator(n_dom, tail->in(j)) && !phase->is_dominator(n, tail->in(j))) { - assert(phase->is_dominator(rep_ctrl, tail->in(j)), "why are we here?"); - // entering loop from below, mark backedge - if (trace) { tty->print("X pushing backedge"); tail->in(j)->dump(); } - controls.push(tail->in(j)); - //assert(n->in(0) == n_dom, "strange flow control"); - } - } - } else if (phase->get_loop(n) != n_dom_loop && phase->is_dominator(n_dom, tail)) { - // entering loop from below, mark backedge - if (trace) { tty->print("X pushing backedge"); tail->dump(); } - controls.push(tail); - //assert(n->in(0) == n_dom, "strange flow control"); - } - } - } - - if (n->is_Loop()) { - Node* c = n->in(LoopNode::EntryControl); - if (trace) { tty->print("X pushing"); c->dump(); } - controls.push(c); - } else if (n->is_Region()) { - for (uint i = 1; i < n->req(); i++) { - Node* c = n->in(i); - if (trace) { tty->print("X pushing"); c->dump(); } - controls.push(c); - } - } else { - Node* c = n->in(0); - if (trace) { tty->print("X pushing"); c->dump(); } - controls.push(c); - } - } -} - -bool ShenandoahWriteBarrierNode::memory_dominates_all_paths(Node* mem, Node* rep_ctrl, int alias, PhaseIdealLoop* phase) { - const bool trace = false; - if (trace) { - tty->print("XXX mem is"); mem->dump(); - tty->print("XXX rep ctrl is"); rep_ctrl->dump(); - tty->print_cr("XXX alias is %d", alias); - } - ResourceMark rm; - Unique_Node_List wq; - Unique_Node_List controls; - wq.push(mem); - for (uint next = 0; next < wq.size(); next++) { - Node *nn = wq.at(next); - if (trace) { tty->print("XX from mem"); nn->dump(); } - assert(nn->bottom_type() == Type::MEMORY, "memory only"); - - if (nn->is_Phi()) { - Node* r = nn->in(0); - for (DUIterator_Fast jmax, j = r->fast_outs(jmax); j < jmax; j++) { - Node* u = r->fast_out(j); - if (u->is_Phi() && u->bottom_type() == Type::MEMORY && u != nn && - (u->adr_type() == TypePtr::BOTTOM || phase->C->get_alias_index(u->adr_type()) == alias)) { - if (trace) { tty->print("XX Next mem (other phi)"); u->dump(); } - wq.push(u); - } - } - } - - for (DUIterator_Fast imax, i = nn->fast_outs(imax); i < imax; i++) { - Node* use = nn->fast_out(i); - - if (trace) { tty->print("XX use %p", use->adr_type()); use->dump(); } - if (use->is_CFG() && use->in(TypeFunc::Memory) == nn) { - Node* c = use->in(0); - if (phase->is_dominator(rep_ctrl, c)) { - memory_dominates_all_paths_helper(c, rep_ctrl, controls, phase); - } else if (use->is_CallStaticJava() && use->as_CallStaticJava()->uncommon_trap_request() != 0 && c->is_Region()) { - Node* region = c; - if (trace) { tty->print("XX unc region"); region->dump(); } - for (uint j = 1; j < region->req(); j++) { - if (phase->is_dominator(rep_ctrl, region->in(j))) { - if (trace) { tty->print("XX unc follows"); region->in(j)->dump(); } - memory_dominates_all_paths_helper(region->in(j), rep_ctrl, controls, phase); - } - } - } - //continue; - } else if (use->is_Phi()) { - assert(use->bottom_type() == Type::MEMORY, "bad phi"); - if ((use->adr_type() == TypePtr::BOTTOM) || - phase->C->get_alias_index(use->adr_type()) == alias) { - for (uint j = 1; j < use->req(); j++) { - if (use->in(j) == nn) { - Node* c = use->in(0)->in(j); - if (phase->is_dominator(rep_ctrl, c)) { - memory_dominates_all_paths_helper(c, rep_ctrl, controls, phase); - } - } - } - } - // continue; - } - - if (use->is_MergeMem()) { - if (use->as_MergeMem()->memory_at(alias) == nn) { - if (trace) { tty->print("XX Next mem"); use->dump(); } - // follow the memory edges - wq.push(use); - } - } else if (use->is_Phi()) { - assert(use->bottom_type() == Type::MEMORY, "bad phi"); - if ((use->adr_type() == TypePtr::BOTTOM) || - phase->C->get_alias_index(use->adr_type()) == alias) { - if (trace) { tty->print("XX Next mem"); use->dump(); } - // follow the memory edges - wq.push(use); - } - } else if (use->bottom_type() == Type::MEMORY && - (use->adr_type() == TypePtr::BOTTOM || phase->C->get_alias_index(use->adr_type()) == alias)) { - if (trace) { tty->print("XX Next mem"); use->dump(); } - // follow the memory edges - wq.push(use); - } else if ((use->is_SafePoint() || use->is_MemBar()) && - (use->adr_type() == TypePtr::BOTTOM || phase->C->get_alias_index(use->adr_type()) == alias)) { - for (DUIterator_Fast jmax, j = use->fast_outs(jmax); j < jmax; j++) { - Node* u = use->fast_out(j); - if (u->bottom_type() == Type::MEMORY) { - if (trace) { tty->print("XX Next mem"); u->dump(); } - // follow the memory edges - wq.push(u); - } - } - } else if (use->Opcode() == Op_ShenandoahWriteBarrier && phase->C->get_alias_index(use->adr_type()) == alias) { - Node* m = use->find_out_with(Op_ShenandoahWBMemProj); - if (m != NULL) { - if (trace) { tty->print("XX Next mem"); m->dump(); } - // follow the memory edges - wq.push(m); - } - } - } - } - - if (controls.size() == 0) { - return false; - } - - for (uint i = 0; i < controls.size(); i++) { - Node *n = controls.at(i); - - if (trace) { tty->print("X checking"); n->dump(); } - - if (n->unique_ctrl_out() != NULL) { - continue; - } - - if (n->Opcode() == Op_NeverBranch) { - Node* taken = n->as_Multi()->proj_out(0); - if (!controls.member(taken)) { - if (trace) { tty->print("X not seen"); taken->dump(); } - return false; - } - continue; - } - - for (DUIterator_Fast jmax, j = n->fast_outs(jmax); j < jmax; j++) { - Node* u = n->fast_out(j); - - if (u->is_CFG()) { - if (!controls.member(u)) { - if (u->is_Proj() && u->as_Proj()->is_uncommon_trap_proj(Deoptimization::Reason_none)) { - if (trace) { tty->print("X not seen but unc"); u->dump(); } - } else { - Node* c = u; - do { - c = c->unique_ctrl_out(); - } while (c != NULL && c->is_Region()); - if (c != NULL && c->Opcode() == Op_Halt) { - if (trace) { tty->print("X not seen but halt"); c->dump(); } - } else { - if (trace) { tty->print("X not seen"); u->dump(); } - return false; - } - } - } else { - if (trace) { tty->print("X seen"); u->dump(); } - } - } - } - } - return true; -} -#endif - -Node* ShenandoahBarrierNode::dom_mem(Node* mem, Node*& mem_ctrl, Node* n, Node* rep_ctrl, int alias, PhaseIdealLoop* phase) { - ResourceMark rm; - VectorSet wq(Thread::current()->resource_area()); - wq.set(mem->_idx); - mem_ctrl = phase->get_ctrl(mem); - while (!is_dominator(mem_ctrl, rep_ctrl, mem, n, phase)) { - mem = next_mem(mem, alias); - if (wq.test_set(mem->_idx)) { - return NULL; // hit an unexpected loop - } - mem_ctrl = phase->ctrl_or_self(mem); - } - if (mem->is_MergeMem()) { - mem = mem->as_MergeMem()->memory_at(alias); - mem_ctrl = phase->ctrl_or_self(mem); - } - return mem; -} - -Node* ShenandoahBarrierNode::dom_mem(Node* mem, Node* ctrl, int alias, Node*& mem_ctrl, PhaseIdealLoop* phase) { +Node* ShenandoahBarrierC2Support::dom_mem(Node* mem, Node* ctrl, int alias, Node*& mem_ctrl, PhaseIdealLoop* phase) { ResourceMark rm; VectorSet wq(Thread::current()->resource_area()); wq.set(mem->_idx); @@ -1626,586 +857,7 @@ return mem; } -static void disconnect_barrier_mem(Node* wb, PhaseIterGVN& igvn) { - Node* mem_in = wb->in(ShenandoahBarrierNode::Memory); - Node* proj = wb->find_out_with(Op_ShenandoahWBMemProj); - - for (DUIterator_Last imin, i = proj->last_outs(imin); i >= imin; ) { - Node* u = proj->last_out(i); - igvn.rehash_node_delayed(u); - int nb = u->replace_edge(proj, mem_in); - assert(nb > 0, "no replacement?"); - i -= nb; - } -} - -Node* ShenandoahWriteBarrierNode::move_above_predicates(LoopNode* cl, Node* val_ctrl, PhaseIdealLoop* phase) { - Node* entry = cl->skip_strip_mined(-1)->in(LoopNode::EntryControl); - Node* above_pred = phase->skip_all_loop_predicates(entry); - Node* ctrl = entry; - while (ctrl != above_pred) { - Node* next = ctrl->in(0); - if (!phase->is_dominator(val_ctrl, next)) { - break; - } - ctrl = next; - } - return ctrl; -} - -static MemoryGraphFixer* find_fixer(GrowableArray& memory_graph_fixers, int alias) { - for (int i = 0; i < memory_graph_fixers.length(); i++) { - if (memory_graph_fixers.at(i)->alias() == alias) { - return memory_graph_fixers.at(i); - } - } - return NULL; -} - -static MemoryGraphFixer* create_fixer(GrowableArray& memory_graph_fixers, int alias, PhaseIdealLoop* phase, bool include_lsm) { - assert(find_fixer(memory_graph_fixers, alias) == NULL, "none should exist yet"); - MemoryGraphFixer* fixer = new MemoryGraphFixer(alias, include_lsm, phase); - memory_graph_fixers.push(fixer); - return fixer; -} - -void ShenandoahWriteBarrierNode::try_move_before_loop_helper(LoopNode* cl, Node* val_ctrl, GrowableArray& memory_graph_fixers, PhaseIdealLoop* phase, bool include_lsm, Unique_Node_List& uses) { - assert(cl->is_Loop(), "bad control"); - Node* ctrl = move_above_predicates(cl, val_ctrl, phase); - Node* mem_ctrl = NULL; - int alias = phase->C->get_alias_index(adr_type()); - - MemoryGraphFixer* fixer = find_fixer(memory_graph_fixers, alias); - if (fixer == NULL) { - fixer = create_fixer(memory_graph_fixers, alias, phase, include_lsm); - } - - Node* proj = find_out_with(Op_ShenandoahWBMemProj); - - fixer->remove(proj); - Node* mem = fixer->find_mem(ctrl, NULL); - - assert(!ShenandoahVerifyOptoBarriers || memory_dominates_all_paths(mem, ctrl, alias, phase), "can't fix the memory graph"); - - phase->set_ctrl_and_loop(this, ctrl); - phase->igvn().replace_input_of(this, Control, ctrl); - - disconnect_barrier_mem(this, phase->igvn()); - - phase->igvn().replace_input_of(this, Memory, mem); - phase->set_ctrl_and_loop(proj, ctrl); - - fixer->fix_mem(ctrl, ctrl, mem, mem, proj, uses); - assert(proj->outcnt() > 0, "disconnected write barrier"); -} - -LoopNode* ShenandoahWriteBarrierNode::try_move_before_pre_loop(Node* c, Node* val_ctrl, PhaseIdealLoop* phase) { - // A write barrier between a pre and main loop can get in the way of - // vectorization. Move it above the pre loop if possible - CountedLoopNode* cl = NULL; - if (c->is_IfFalse() && - c->in(0)->is_CountedLoopEnd()) { - cl = c->in(0)->as_CountedLoopEnd()->loopnode(); - } else if (c->is_IfProj() && - c->in(0)->is_If() && - c->in(0)->in(0)->is_IfFalse() && - c->in(0)->in(0)->in(0)->is_CountedLoopEnd()) { - cl = c->in(0)->in(0)->in(0)->as_CountedLoopEnd()->loopnode(); - } - if (cl != NULL && - cl->is_pre_loop() && - val_ctrl != cl && - phase->is_dominator(val_ctrl, cl)) { - return cl; - } - return NULL; -} - -void ShenandoahWriteBarrierNode::try_move_before_loop(GrowableArray& memory_graph_fixers, PhaseIdealLoop* phase, bool include_lsm, Unique_Node_List& uses) { - Node *n_ctrl = phase->get_ctrl(this); - IdealLoopTree *n_loop = phase->get_loop(n_ctrl); - Node* val = in(ValueIn); - Node* val_ctrl = phase->get_ctrl(val); - if (n_loop != phase->ltree_root() && !n_loop->_irreducible) { - IdealLoopTree *val_loop = phase->get_loop(val_ctrl); - Node* mem = in(Memory); - IdealLoopTree *mem_loop = phase->get_loop(phase->get_ctrl(mem)); - if (!n_loop->is_member(val_loop) && - n_loop->is_member(mem_loop)) { - Node* n_loop_head = n_loop->_head; - - if (n_loop_head->is_Loop()) { - LoopNode* loop = n_loop_head->as_Loop(); - if (n_loop_head->is_CountedLoop() && n_loop_head->as_CountedLoop()->is_main_loop()) { - LoopNode* res = try_move_before_pre_loop(n_loop_head->in(LoopNode::EntryControl), val_ctrl, phase); - if (res != NULL) { - loop = res; - } - } - - try_move_before_loop_helper(loop, val_ctrl, memory_graph_fixers, phase, include_lsm, uses); - } - } - } - LoopNode* ctrl = try_move_before_pre_loop(in(0), val_ctrl, phase); - if (ctrl != NULL) { - try_move_before_loop_helper(ctrl, val_ctrl, memory_graph_fixers, phase, include_lsm, uses); - } -} - -Node* ShenandoahWriteBarrierNode::would_subsume(ShenandoahBarrierNode* other, PhaseIdealLoop* phase) { - Node* val = in(ValueIn); - Node* val_ctrl = phase->get_ctrl(val); - Node* other_mem = other->in(Memory); - Node* other_ctrl = phase->get_ctrl(other); - Node* this_ctrl = phase->get_ctrl(this); - IdealLoopTree* this_loop = phase->get_loop(this_ctrl); - IdealLoopTree* other_loop = phase->get_loop(other_ctrl); - - Node* ctrl = phase->dom_lca(other_ctrl, this_ctrl); - - if (ctrl->is_Proj() && - ctrl->in(0)->is_Call() && - ctrl->unique_ctrl_out() != NULL && - ctrl->unique_ctrl_out()->Opcode() == Op_Catch && - !phase->is_dominator(val_ctrl, ctrl->in(0)->in(0))) { - return NULL; - } - - IdealLoopTree* loop = phase->get_loop(ctrl); - - // We don't want to move a write barrier in a loop - // If the LCA is in a inner loop, try a control out of loop if possible - while (!loop->is_member(this_loop) && (other->Opcode() != Op_ShenandoahWriteBarrier || !loop->is_member(other_loop))) { - ctrl = phase->idom(ctrl); - if (ctrl->is_MultiBranch()) { - ctrl = ctrl->in(0); - } - if (ctrl != val_ctrl && phase->is_dominator(ctrl, val_ctrl)) { - return NULL; - } - loop = phase->get_loop(ctrl); - } - - if (ShenandoahDontIncreaseWBFreq) { - Node* this_iffproj = no_branches(this_ctrl, ctrl, true, phase); - if (other->Opcode() == Op_ShenandoahWriteBarrier) { - Node* other_iffproj = no_branches(other_ctrl, ctrl, true, phase); - if (other_iffproj == NULL || this_iffproj == NULL) { - return ctrl; - } else if (other_iffproj != NodeSentinel && this_iffproj != NodeSentinel && - other_iffproj->in(0) == this_iffproj->in(0)) { - return ctrl; - } - } else if (this_iffproj == NULL) { - return ctrl; - } - return NULL; - } - - return ctrl; -} - -void ShenandoahWriteBarrierNode::optimize_before_expansion(PhaseIdealLoop* phase, GrowableArray memory_graph_fixers, bool include_lsm) { - bool progress = false; - Unique_Node_List uses; - do { - progress = false; - for (int i = 0; i < ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_barriers_count(); i++) { - ShenandoahWriteBarrierNode* wb = ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_barrier(i); - - wb->try_move_before_loop(memory_graph_fixers, phase, include_lsm, uses); - - Node* val = wb->in(ValueIn); - - for (DUIterator_Fast jmax, j = val->fast_outs(jmax); j < jmax; j++) { - Node* u = val->fast_out(j); - if (u != wb && u->is_ShenandoahBarrier()) { - Node* rep_ctrl = wb->would_subsume(u->as_ShenandoahBarrier(), phase); - - if (rep_ctrl != NULL) { - Node* other = u; - Node* val_ctrl = phase->get_ctrl(val); - if (rep_ctrl->is_Proj() && - rep_ctrl->in(0)->is_Call() && - rep_ctrl->unique_ctrl_out() != NULL && - rep_ctrl->unique_ctrl_out()->Opcode() == Op_Catch) { - rep_ctrl = rep_ctrl->in(0)->in(0); - - assert(phase->is_dominator(val_ctrl, rep_ctrl), "bad control"); - } else { - LoopNode* c = ShenandoahWriteBarrierNode::try_move_before_pre_loop(rep_ctrl, val_ctrl, phase); - if (c != NULL) { - rep_ctrl = ShenandoahWriteBarrierNode::move_above_predicates(c, val_ctrl, phase); - } else { - while (rep_ctrl->is_IfProj()) { - CallStaticJavaNode* unc = rep_ctrl->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none); - if (unc != NULL) { - int req = unc->uncommon_trap_request(); - Deoptimization::DeoptReason trap_reason = Deoptimization::trap_request_reason(req); - if ((trap_reason == Deoptimization::Reason_loop_limit_check || - trap_reason == Deoptimization::Reason_predicate || - trap_reason == Deoptimization::Reason_profile_predicate) && - phase->is_dominator(val_ctrl, rep_ctrl->in(0)->in(0))) { - rep_ctrl = rep_ctrl->in(0)->in(0); - continue; - } - } - break; - } - } - } - - Node* wb_ctrl = phase->get_ctrl(wb); - Node* other_ctrl = phase->get_ctrl(other); - int alias = phase->C->get_alias_index(wb->adr_type()); - MemoryGraphFixer* fixer = find_fixer(memory_graph_fixers, alias);; - if (!is_dominator(wb_ctrl, other_ctrl, wb, other, phase)) { - if (fixer == NULL) { - fixer = create_fixer(memory_graph_fixers, alias, phase, include_lsm); - } - Node* mem = fixer->find_mem(rep_ctrl, phase->get_ctrl(other) == rep_ctrl ? other : NULL); - - if (mem->has_out_with(Op_Lock) || mem->has_out_with(Op_Unlock)) { - continue; - } - - Node* wb_proj = wb->find_out_with(Op_ShenandoahWBMemProj); - fixer->remove(wb_proj); - Node* mem_for_ctrl = fixer->find_mem(rep_ctrl, NULL); - - if (wb->in(Memory) != mem) { - disconnect_barrier_mem(wb, phase->igvn()); - phase->igvn().replace_input_of(wb, Memory, mem); - } - if (rep_ctrl != wb_ctrl) { - phase->set_ctrl_and_loop(wb, rep_ctrl); - phase->igvn().replace_input_of(wb, Control, rep_ctrl); - phase->set_ctrl_and_loop(wb_proj, rep_ctrl); - progress = true; - } - - fixer->fix_mem(rep_ctrl, rep_ctrl, mem, mem_for_ctrl, wb_proj, uses); - - assert(!ShenandoahVerifyOptoBarriers || ShenandoahWriteBarrierNode::memory_dominates_all_paths(mem, rep_ctrl, alias, phase), "can't fix the memory graph"); - } - - if (other->Opcode() == Op_ShenandoahWriteBarrier) { - Node* other_proj = other->find_out_with(Op_ShenandoahWBMemProj); - if (fixer != NULL) { - fixer->remove(other_proj); - } - phase->igvn().replace_node(other_proj, other->in(Memory)); - } - phase->igvn().replace_node(other, wb); - --j; --jmax; - } - } - } - } - } while(progress); -} - -void ShenandoahReadBarrierNode::try_move(Node *n_ctrl, PhaseIdealLoop* phase) { - Node* mem = in(MemNode::Memory); - int alias = phase->C->get_alias_index(adr_type()); - const bool trace = false; - -#ifdef ASSERT - if (trace) { tty->print("Trying to move mem of"); dump(); } -#endif - - Node* new_mem = mem; - - ResourceMark rm; - VectorSet seen(Thread::current()->resource_area()); - Node_List phis; - - for (;;) { -#ifdef ASSERT - if (trace) { tty->print("Looking for dominator from"); mem->dump(); } -#endif - if (mem->is_Proj() && mem->in(0)->is_Start()) { - if (new_mem != in(MemNode::Memory)) { -#ifdef ASSERT - if (trace) { tty->print("XXX Setting mem to"); new_mem->dump(); tty->print(" for "); dump(); } -#endif - phase->igvn().replace_input_of(this, MemNode::Memory, new_mem); - } - return; - } - - Node* candidate = mem; - do { - if (!is_independent(mem)) { - if (trace) { tty->print_cr("Not independent"); } - if (new_mem != in(MemNode::Memory)) { -#ifdef ASSERT - if (trace) { tty->print("XXX Setting mem to"); new_mem->dump(); tty->print(" for "); dump(); } -#endif - phase->igvn().replace_input_of(this, MemNode::Memory, new_mem); - } - return; - } - if (seen.test_set(mem->_idx)) { - if (trace) { tty->print_cr("Already seen"); } - ShouldNotReachHere(); - // Strange graph - if (new_mem != in(MemNode::Memory)) { -#ifdef ASSERT - if (trace) { tty->print("XXX Setting mem to"); new_mem->dump(); tty->print(" for "); dump(); } -#endif - phase->igvn().replace_input_of(this, MemNode::Memory, new_mem); - } - return; - } - if (mem->is_Phi()) { - phis.push(mem); - } - mem = next_mem(mem, alias); - if (mem->bottom_type() == Type::MEMORY) { - candidate = mem; - } - assert(is_dominator(phase->ctrl_or_self(mem), n_ctrl, mem, this, phase) == phase->is_dominator(phase->ctrl_or_self(mem), n_ctrl), "strange dominator"); -#ifdef ASSERT - if (trace) { tty->print("Next mem is"); mem->dump(); } -#endif - } while (mem->bottom_type() != Type::MEMORY || !phase->is_dominator(phase->ctrl_or_self(mem), n_ctrl)); - - assert(mem->bottom_type() == Type::MEMORY, "bad mem"); - - bool not_dom = false; - for (uint i = 0; i < phis.size() && !not_dom; i++) { - Node* nn = phis.at(i); - -#ifdef ASSERT - if (trace) { tty->print("Looking from phi"); nn->dump(); } -#endif - assert(nn->is_Phi(), "phis only"); - for (uint j = 2; j < nn->req() && !not_dom; j++) { - Node* m = nn->in(j); -#ifdef ASSERT - if (trace) { tty->print("Input %d is", j); m->dump(); } -#endif - while (m != mem && !seen.test_set(m->_idx)) { - if (is_dominator(phase->ctrl_or_self(m), phase->ctrl_or_self(mem), m, mem, phase)) { - not_dom = true; - // Scheduling anomaly -#ifdef ASSERT - if (trace) { tty->print("Giving up"); m->dump(); } -#endif - break; - } - if (!is_independent(m)) { - if (trace) { tty->print_cr("Not independent"); } - if (new_mem != in(MemNode::Memory)) { -#ifdef ASSERT - if (trace) { tty->print("XXX Setting mem to"); new_mem->dump(); tty->print(" for "); dump(); } -#endif - phase->igvn().replace_input_of(this, MemNode::Memory, new_mem); - } - return; - } - if (m->is_Phi()) { - phis.push(m); - } - m = next_mem(m, alias); -#ifdef ASSERT - if (trace) { tty->print("Next mem is"); m->dump(); } -#endif - } - } - } - if (!not_dom) { - new_mem = mem; - phis.clear(); - } else { - seen.Clear(); - } - } -} - -CallStaticJavaNode* ShenandoahWriteBarrierNode::pin_and_expand_null_check(PhaseIterGVN& igvn) { - Node* val = in(ValueIn); - - const Type* val_t = igvn.type(val); - - if (val_t->meet(TypePtr::NULL_PTR) != val_t && - val->Opcode() == Op_CastPP && - val->in(0) != NULL && - val->in(0)->Opcode() == Op_IfTrue && - val->in(0)->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none) && - val->in(0)->in(0)->is_If() && - val->in(0)->in(0)->in(1)->Opcode() == Op_Bool && - val->in(0)->in(0)->in(1)->as_Bool()->_test._test == BoolTest::ne && - val->in(0)->in(0)->in(1)->in(1)->Opcode() == Op_CmpP && - val->in(0)->in(0)->in(1)->in(1)->in(1) == val->in(1) && - val->in(0)->in(0)->in(1)->in(1)->in(2)->bottom_type() == TypePtr::NULL_PTR) { - assert(val->in(0)->in(0)->in(1)->in(1)->in(1) == val->in(1), ""); - CallStaticJavaNode* unc = val->in(0)->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none); - return unc; - } - return NULL; -} - -void ShenandoahWriteBarrierNode::pin_and_expand_move_barrier(PhaseIdealLoop* phase, GrowableArray& memory_graph_fixers, Unique_Node_List& uses) { - Node* unc = pin_and_expand_null_check(phase->igvn()); - Node* val = in(ValueIn); - - if (unc != NULL) { - Node* ctrl = phase->get_ctrl(this); - Node* unc_ctrl = val->in(0); - - // Don't move write barrier in a loop - IdealLoopTree* loop = phase->get_loop(ctrl); - IdealLoopTree* unc_loop = phase->get_loop(unc_ctrl); - - if (!unc_loop->is_member(loop)) { - return; - } - - Node* branch = no_branches(ctrl, unc_ctrl, false, phase); - assert(branch == NULL || branch == NodeSentinel, "was not looking for a branch"); - if (branch == NodeSentinel) { - return; - } - - RegionNode* r = new RegionNode(3); - IfNode* iff = unc_ctrl->in(0)->as_If(); - - Node* ctrl_use = unc_ctrl->unique_ctrl_out(); - Node* unc_ctrl_clone = unc_ctrl->clone(); - phase->register_control(unc_ctrl_clone, loop, iff); - Node* c = unc_ctrl_clone; - Node* new_cast = clone_null_check(c, val, unc_ctrl_clone, phase); - r->init_req(1, new_cast->in(0)->in(0)->as_If()->proj_out(0)); - - phase->igvn().replace_input_of(unc_ctrl, 0, c->in(0)); - phase->set_idom(unc_ctrl, c->in(0), phase->dom_depth(unc_ctrl)); - phase->lazy_replace(c, unc_ctrl); - c = NULL;; - phase->igvn().replace_input_of(val, 0, unc_ctrl_clone); - phase->set_ctrl(val, unc_ctrl_clone); - - IfNode* new_iff = new_cast->in(0)->in(0)->as_If(); - fix_null_check(unc, unc_ctrl_clone, r, uses, phase); - Node* iff_proj = iff->proj_out(0); - r->init_req(2, iff_proj); - phase->register_control(r, phase->ltree_root(), iff); - - Node* new_bol = new_iff->in(1)->clone(); - Node* new_cmp = new_bol->in(1)->clone(); - assert(new_cmp->Opcode() == Op_CmpP, "broken"); - assert(new_cmp->in(1) == val->in(1), "broken"); - new_bol->set_req(1, new_cmp); - new_cmp->set_req(1, this); - phase->register_new_node(new_bol, new_iff->in(0)); - phase->register_new_node(new_cmp, new_iff->in(0)); - phase->igvn().replace_input_of(new_iff, 1, new_bol); - phase->igvn().replace_input_of(new_cast, 1, this); - - for (DUIterator_Fast imax, i = this->fast_outs(imax); i < imax; i++) { - Node* u = this->fast_out(i); - if (u == new_cast || u->Opcode() == Op_ShenandoahWBMemProj || u == new_cmp) { - continue; - } - phase->igvn().rehash_node_delayed(u); - int nb = u->replace_edge(this, new_cast); - assert(nb > 0, "no update?"); - --i; imax -= nb; - } - - for (DUIterator_Fast imax, i = val->fast_outs(imax); i < imax; i++) { - Node* u = val->fast_out(i); - if (u == this) { - continue; - } - phase->igvn().rehash_node_delayed(u); - int nb = u->replace_edge(val, new_cast); - assert(nb > 0, "no update?"); - --i; imax -= nb; - } - - Node* new_ctrl = unc_ctrl_clone; - - int alias = phase->C->get_alias_index(adr_type()); - MemoryGraphFixer* fixer = find_fixer(memory_graph_fixers, alias); - if (fixer == NULL) { - fixer = create_fixer(memory_graph_fixers, alias, phase, true); - } - - Node* proj = find_out_with(Op_ShenandoahWBMemProj); - fixer->remove(proj); - Node* mem = fixer->find_mem(new_ctrl, NULL); - - if (in(Memory) != mem) { - disconnect_barrier_mem(this, phase->igvn()); - phase->igvn().replace_input_of(this, Memory, mem); - } - - phase->set_ctrl_and_loop(this, new_ctrl); - phase->igvn().replace_input_of(this, Control, new_ctrl); - phase->set_ctrl_and_loop(proj, new_ctrl); - - fixer->fix_mem(new_ctrl, new_ctrl, mem, mem, proj, uses); - } -} - -void ShenandoahWriteBarrierNode::pin_and_expand_helper(PhaseIdealLoop* phase) { - Node* val = in(ValueIn); - CallStaticJavaNode* unc = pin_and_expand_null_check(phase->igvn()); - Node* rep = this; - Node* ctrl = phase->get_ctrl(this); - if (unc != NULL && val->in(0) == ctrl) { - Node* unc_ctrl = val->in(0); - IfNode* other_iff = unc_ctrl->unique_ctrl_out()->as_If(); - ProjNode* other_unc_ctrl = other_iff->proj_out(1); - Node* cast = NULL; - for (DUIterator_Fast imax, i = other_unc_ctrl->fast_outs(imax); i < imax && cast == NULL; i++) { - Node* u = other_unc_ctrl->fast_out(i); - if (u->Opcode() == Op_CastPP && u->in(1) == this) { - cast = u; - } - } - assert(other_unc_ctrl->is_uncommon_trap_if_pattern(Deoptimization::Reason_none) == unc, "broken"); - rep = cast; - } - - // Replace all uses of barrier's input that are dominated by ctrl - // with the value returned by the barrier: no need to keep both - // live. - for (DUIterator_Fast imax, i = val->fast_outs(imax); i < imax; i++) { - Node* u = val->fast_out(i); - if (u != this) { - if (u->is_Phi()) { - int nb = 0; - for (uint j = 1; j < u->req(); j++) { - if (u->in(j) == val) { - Node* c = u->in(0)->in(j); - if (phase->is_dominator(ctrl, c)) { - phase->igvn().replace_input_of(u, j, rep); - nb++; - } - } - } - if (nb > 0) { - imax -= nb; - --i; - } - } else { - Node* c = phase->ctrl_or_self(u); - if (is_dominator(ctrl, c, this, u, phase)) { - phase->igvn().rehash_node_delayed(u); - int nb = u->replace_edge(val, rep); - assert(nb > 0, "no update?"); - --i, imax -= nb; - } - } - } - } -} - -Node* ShenandoahWriteBarrierNode::find_bottom_mem(Node* ctrl, PhaseIdealLoop* phase) { +Node* ShenandoahBarrierC2Support::find_bottom_mem(Node* ctrl, PhaseIdealLoop* phase) { Node* mem = NULL; Node* c = ctrl; do { @@ -2262,7 +914,7 @@ return mem; } -void ShenandoahWriteBarrierNode::follow_barrier_uses(Node* n, Node* ctrl, Unique_Node_List& uses, PhaseIdealLoop* phase) { +void ShenandoahBarrierC2Support::follow_barrier_uses(Node* n, Node* ctrl, Unique_Node_List& uses, PhaseIdealLoop* phase) { for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) { Node* u = n->fast_out(i); if (!u->is_CFG() && phase->get_ctrl(u) == ctrl && (!u->is_Phi() || !u->in(0)->is_Loop() || u->in(LoopNode::LoopBackControl) != n)) { @@ -2282,7 +934,7 @@ inner->clear_strip_mined(); } -void ShenandoahWriteBarrierNode::test_heap_stable(Node*& ctrl, Node* raw_mem, Node*& heap_stable_ctrl, +void ShenandoahBarrierC2Support::test_heap_stable(Node*& ctrl, Node* raw_mem, Node*& heap_stable_ctrl, PhaseIdealLoop* phase) { IdealLoopTree* loop = phase->get_loop(ctrl); Node* thread = new ThreadLocalNode(); @@ -2314,7 +966,7 @@ assert(is_heap_stable_test(heap_stable_iff), "Should match the shape"); } -void ShenandoahWriteBarrierNode::test_null(Node*& ctrl, Node* val, Node*& null_ctrl, PhaseIdealLoop* phase) { +void ShenandoahBarrierC2Support::test_null(Node*& ctrl, Node* val, Node*& null_ctrl, PhaseIdealLoop* phase) { const Type* val_t = phase->igvn().type(val); if (val_t->meet(TypePtr::NULL_PTR) == val_t) { IdealLoopTree* loop = phase->get_loop(ctrl); @@ -2331,7 +983,7 @@ } } -Node* ShenandoahWriteBarrierNode::clone_null_check(Node*& c, Node* val, Node* unc_ctrl, PhaseIdealLoop* phase) { +Node* ShenandoahBarrierC2Support::clone_null_check(Node*& c, Node* val, Node* unc_ctrl, PhaseIdealLoop* phase) { IdealLoopTree *loop = phase->get_loop(c); Node* iff = unc_ctrl->in(0); assert(iff->is_If(), "broken"); @@ -2352,7 +1004,7 @@ return val; } -void ShenandoahWriteBarrierNode::fix_null_check(Node* unc, Node* unc_ctrl, Node* new_unc_ctrl, +void ShenandoahBarrierC2Support::fix_null_check(Node* unc, Node* unc_ctrl, Node* new_unc_ctrl, Unique_Node_List& uses, PhaseIdealLoop* phase) { IfNode* iff = unc_ctrl->in(0)->as_If(); Node* proj = iff->proj_out(0); @@ -2401,7 +1053,7 @@ assert(nb == 1, "only use expected"); } -void ShenandoahWriteBarrierNode::in_cset_fast_test(Node*& ctrl, Node*& not_cset_ctrl, Node* val, Node* raw_mem, PhaseIdealLoop* phase) { +void ShenandoahBarrierC2Support::in_cset_fast_test(Node*& ctrl, Node*& not_cset_ctrl, Node* val, Node* raw_mem, PhaseIdealLoop* phase) { IdealLoopTree *loop = phase->get_loop(ctrl); Node* raw_rbtrue = new CastP2XNode(ctrl, val); phase->register_new_node(raw_rbtrue, ctrl); @@ -2430,23 +1082,18 @@ phase->register_control(ctrl, loop, in_cset_fast_test_iff); } -void ShenandoahWriteBarrierNode::call_wb_stub(Node*& ctrl, Node*& val, Node*& result_mem, - Node* raw_mem, Node* wb_mem, - int alias, - PhaseIdealLoop* phase) { +void ShenandoahBarrierC2Support::call_lrb_stub(Node*& ctrl, Node*& val, Node*& result_mem, Node* raw_mem, PhaseIdealLoop* phase) { IdealLoopTree*loop = phase->get_loop(ctrl); const TypePtr* obj_type = phase->igvn().type(val)->is_oopptr()->cast_to_nonconst(); // The slow path stub consumes and produces raw memory in addition // to the existing memory edges Node* base = find_bottom_mem(ctrl, phase); - MergeMemNode* mm = MergeMemNode::make(base); - mm->set_memory_at(alias, wb_mem); mm->set_memory_at(Compile::AliasIdxRaw, raw_mem); phase->register_new_node(mm, ctrl); - Node* call = new CallLeafNode(ShenandoahBarrierSetC2::shenandoah_write_barrier_Type(), CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_JRT), "shenandoah_write_barrier", TypeRawPtr::BOTTOM); + Node* call = new CallLeafNode(ShenandoahBarrierSetC2::shenandoah_write_barrier_Type(), CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_JRT), "shenandoah_write_barrier", TypeRawPtr::BOTTOM); call->init_req(TypeFunc::Control, ctrl); call->init_req(TypeFunc::I_O, phase->C->top()); call->init_req(TypeFunc::Memory, mm); @@ -2464,7 +1111,7 @@ phase->register_new_node(val, ctrl); } -void ShenandoahWriteBarrierNode::fix_ctrl(Node* barrier, Node* region, const MemoryGraphFixer& fixer, Unique_Node_List& uses, Unique_Node_List& uses_to_ignore, uint last, PhaseIdealLoop* phase) { +void ShenandoahBarrierC2Support::fix_ctrl(Node* barrier, Node* region, const MemoryGraphFixer& fixer, Unique_Node_List& uses, Unique_Node_List& uses_to_ignore, uint last, PhaseIdealLoop* phase) { Node* ctrl = phase->get_ctrl(barrier); Node* init_raw_mem = fixer.find_mem(ctrl, barrier); @@ -2517,26 +1164,32 @@ } } -void ShenandoahWriteBarrierNode::pin_and_expand(PhaseIdealLoop* phase) { - Node_List enqueue_barriers; - if (ShenandoahStoreValEnqueueBarrier) { - Unique_Node_List wq; - wq.push(phase->C->root()); - for (uint i = 0; i < wq.size(); i++) { - Node* n = wq.at(i); - if (n->Opcode() == Op_ShenandoahEnqueueBarrier) { - enqueue_barriers.push(n); - } - for (uint i = 0; i < n->req(); i++) { - Node* in = n->in(i); - if (in != NULL) { - wq.push(in); - } - } +static Node* create_phis_on_call_return(Node* ctrl, Node* c, Node* n, Node* n_clone, const CallProjections& projs, PhaseIdealLoop* phase) { + Node* region = NULL; + while (c != ctrl) { + if (c->is_Region()) { + region = c; + } + c = phase->idom(c); + } + assert(region != NULL, ""); + Node* phi = new PhiNode(region, n->bottom_type()); + for (uint j = 1; j < region->req(); j++) { + Node* in = region->in(j); + if (phase->is_dominator(projs.fallthrough_catchproj, in)) { + phi->init_req(j, n); + } else if (phase->is_dominator(projs.catchall_catchproj, in)) { + phi->init_req(j, n_clone); + } else { + phi->init_req(j, create_phis_on_call_return(ctrl, in, n, n_clone, projs, phase)); } } + phase->register_new_node(phi, region); + return phi; +} - const bool trace = false; +void ShenandoahBarrierC2Support::pin_and_expand(PhaseIdealLoop* phase) { + ShenandoahBarrierSetC2State* state = ShenandoahBarrierSetC2::bsc2()->state(); // Collect raw memory state at CFG points in the entire graph and // record it in memory_nodes. Optimize the raw memory graph in the @@ -2544,34 +1197,9 @@ // simpler. GrowableArray memory_graph_fixers; - // Let's try to common write barriers again - optimize_before_expansion(phase, memory_graph_fixers, true); - Unique_Node_List uses; - for (int i = 0; i < ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_barriers_count(); i++) { - ShenandoahWriteBarrierNode* wb = ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_barrier(i); - Node* ctrl = phase->get_ctrl(wb); - - Node* val = wb->in(ValueIn); - if (ctrl->is_Proj() && ctrl->in(0)->is_CallJava()) { - assert(is_dominator(phase->get_ctrl(val), ctrl->in(0)->in(0), val, ctrl->in(0), phase), "can't move"); - phase->set_ctrl(wb, ctrl->in(0)->in(0)); - } else if (ctrl->is_CallRuntime()) { - assert(is_dominator(phase->get_ctrl(val), ctrl->in(0), val, ctrl, phase), "can't move"); - phase->set_ctrl(wb, ctrl->in(0)); - } - - assert(wb->Opcode() == Op_ShenandoahWriteBarrier, "only for write barriers"); - // Look for a null check that dominates this barrier and move the - // barrier right after the null check to enable implicit null - // checks - wb->pin_and_expand_move_barrier(phase, memory_graph_fixers, uses); - - wb->pin_and_expand_helper(phase); - } - - for (uint i = 0; i < enqueue_barriers.size(); i++) { - Node* barrier = enqueue_barriers.at(i); + for (int i = 0; i < state->enqueue_barriers_count(); i++) { + Node* barrier = state->enqueue_barrier(i); Node* ctrl = phase->get_ctrl(barrier); IdealLoopTree* loop = phase->get_loop(ctrl); if (loop->_head->is_OuterStripMinedLoop()) { @@ -2583,24 +1211,386 @@ } } - for (int i = ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_barriers_count(); i > 0; i--) { - int cnt = ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_barriers_count(); - ShenandoahWriteBarrierNode* wb = ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_barrier(i-1); - Node* ctrl = phase->get_ctrl(wb); - IdealLoopTree* loop = phase->get_loop(ctrl); - if (loop->_head->is_OuterStripMinedLoop()) { - // Expanding a barrier here will break loop strip mining - // verification. Transform the loop so the loop nest doesn't - // appear as strip mined. - OuterStripMinedLoopNode* outer = loop->_head->as_OuterStripMinedLoop(); - hide_strip_mined_loop(outer, outer->unique_ctrl_out()->as_CountedLoop(), phase); + Node_Stack stack(0); + Node_List clones; + for (int i = state->load_reference_barriers_count() - 1; i >= 0; i--) { + ShenandoahLoadReferenceBarrierNode* lrb = state->load_reference_barrier(i); + if (lrb->get_barrier_strength() == ShenandoahLoadReferenceBarrierNode::NONE) { + continue; + } + + Node* ctrl = phase->get_ctrl(lrb); + Node* val = lrb->in(ShenandoahLoadReferenceBarrierNode::ValueIn); + + CallStaticJavaNode* unc = NULL; + Node* unc_ctrl = NULL; + Node* uncasted_val = val; + + for (DUIterator_Fast imax, i = lrb->fast_outs(imax); i < imax; i++) { + Node* u = lrb->fast_out(i); + if (u->Opcode() == Op_CastPP && + u->in(0) != NULL && + phase->is_dominator(u->in(0), ctrl)) { + const Type* u_t = phase->igvn().type(u); + + if (u_t->meet(TypePtr::NULL_PTR) != u_t && + u->in(0)->Opcode() == Op_IfTrue && + u->in(0)->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none) && + u->in(0)->in(0)->is_If() && + u->in(0)->in(0)->in(1)->Opcode() == Op_Bool && + u->in(0)->in(0)->in(1)->as_Bool()->_test._test == BoolTest::ne && + u->in(0)->in(0)->in(1)->in(1)->Opcode() == Op_CmpP && + u->in(0)->in(0)->in(1)->in(1)->in(1) == val && + u->in(0)->in(0)->in(1)->in(1)->in(2)->bottom_type() == TypePtr::NULL_PTR) { + IdealLoopTree* loop = phase->get_loop(ctrl); + IdealLoopTree* unc_loop = phase->get_loop(u->in(0)); + + if (!unc_loop->is_member(loop)) { + continue; + } + + Node* branch = no_branches(ctrl, u->in(0), false, phase); + assert(branch == NULL || branch == NodeSentinel, "was not looking for a branch"); + if (branch == NodeSentinel) { + continue; + } + + phase->igvn().replace_input_of(u, 1, val); + phase->igvn().replace_input_of(lrb, ShenandoahLoadReferenceBarrierNode::ValueIn, u); + phase->set_ctrl(u, u->in(0)); + phase->set_ctrl(lrb, u->in(0)); + unc = u->in(0)->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none); + unc_ctrl = u->in(0); + val = u; + + for (DUIterator_Fast jmax, j = val->fast_outs(jmax); j < jmax; j++) { + Node* u = val->fast_out(j); + if (u == lrb) continue; + phase->igvn().rehash_node_delayed(u); + int nb = u->replace_edge(val, lrb); + --j; jmax -= nb; + } + + RegionNode* r = new RegionNode(3); + IfNode* iff = unc_ctrl->in(0)->as_If(); + + Node* ctrl_use = unc_ctrl->unique_ctrl_out(); + Node* unc_ctrl_clone = unc_ctrl->clone(); + phase->register_control(unc_ctrl_clone, loop, iff); + Node* c = unc_ctrl_clone; + Node* new_cast = clone_null_check(c, val, unc_ctrl_clone, phase); + r->init_req(1, new_cast->in(0)->in(0)->as_If()->proj_out(0)); + + phase->igvn().replace_input_of(unc_ctrl, 0, c->in(0)); + phase->set_idom(unc_ctrl, c->in(0), phase->dom_depth(unc_ctrl)); + phase->lazy_replace(c, unc_ctrl); + c = NULL;; + phase->igvn().replace_input_of(val, 0, unc_ctrl_clone); + phase->set_ctrl(val, unc_ctrl_clone); + + IfNode* new_iff = new_cast->in(0)->in(0)->as_If(); + fix_null_check(unc, unc_ctrl_clone, r, uses, phase); + Node* iff_proj = iff->proj_out(0); + r->init_req(2, iff_proj); + phase->register_control(r, phase->ltree_root(), iff); + + Node* new_bol = new_iff->in(1)->clone(); + Node* new_cmp = new_bol->in(1)->clone(); + assert(new_cmp->Opcode() == Op_CmpP, "broken"); + assert(new_cmp->in(1) == val->in(1), "broken"); + new_bol->set_req(1, new_cmp); + new_cmp->set_req(1, lrb); + phase->register_new_node(new_bol, new_iff->in(0)); + phase->register_new_node(new_cmp, new_iff->in(0)); + phase->igvn().replace_input_of(new_iff, 1, new_bol); + phase->igvn().replace_input_of(new_cast, 1, lrb); + + for (DUIterator_Fast imax, i = lrb->fast_outs(imax); i < imax; i++) { + Node* u = lrb->fast_out(i); + if (u == new_cast || u == new_cmp) { + continue; + } + phase->igvn().rehash_node_delayed(u); + int nb = u->replace_edge(lrb, new_cast); + assert(nb > 0, "no update?"); + --i; imax -= nb; + } + + for (DUIterator_Fast imax, i = val->fast_outs(imax); i < imax; i++) { + Node* u = val->fast_out(i); + if (u == lrb) { + continue; + } + phase->igvn().rehash_node_delayed(u); + int nb = u->replace_edge(val, new_cast); + assert(nb > 0, "no update?"); + --i; imax -= nb; + } + + ctrl = unc_ctrl_clone; + phase->set_ctrl_and_loop(lrb, ctrl); + break; + } + } + } + if (ctrl->is_Proj() && ctrl->in(0)->is_CallJava()) { + CallNode* call = ctrl->in(0)->as_CallJava(); + CallProjections projs; + call->extract_projections(&projs, false, false); + + Node* lrb_clone = lrb->clone(); + phase->register_new_node(lrb_clone, projs.catchall_catchproj); + phase->set_ctrl(lrb, projs.fallthrough_catchproj); + + stack.push(lrb, 0); + clones.push(lrb_clone); + + do { + assert(stack.size() == clones.size(), ""); + Node* n = stack.node(); +#ifdef ASSERT + if (n->is_Load()) { + Node* mem = n->in(MemNode::Memory); + for (DUIterator_Fast jmax, j = mem->fast_outs(jmax); j < jmax; j++) { + Node* u = mem->fast_out(j); + assert(!u->is_Store() || !u->is_LoadStore() || phase->get_ctrl(u) != ctrl, "anti dependent store?"); + } + } +#endif + uint idx = stack.index(); + Node* n_clone = clones.at(clones.size()-1); + if (idx < n->outcnt()) { + Node* u = n->raw_out(idx); + Node* c = phase->ctrl_or_self(u); + if (c == ctrl) { + stack.set_index(idx+1); + assert(!u->is_CFG(), ""); + stack.push(u, 0); + Node* u_clone = u->clone(); + int nb = u_clone->replace_edge(n, n_clone); + assert(nb > 0, "should have replaced some uses"); + phase->register_new_node(u_clone, projs.catchall_catchproj); + clones.push(u_clone); + phase->set_ctrl(u, projs.fallthrough_catchproj); + } else { + bool replaced = false; + if (u->is_Phi()) { + for (uint k = 1; k < u->req(); k++) { + if (u->in(k) == n) { + if (phase->is_dominator(projs.catchall_catchproj, u->in(0)->in(k))) { + phase->igvn().replace_input_of(u, k, n_clone); + replaced = true; + } else if (!phase->is_dominator(projs.fallthrough_catchproj, u->in(0)->in(k))) { + phase->igvn().replace_input_of(u, k, create_phis_on_call_return(ctrl, u->in(0)->in(k), n, n_clone, projs, phase)); + replaced = true; + } + } + } + } else { + if (phase->is_dominator(projs.catchall_catchproj, c)) { + phase->igvn().rehash_node_delayed(u); + int nb = u->replace_edge(n, n_clone); + assert(nb > 0, "should have replaced some uses"); + replaced = true; + } else if (!phase->is_dominator(projs.fallthrough_catchproj, c)) { + phase->igvn().rehash_node_delayed(u); + int nb = u->replace_edge(n, create_phis_on_call_return(ctrl, c, n, n_clone, projs, phase)); + assert(nb > 0, "should have replaced some uses"); + replaced = true; + } + } + if (!replaced) { + stack.set_index(idx+1); + } + } + } else { + // assert(n_clone->outcnt() > 0, ""); + // assert(n->outcnt() > 0, ""); + stack.pop(); + clones.pop(); + } + } while (stack.size() > 0); + assert(stack.size() == 0 && clones.size() == 0, ""); + ctrl = projs.fallthrough_catchproj; } } + // Expand load-reference-barriers MemoryGraphFixer fixer(Compile::AliasIdxRaw, true, phase); Unique_Node_List uses_to_ignore; - for (uint i = 0; i < enqueue_barriers.size(); i++) { - Node* barrier = enqueue_barriers.at(i); + for (int i = state->load_reference_barriers_count() - 1; i >= 0; i--) { + ShenandoahLoadReferenceBarrierNode* lrb = state->load_reference_barrier(i); + if (lrb->get_barrier_strength() == ShenandoahLoadReferenceBarrierNode::NONE) { + phase->igvn().replace_node(lrb, lrb->in(ShenandoahLoadReferenceBarrierNode::ValueIn)); + continue; + } + uint last = phase->C->unique(); + Node* ctrl = phase->get_ctrl(lrb); + Node* val = lrb->in(ShenandoahLoadReferenceBarrierNode::ValueIn); + + + Node* orig_ctrl = ctrl; + + Node* raw_mem = fixer.find_mem(ctrl, lrb); + Node* init_raw_mem = raw_mem; + Node* raw_mem_for_ctrl = fixer.find_mem(ctrl, NULL); + // int alias = phase->C->get_alias_index(lrb->adr_type()); + + IdealLoopTree *loop = phase->get_loop(ctrl); + CallStaticJavaNode* unc = lrb->pin_and_expand_null_check(phase->igvn()); + Node* unc_ctrl = NULL; + if (unc != NULL) { + if (val->in(ShenandoahLoadReferenceBarrierNode::Control) != ctrl) { + unc = NULL; + } else { + unc_ctrl = val->in(ShenandoahLoadReferenceBarrierNode::Control); + } + } + + Node* uncasted_val = val; + if (unc != NULL) { + uncasted_val = val->in(1); + } + + Node* heap_stable_ctrl = NULL; + Node* null_ctrl = NULL; + + assert(val->bottom_type()->make_oopptr(), "need oop"); + assert(val->bottom_type()->make_oopptr()->const_oop() == NULL, "expect non-constant"); + + enum { _heap_stable = 1, _not_cset, _not_equal, _evac_path, _null_path, PATH_LIMIT }; + Node* region = new RegionNode(PATH_LIMIT); + Node* val_phi = new PhiNode(region, uncasted_val->bottom_type()->is_oopptr()); + Node* raw_mem_phi = PhiNode::make(region, raw_mem, Type::MEMORY, TypeRawPtr::BOTTOM); + + // Stable path. + test_heap_stable(ctrl, raw_mem, heap_stable_ctrl, phase); + IfNode* heap_stable_iff = heap_stable_ctrl->in(0)->as_If(); + + // Heap stable case + region->init_req(_heap_stable, heap_stable_ctrl); + val_phi->init_req(_heap_stable, uncasted_val); + raw_mem_phi->init_req(_heap_stable, raw_mem); + + Node* reg2_ctrl = NULL; + // Null case + test_null(ctrl, val, null_ctrl, phase); + if (null_ctrl != NULL) { + reg2_ctrl = null_ctrl->in(0); + region->init_req(_null_path, null_ctrl); + val_phi->init_req(_null_path, uncasted_val); + raw_mem_phi->init_req(_null_path, raw_mem); + } else { + region->del_req(_null_path); + val_phi->del_req(_null_path); + raw_mem_phi->del_req(_null_path); + } + + // Test for in-cset. + // Wires !in_cset(obj) to slot 2 of region and phis + Node* not_cset_ctrl = NULL; + in_cset_fast_test(ctrl, not_cset_ctrl, uncasted_val, raw_mem, phase); + if (not_cset_ctrl != NULL) { + if (reg2_ctrl == NULL) reg2_ctrl = not_cset_ctrl->in(0); + region->init_req(_not_cset, not_cset_ctrl); + val_phi->init_req(_not_cset, uncasted_val); + raw_mem_phi->init_req(_not_cset, raw_mem); + } + + // Resolve object when orig-value is in cset. + // Make the unconditional resolve for fwdptr. + Node* new_val = uncasted_val; + if (unc_ctrl != NULL) { + // Clone the null check in this branch to allow implicit null check + new_val = clone_null_check(ctrl, val, unc_ctrl, phase); + fix_null_check(unc, unc_ctrl, ctrl->in(0)->as_If()->proj_out(0), uses, phase); + + IfNode* iff = unc_ctrl->in(0)->as_If(); + phase->igvn().replace_input_of(iff, 1, phase->igvn().intcon(1)); + } + Node* addr = new AddPNode(new_val, uncasted_val, phase->igvn().MakeConX(ShenandoahBrooksPointer::byte_offset())); + phase->register_new_node(addr, ctrl); + assert(val->bottom_type()->isa_oopptr(), "what else?"); + const TypePtr* obj_type = val->bottom_type()->is_oopptr(); + const TypePtr* adr_type = TypeRawPtr::BOTTOM; + Node* fwd = new LoadPNode(ctrl, raw_mem, addr, adr_type, obj_type, MemNode::unordered); + phase->register_new_node(fwd, ctrl); + + // Only branch to LRB stub if object is not forwarded; otherwise reply with fwd ptr + Node* cmp = new CmpPNode(fwd, new_val); + phase->register_new_node(cmp, ctrl); + Node* bol = new BoolNode(cmp, BoolTest::eq); + phase->register_new_node(bol, ctrl); + + IfNode* iff = new IfNode(ctrl, bol, PROB_UNLIKELY(0.999), COUNT_UNKNOWN); + if (reg2_ctrl == NULL) reg2_ctrl = iff; + phase->register_control(iff, loop, ctrl); + Node* if_not_eq = new IfFalseNode(iff); + phase->register_control(if_not_eq, loop, iff); + Node* if_eq = new IfTrueNode(iff); + phase->register_control(if_eq, loop, iff); + + // Wire up not-equal-path in slots 3. + region->init_req(_not_equal, if_not_eq); + val_phi->init_req(_not_equal, fwd); + raw_mem_phi->init_req(_not_equal, raw_mem); + + // Call wb-stub and wire up that path in slots 4 + Node* result_mem = NULL; + ctrl = if_eq; + call_lrb_stub(ctrl, fwd, result_mem, raw_mem, phase); + region->init_req(_evac_path, ctrl); + val_phi->init_req(_evac_path, fwd); + raw_mem_phi->init_req(_evac_path, result_mem); + + phase->register_control(region, loop, heap_stable_iff); + Node* out_val = val_phi; + phase->register_new_node(val_phi, region); + phase->register_new_node(raw_mem_phi, region); + + fix_ctrl(lrb, region, fixer, uses, uses_to_ignore, last, phase); + + ctrl = orig_ctrl; + + if (unc != NULL) { + for (DUIterator_Fast imax, i = val->fast_outs(imax); i < imax; i++) { + Node* u = val->fast_out(i); + Node* c = phase->ctrl_or_self(u); + if (u != lrb && (c != ctrl || is_dominator_same_ctrl(c, lrb, u, phase))) { + phase->igvn().rehash_node_delayed(u); + int nb = u->replace_edge(val, out_val); + --i, imax -= nb; + } + } + if (val->outcnt() == 0) { + phase->igvn()._worklist.push(val); + } + } + phase->igvn().replace_node(lrb, out_val); + + follow_barrier_uses(out_val, ctrl, uses, phase); + + for(uint next = 0; next < uses.size(); next++ ) { + Node *n = uses.at(next); + assert(phase->get_ctrl(n) == ctrl, "bad control"); + assert(n != init_raw_mem, "should leave input raw mem above the barrier"); + phase->set_ctrl(n, region); + follow_barrier_uses(n, ctrl, uses, phase); + } + + // The slow path call produces memory: hook the raw memory phi + // from the expanded load reference barrier with the rest of the graph + // which may require adding memory phis at every post dominated + // region and at enclosing loop heads. Use the memory state + // collected in memory_nodes to fix the memory graph. Update that + // memory state as we go. + fixer.fix_mem(ctrl, region, init_raw_mem, raw_mem_for_ctrl, raw_mem_phi, uses); + } + // Done expanding load-reference-barriers. + assert(ShenandoahBarrierSetC2::bsc2()->state()->load_reference_barriers_count() == 0, "all load reference barrier nodes should have been replaced"); + + for (int i = state->enqueue_barriers_count() - 1; i >= 0; i--) { + Node* barrier = state->enqueue_barrier(i); Node* pre_val = barrier->in(1); if (phase->igvn().type(pre_val)->higher_equal(TypePtr::NULL_PTR)) { @@ -2747,212 +1737,11 @@ phase->igvn().replace_node(barrier, pre_val); } - - for (int i = ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_barriers_count(); i > 0; i--) { - int cnt = ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_barriers_count(); - ShenandoahWriteBarrierNode* wb = ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_barrier(i-1); - - uint last = phase->C->unique(); - Node* ctrl = phase->get_ctrl(wb); - Node* orig_ctrl = ctrl; - - Node* raw_mem = fixer.find_mem(ctrl, wb); - Node* init_raw_mem = raw_mem; - Node* raw_mem_for_ctrl = fixer.find_mem(ctrl, NULL); - int alias = phase->C->get_alias_index(wb->adr_type()); - Node* wb_mem = wb->in(Memory); - Node* init_wb_mem = wb_mem; - - Node* val = wb->in(ValueIn); - Node* wbproj = wb->find_out_with(Op_ShenandoahWBMemProj); - IdealLoopTree *loop = phase->get_loop(ctrl); - - assert(val->Opcode() != Op_ShenandoahWriteBarrier, "No chain of write barriers"); - - CallStaticJavaNode* unc = wb->pin_and_expand_null_check(phase->igvn()); - Node* unc_ctrl = NULL; - if (unc != NULL) { - if (val->in(0) != ctrl) { - unc = NULL; - } else { - unc_ctrl = val->in(0); - } - } - - Node* uncasted_val = val; - if (unc != NULL) { - uncasted_val = val->in(1); - } - - Node* heap_stable_ctrl = NULL; - Node* null_ctrl = NULL; - - assert(val->bottom_type()->make_oopptr(), "need oop"); - assert(val->bottom_type()->make_oopptr()->const_oop() == NULL, "expect non-constant"); - - enum { _heap_stable = 1, _heap_unstable, PATH_LIMIT }; - Node* region = new RegionNode(PATH_LIMIT); - Node* val_phi = new PhiNode(region, uncasted_val->bottom_type()->is_oopptr()); - Node* mem_phi = PhiNode::make(region, wb_mem, Type::MEMORY, phase->C->alias_type(wb->adr_type())->adr_type()); - Node* raw_mem_phi = PhiNode::make(region, raw_mem, Type::MEMORY, TypeRawPtr::BOTTOM); - - enum { _not_cset = 1, _not_equal, _evac_path, _null_path, PATH_LIMIT2 }; - Node* region2 = new RegionNode(PATH_LIMIT2); - Node* val_phi2 = new PhiNode(region2, uncasted_val->bottom_type()->is_oopptr()); - Node* mem_phi2 = PhiNode::make(region2, wb_mem, Type::MEMORY, phase->C->alias_type(wb->adr_type())->adr_type()); - Node* raw_mem_phi2 = PhiNode::make(region2, raw_mem, Type::MEMORY, TypeRawPtr::BOTTOM); - - // Stable path. - test_heap_stable(ctrl, raw_mem, heap_stable_ctrl, phase); - IfNode* heap_stable_iff = heap_stable_ctrl->in(0)->as_If(); - - // Heap stable case - region->init_req(_heap_stable, heap_stable_ctrl); - val_phi->init_req(_heap_stable, uncasted_val); - mem_phi->init_req(_heap_stable, wb_mem); - raw_mem_phi->init_req(_heap_stable, raw_mem); - - Node* reg2_ctrl = NULL; - // Null case - test_null(ctrl, val, null_ctrl, phase); - if (null_ctrl != NULL) { - reg2_ctrl = null_ctrl->in(0); - region2->init_req(_null_path, null_ctrl); - val_phi2->init_req(_null_path, uncasted_val); - mem_phi2->init_req(_null_path, wb_mem); - raw_mem_phi2->init_req(_null_path, raw_mem); - } else { - region2->del_req(_null_path); - val_phi2->del_req(_null_path); - mem_phi2->del_req(_null_path); - raw_mem_phi2->del_req(_null_path); - } - - // Test for in-cset. - // Wires !in_cset(obj) to slot 2 of region and phis - Node* not_cset_ctrl = NULL; - in_cset_fast_test(ctrl, not_cset_ctrl, uncasted_val, raw_mem, phase); - if (not_cset_ctrl != NULL) { - if (reg2_ctrl == NULL) reg2_ctrl = not_cset_ctrl->in(0); - region2->init_req(_not_cset, not_cset_ctrl); - val_phi2->init_req(_not_cset, uncasted_val); - mem_phi2->init_req(_not_cset, wb_mem); - raw_mem_phi2->init_req(_not_cset, raw_mem); - } - - // Resolve object when orig-value is in cset. - // Make the unconditional resolve for fwdptr, not the read barrier. - Node* new_val = uncasted_val; - if (unc_ctrl != NULL) { - // Clone the null check in this branch to allow implicit null check - new_val = clone_null_check(ctrl, val, unc_ctrl, phase); - fix_null_check(unc, unc_ctrl, ctrl->in(0)->as_If()->proj_out(0), uses, phase); + assert(state->enqueue_barriers_count() == 0, "all enqueue barrier nodes should have been replaced"); - IfNode* iff = unc_ctrl->in(0)->as_If(); - phase->igvn().replace_input_of(iff, 1, phase->igvn().intcon(1)); - } - Node* addr = new AddPNode(new_val, uncasted_val, phase->igvn().MakeConX(ShenandoahBrooksPointer::byte_offset())); - phase->register_new_node(addr, ctrl); - assert(val->bottom_type()->isa_oopptr(), "what else?"); - const TypePtr* obj_type = val->bottom_type()->is_oopptr(); - const TypePtr* adr_type = ShenandoahBarrierNode::brooks_pointer_type(obj_type); - Node* fwd = new LoadPNode(ctrl, wb_mem, addr, adr_type, obj_type, MemNode::unordered); - phase->register_new_node(fwd, ctrl); - - // Only branch to WB stub if object is not forwarded; otherwise reply with fwd ptr - Node* cmp = new CmpPNode(fwd, new_val); - phase->register_new_node(cmp, ctrl); - Node* bol = new BoolNode(cmp, BoolTest::eq); - phase->register_new_node(bol, ctrl); - - IfNode* iff = new IfNode(ctrl, bol, PROB_UNLIKELY(0.999), COUNT_UNKNOWN); - if (reg2_ctrl == NULL) reg2_ctrl = iff; - phase->register_control(iff, loop, ctrl); - Node* if_not_eq = new IfFalseNode(iff); - phase->register_control(if_not_eq, loop, iff); - Node* if_eq = new IfTrueNode(iff); - phase->register_control(if_eq, loop, iff); - - // Wire up not-equal-path in slots 3. - region2->init_req(_not_equal, if_not_eq); - val_phi2->init_req(_not_equal, fwd); - mem_phi2->init_req(_not_equal, wb_mem); - raw_mem_phi2->init_req(_not_equal, raw_mem); - - // Call wb-stub and wire up that path in slots 4 - Node* result_mem = NULL; - ctrl = if_eq; - call_wb_stub(ctrl, new_val, result_mem, - raw_mem, wb_mem, - alias, phase); - region2->init_req(_evac_path, ctrl); - val_phi2->init_req(_evac_path, new_val); - mem_phi2->init_req(_evac_path, result_mem); - raw_mem_phi2->init_req(_evac_path, result_mem); - - phase->register_control(region2, loop, reg2_ctrl); - phase->register_new_node(val_phi2, region2); - phase->register_new_node(mem_phi2, region2); - phase->register_new_node(raw_mem_phi2, region2); - - region->init_req(_heap_unstable, region2); - val_phi->init_req(_heap_unstable, val_phi2); - mem_phi->init_req(_heap_unstable, mem_phi2); - raw_mem_phi->init_req(_heap_unstable, raw_mem_phi2); - - phase->register_control(region, loop, heap_stable_iff); - Node* out_val = val_phi; - phase->register_new_node(val_phi, region); - phase->register_new_node(mem_phi, region); - phase->register_new_node(raw_mem_phi, region); - - fix_ctrl(wb, region, fixer, uses, uses_to_ignore, last, phase); - - ctrl = orig_ctrl; - - phase->igvn().replace_input_of(wbproj, ShenandoahWBMemProjNode::WriteBarrier, phase->C->top()); - phase->igvn().replace_node(wbproj, mem_phi); - if (unc != NULL) { - for (DUIterator_Fast imax, i = val->fast_outs(imax); i < imax; i++) { - Node* u = val->fast_out(i); - Node* c = phase->ctrl_or_self(u); - if (u != wb && (c != ctrl || is_dominator_same_ctrl(c, wb, u, phase))) { - phase->igvn().rehash_node_delayed(u); - int nb = u->replace_edge(val, out_val); - --i, imax -= nb; - } - } - if (val->outcnt() == 0) { - phase->igvn()._worklist.push(val); - } - } - phase->igvn().replace_node(wb, out_val); - - follow_barrier_uses(mem_phi, ctrl, uses, phase); - follow_barrier_uses(out_val, ctrl, uses, phase); - - for(uint next = 0; next < uses.size(); next++ ) { - Node *n = uses.at(next); - assert(phase->get_ctrl(n) == ctrl, "bad control"); - assert(n != init_raw_mem, "should leave input raw mem above the barrier"); - phase->set_ctrl(n, region); - follow_barrier_uses(n, ctrl, uses, phase); - } - - // The slow path call produces memory: hook the raw memory phi - // from the expanded write barrier with the rest of the graph - // which may require adding memory phis at every post dominated - // region and at enclosing loop heads. Use the memory state - // collected in memory_nodes to fix the memory graph. Update that - // memory state as we go. - fixer.fix_mem(ctrl, region, init_raw_mem, raw_mem_for_ctrl, raw_mem_phi, uses); - assert(ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_barriers_count() == cnt - 1, "not replaced"); - } - - assert(ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_barriers_count() == 0, "all write barrier nodes should have been replaced"); } -void ShenandoahWriteBarrierNode::move_heap_stable_test_out_of_loop(IfNode* iff, PhaseIdealLoop* phase) { +void ShenandoahBarrierC2Support::move_heap_stable_test_out_of_loop(IfNode* iff, PhaseIdealLoop* phase) { IdealLoopTree *loop = phase->get_loop(iff); Node* loop_head = loop->_head; Node* entry_c = loop_head->in(LoopNode::EntryControl); @@ -2985,7 +1774,7 @@ } } -bool ShenandoahWriteBarrierNode::identical_backtoback_ifs(Node *n, PhaseIdealLoop* phase) { +bool ShenandoahBarrierC2Support::identical_backtoback_ifs(Node* n, PhaseIdealLoop* phase) { if (!n->is_If() || n->is_CountedLoopEnd()) { return false; } @@ -3020,7 +1809,7 @@ return true; } -void ShenandoahWriteBarrierNode::merge_back_to_back_tests(Node* n, PhaseIdealLoop* phase) { +void ShenandoahBarrierC2Support::merge_back_to_back_tests(Node* n, PhaseIdealLoop* phase) { assert(is_heap_stable_test(n), "no other tests"); if (identical_backtoback_ifs(n, phase)) { Node* n_ctrl = n->in(0); @@ -3056,10 +1845,54 @@ } } -void ShenandoahWriteBarrierNode::optimize_after_expansion(VectorSet &visited, Node_Stack &stack, Node_List &old_new, PhaseIdealLoop* phase) { +IfNode* ShenandoahBarrierC2Support::find_unswitching_candidate(const IdealLoopTree* loop, PhaseIdealLoop* phase) { + // Find first invariant test that doesn't exit the loop + LoopNode *head = loop->_head->as_Loop(); + IfNode* unswitch_iff = NULL; + Node* n = head->in(LoopNode::LoopBackControl); + int loop_has_sfpts = -1; + while (n != head) { + Node* n_dom = phase->idom(n); + if (n->is_Region()) { + if (n_dom->is_If()) { + IfNode* iff = n_dom->as_If(); + if (iff->in(1)->is_Bool()) { + BoolNode* bol = iff->in(1)->as_Bool(); + if (bol->in(1)->is_Cmp()) { + // If condition is invariant and not a loop exit, + // then found reason to unswitch. + if (is_heap_stable_test(iff) && + (loop_has_sfpts == -1 || loop_has_sfpts == 0)) { + assert(!loop->is_loop_exit(iff), "both branches should be in the loop"); + if (loop_has_sfpts == -1) { + for(uint i = 0; i < loop->_body.size(); i++) { + Node *m = loop->_body[i]; + if (m->is_SafePoint() && !m->is_CallLeaf()) { + loop_has_sfpts = 1; + break; + } + } + if (loop_has_sfpts == -1) { + loop_has_sfpts = 0; + } + } + if (!loop_has_sfpts) { + unswitch_iff = iff; + } + } + } + } + } + } + n = n_dom; + } + return unswitch_iff; +} + + +void ShenandoahBarrierC2Support::optimize_after_expansion(VectorSet &visited, Node_Stack &stack, Node_List &old_new, PhaseIdealLoop* phase) { Node_List heap_stable_tests; Node_List gc_state_loads; - stack.push(phase->C->start(), 0); do { Node* n = stack.node(); @@ -3073,10 +1906,10 @@ } } else { stack.pop(); - if (ShenandoahCommonGCStateLoads && ShenandoahWriteBarrierNode::is_gc_state_load(n)) { + if (ShenandoahCommonGCStateLoads && is_gc_state_load(n)) { gc_state_loads.push(n); } - if (n->is_If() && ShenandoahWriteBarrierNode::is_heap_stable_test(n)) { + if (n->is_If() && is_heap_stable_test(n)) { heap_stable_tests.push(n); } } @@ -3088,7 +1921,7 @@ for (uint i = 0; i < gc_state_loads.size(); i++) { Node* n = gc_state_loads.at(i); if (n->outcnt() != 0) { - progress |= ShenandoahWriteBarrierNode::try_common_gc_state_load(n, phase); + progress |= try_common_gc_state_load(n, phase); } } } while (progress); @@ -3109,23 +1942,25 @@ !loop->_irreducible) { LoopNode* head = loop->_head->as_Loop(); if ((!head->is_CountedLoop() || head->as_CountedLoop()->is_main_loop() || head->as_CountedLoop()->is_normal_loop()) && - !seen.test_set(head->_idx) && - loop->policy_unswitching(phase, true)) { - IfNode* iff = phase->find_unswitching_candidate(loop, true); - if (iff != NULL && is_heap_stable_test(iff)) { + !seen.test_set(head->_idx)) { + IfNode* iff = find_unswitching_candidate(loop, phase); + if (iff != NULL) { + Node* bol = iff->in(1); if (head->is_strip_mined()) { head->verify_strip_mined(0); - OuterStripMinedLoopNode* outer = head->as_CountedLoop()->outer_loop(); - OuterStripMinedLoopEndNode* le = head->outer_loop_end(); - Node* new_outer = new LoopNode(outer->in(LoopNode::EntryControl), outer->in(LoopNode::LoopBackControl)); - phase->register_control(new_outer, phase->get_loop(outer), outer->in(LoopNode::EntryControl)); - Node* new_le = new IfNode(le->in(0), le->in(1), le->_prob, le->_fcnt); - phase->register_control(new_le, phase->get_loop(le), le->in(0)); - phase->lazy_replace(outer, new_outer); - phase->lazy_replace(le, new_le); - head->clear_strip_mined(); } - phase->do_unswitching(loop, old_new, true); + move_heap_stable_test_out_of_loop(iff, phase); + if (loop->policy_unswitching(phase)) { + if (head->is_strip_mined()) { + OuterStripMinedLoopNode* outer = head->as_CountedLoop()->outer_loop(); + hide_strip_mined_loop(outer, head->as_CountedLoop(), phase); + } + phase->do_unswitching(loop, old_new); + } else { + // Not proceeding with unswitching. Move load back in + // the loop. + phase->igvn().replace_input_of(iff, 1, bol); + } } } } @@ -3134,7 +1969,7 @@ } #ifdef ASSERT -void ShenandoahBarrierNode::verify_raw_mem(RootNode* root) { +void ShenandoahBarrierC2Support::verify_raw_mem(RootNode* root) { const bool trace = false; ResourceMark rm; Unique_Node_List nodes; @@ -3149,15 +1984,13 @@ if (trace) { tty->print("XXXXXX verifying"); n->dump(); } for (uint next2 = 0; next2 < controls.size(); next2++) { Node *m = controls.at(next2); - if (!m->is_Loop() || controls.member(m->in(LoopNode::EntryControl)) || 1) { - for (DUIterator_Fast imax, i = m->fast_outs(imax); i < imax; i++) { - Node* u = m->fast_out(i); - if (u->is_CFG() && !u->is_Root() && - !(u->Opcode() == Op_CProj && u->in(0)->Opcode() == Op_NeverBranch && u->as_Proj()->_con == 1) && - !(u->is_Region() && u->unique_ctrl_out()->Opcode() == Op_Halt)) { - if (trace) { tty->print("XXXXXX pushing control"); u->dump(); } - controls.push(u); - } + for (DUIterator_Fast imax, i = m->fast_outs(imax); i < imax; i++) { + Node* u = m->fast_out(i); + if (u->is_CFG() && !u->is_Root() && + !(u->Opcode() == Op_CProj && u->in(0)->Opcode() == Op_NeverBranch && u->as_Proj()->_con == 1) && + !(u->is_Region() && u->unique_ctrl_out()->Opcode() == Op_Halt)) { + if (trace) { tty->print("XXXXXX pushing control"); u->dump(); } + controls.push(u); } } } @@ -3165,32 +1998,30 @@ for (uint next2 = 0; next2 < memories.size(); next2++) { Node *m = memories.at(next2); assert(m->bottom_type() == Type::MEMORY, ""); - if (!m->is_Phi() || !m->in(0)->is_Loop() || controls.member(m->in(0)->in(LoopNode::EntryControl)) || 1) { - for (DUIterator_Fast imax, i = m->fast_outs(imax); i < imax; i++) { - Node* u = m->fast_out(i); - if (u->bottom_type() == Type::MEMORY && (u->is_Mem() || u->is_ClearArray())) { - if (trace) { tty->print("XXXXXX pushing memory"); u->dump(); } - memories.push(u); - } else if (u->is_LoadStore()) { - if (trace) { tty->print("XXXXXX pushing memory"); u->find_out_with(Op_SCMemProj)->dump(); } - memories.push(u->find_out_with(Op_SCMemProj)); - } else if (u->is_MergeMem() && u->as_MergeMem()->memory_at(Compile::AliasIdxRaw) == m) { + for (DUIterator_Fast imax, i = m->fast_outs(imax); i < imax; i++) { + Node* u = m->fast_out(i); + if (u->bottom_type() == Type::MEMORY && (u->is_Mem() || u->is_ClearArray())) { + if (trace) { tty->print("XXXXXX pushing memory"); u->dump(); } + memories.push(u); + } else if (u->is_LoadStore()) { + if (trace) { tty->print("XXXXXX pushing memory"); u->find_out_with(Op_SCMemProj)->dump(); } + memories.push(u->find_out_with(Op_SCMemProj)); + } else if (u->is_MergeMem() && u->as_MergeMem()->memory_at(Compile::AliasIdxRaw) == m) { + if (trace) { tty->print("XXXXXX pushing memory"); u->dump(); } + memories.push(u); + } else if (u->is_Phi()) { + assert(u->bottom_type() == Type::MEMORY, ""); + if (u->adr_type() == TypeRawPtr::BOTTOM || u->adr_type() == TypePtr::BOTTOM) { + assert(controls.member(u->in(0)), ""); if (trace) { tty->print("XXXXXX pushing memory"); u->dump(); } memories.push(u); - } else if (u->is_Phi()) { - assert(u->bottom_type() == Type::MEMORY, ""); - if (u->adr_type() == TypeRawPtr::BOTTOM || u->adr_type() == TypePtr::BOTTOM) { - assert(controls.member(u->in(0)), ""); - if (trace) { tty->print("XXXXXX pushing memory"); u->dump(); } - memories.push(u); - } - } else if (u->is_SafePoint() || u->is_MemBar()) { - for (DUIterator_Fast jmax, j = u->fast_outs(jmax); j < jmax; j++) { - Node* uu = u->fast_out(j); - if (uu->bottom_type() == Type::MEMORY) { - if (trace) { tty->print("XXXXXX pushing memory"); uu->dump(); } - memories.push(uu); - } + } + } else if (u->is_SafePoint() || u->is_MemBar()) { + for (DUIterator_Fast jmax, j = u->fast_outs(jmax); j < jmax; j++) { + Node* uu = u->fast_out(j); + if (uu->bottom_type() == Type::MEMORY) { + if (trace) { tty->print("XXXXXX pushing memory"); uu->dump(); } + memories.push(uu); } } } @@ -3236,6 +2067,10 @@ } #endif +ShenandoahEnqueueBarrierNode::ShenandoahEnqueueBarrierNode(Node* val) : Node(NULL, val) { + ShenandoahBarrierSetC2::bsc2()->state()->add_enqueue_barrier(this); +} + const Type* ShenandoahEnqueueBarrierNode::bottom_type() const { if (in(1) == NULL || in(1)->is_top()) { return Type::TOP; @@ -3395,6 +2230,26 @@ Node* call = in->in(0)->in(0); assert(call->is_Call(), ""); mem = call->in(TypeFunc::Memory); + } else if (in->Opcode() == Op_NeverBranch) { + ResourceMark rm; + Unique_Node_List wq; + wq.push(in); + wq.push(in->as_Multi()->proj_out(0)); + for (uint j = 1; j < wq.size(); j++) { + Node* c = wq.at(j); + assert(!c->is_Root(), "shouldn't leave loop"); + if (c->is_SafePoint()) { + assert(mem == NULL, "only one safepoint"); + mem = c->in(TypeFunc::Memory); + } + for (DUIterator_Fast kmax, k = c->fast_outs(kmax); k < kmax; k++) { + Node* u = c->fast_out(k); + if (u->is_CFG()) { + wq.push(u); + } + } + } + assert(mem != NULL, "should have found safepoint"); } } } else { @@ -3433,12 +2288,6 @@ assert(_alias == Compile::AliasIdxRaw, ""); stack.push(mem, mem->req()); mem = mem->in(MemNode::Memory); - } else if (mem->Opcode() == Op_ShenandoahWriteBarrier) { - assert(_alias != Compile::AliasIdxRaw, ""); - mem = mem->in(ShenandoahBarrierNode::Memory); - } else if (mem->Opcode() == Op_ShenandoahWBMemProj) { - stack.push(mem, mem->req()); - mem = mem->in(ShenandoahWBMemProjNode::WriteBarrier); } else { #ifdef ASSERT mem->dump(); @@ -3492,7 +2341,7 @@ while (progress) { progress = false; iteration++; - assert(iteration <= 2+max_depth || _phase->C->has_irreducible_loop(), ""); + assert(iteration <= 2+max_depth || _phase->C->has_irreducible_loop() || has_never_branch(_phase->C->root()), ""); if (trace) { tty->print_cr("XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"); } IdealLoopTree* last_updated_ilt = NULL; for (int i = rpo_list.size() - 1; i >= 0; i--) { @@ -3660,7 +2509,7 @@ mem = _memory_nodes[c->_idx]; } if (n != NULL && mem_is_valid(mem, c)) { - while (!ShenandoahWriteBarrierNode::is_dominator_same_ctrl(c, mem, n, _phase) && _phase->ctrl_or_self(mem) == ctrl) { + while (!ShenandoahBarrierC2Support::is_dominator_same_ctrl(c, mem, n, _phase) && _phase->ctrl_or_self(mem) == ctrl) { mem = next_mem(mem, _alias); } if (mem->is_MergeMem()) { @@ -3706,12 +2555,6 @@ } else if (old->Opcode() == Op_SCMemProj) { assert(_alias == Compile::AliasIdxRaw, ""); old = old->in(0); - } else if (old->Opcode() == Op_ShenandoahWBMemProj) { - assert(_alias != Compile::AliasIdxRaw, ""); - old = old->in(ShenandoahWBMemProjNode::WriteBarrier); - } else if (old->Opcode() == Op_ShenandoahWriteBarrier) { - assert(_alias != Compile::AliasIdxRaw, ""); - old = old->in(ShenandoahBarrierNode::Memory); } else { ShouldNotReachHere(); } @@ -3721,7 +2564,7 @@ _memory_nodes.map(ctrl->_idx, mem); _memory_nodes.map(new_ctrl->_idx, mem_for_ctrl); } - uint input = prev->Opcode() == Op_ShenandoahWriteBarrier ? (uint)ShenandoahBarrierNode::Memory : (uint)MemNode::Memory; + uint input = (uint)MemNode::Memory; _phase->igvn().replace_input_of(prev, input, new_mem); } else { uses.clear(); @@ -3789,19 +2632,14 @@ } else { DEBUG_ONLY(if (trace) { tty->print("ZZZ NOT setting mem"); m->dump(); }); for (;;) { - assert(m->is_Mem() || m->is_LoadStore() || m->is_Proj() || m->Opcode() == Op_ShenandoahWriteBarrier || m->Opcode() == Op_ShenandoahWBMemProj, ""); + assert(m->is_Mem() || m->is_LoadStore() || m->is_Proj(), ""); Node* next = NULL; if (m->is_Proj()) { next = m->in(0); - } else if (m->Opcode() == Op_ShenandoahWBMemProj) { - next = m->in(ShenandoahWBMemProjNode::WriteBarrier); - } else if (m->is_Mem() || m->is_LoadStore()) { + } else { + assert(m->is_Mem() || m->is_LoadStore(), ""); assert(_alias == Compile::AliasIdxRaw, ""); next = m->in(MemNode::Memory); - } else { - assert(_alias != Compile::AliasIdxRaw, ""); - assert (m->Opcode() == Op_ShenandoahWriteBarrier, ""); - next = m->in(ShenandoahBarrierNode::Memory); } if (_phase->get_ctrl(next) != u) { break; @@ -3818,8 +2656,8 @@ } DEBUG_ONLY(if (trace) { tty->print("ZZZ setting to phi"); m->dump(); }); - assert(m->is_Mem() || m->is_LoadStore() || m->Opcode() == Op_ShenandoahWriteBarrier, ""); - uint input = (m->is_Mem() || m->is_LoadStore()) ? (uint)MemNode::Memory : (uint)ShenandoahBarrierNode::Memory; + assert(m->is_Mem() || m->is_LoadStore(), ""); + uint input = (uint)MemNode::Memory; _phase->igvn().replace_input_of(m, input, phi); push = false; } @@ -4045,20 +2883,7 @@ for (DUIterator i = mem->outs(); mem->has_out(i); i++) { Node* u = mem->out(i); if (u != replacement && u->_idx < last) { - if (u->is_ShenandoahBarrier() && _alias != Compile::AliasIdxRaw) { - if (_phase->C->get_alias_index(u->adr_type()) == _alias && ShenandoahWriteBarrierNode::is_dominator(rep_ctrl, _phase->ctrl_or_self(u), replacement, u, _phase)) { - _phase->igvn().replace_input_of(u, u->find_edge(mem), rep_proj); - assert(u->find_edge(mem) == -1, "only one edge"); - --i; - } - } else if (u->is_Mem()) { - if (_phase->C->get_alias_index(u->adr_type()) == _alias && ShenandoahWriteBarrierNode::is_dominator(rep_ctrl, _phase->ctrl_or_self(u), replacement, u, _phase)) { - assert(_alias == Compile::AliasIdxRaw , "only raw memory can lead to a memory operation"); - _phase->igvn().replace_input_of(u, u->find_edge(mem), rep_proj); - assert(u->find_edge(mem) == -1, "only one edge"); - --i; - } - } else if (u->is_MergeMem()) { + if (u->is_MergeMem()) { MergeMemNode* u_mm = u->as_MergeMem(); if (u_mm->memory_at(_alias) == mem) { MergeMemNode* newmm = NULL; @@ -4086,7 +2911,7 @@ } } } else { - if (rep_ctrl != uu && ShenandoahWriteBarrierNode::is_dominator(rep_ctrl, _phase->ctrl_or_self(uu), replacement, uu, _phase)) { + if (rep_ctrl != uu && ShenandoahBarrierC2Support::is_dominator(rep_ctrl, _phase->ctrl_or_self(uu), replacement, uu, _phase)) { if (newmm == NULL) { newmm = clone_merge_mem(u, mem, rep_proj, rep_ctrl, i); } @@ -4127,10 +2952,11 @@ u->Opcode() == Op_Rethrow || u->Opcode() == Op_Return || u->Opcode() == Op_SafePoint || + u->Opcode() == Op_StoreLConditional || (u->is_CallStaticJava() && u->as_CallStaticJava()->uncommon_trap_request() != 0) || (u->is_CallStaticJava() && u->as_CallStaticJava()->_entry_point == OptoRuntime::rethrow_stub()) || u->Opcode() == Op_CallLeaf, ""); - if (ShenandoahWriteBarrierNode::is_dominator(rep_ctrl, _phase->ctrl_or_self(u), replacement, u, _phase)) { + if (ShenandoahBarrierC2Support::is_dominator(rep_ctrl, _phase->ctrl_or_self(u), replacement, u, _phase)) { if (mm == NULL) { mm = allocate_merge_mem(mem, rep_proj, rep_ctrl); } @@ -4138,7 +2964,7 @@ --i; } } else if (_phase->C->get_alias_index(u->adr_type()) == _alias) { - if (ShenandoahWriteBarrierNode::is_dominator(rep_ctrl, _phase->ctrl_or_self(u), replacement, u, _phase)) { + if (ShenandoahBarrierC2Support::is_dominator(rep_ctrl, _phase->ctrl_or_self(u), replacement, u, _phase)) { _phase->igvn().replace_input_of(u, u->find_edge(mem), rep_proj); --i; } @@ -4147,12 +2973,322 @@ } } -void MemoryGraphFixer::remove(Node* n) { - assert(n->Opcode() == Op_ShenandoahWBMemProj, ""); - Node* c = _phase->get_ctrl(n); - Node* mem = find_mem(c, NULL); - if (mem == n) { - _memory_nodes.map(c->_idx, mem->in(ShenandoahWBMemProjNode::WriteBarrier)->in(ShenandoahBarrierNode::Memory)); +ShenandoahLoadReferenceBarrierNode::ShenandoahLoadReferenceBarrierNode(Node* ctrl, Node* obj) +: Node(ctrl, obj) { + ShenandoahBarrierSetC2::bsc2()->state()->add_load_reference_barrier(this); +} + +const Type* ShenandoahLoadReferenceBarrierNode::bottom_type() const { + if (in(ValueIn) == NULL || in(ValueIn)->is_top()) { + return Type::TOP; + } + const Type* t = in(ValueIn)->bottom_type(); + if (t == TypePtr::NULL_PTR) { + return t; + } + return t->is_oopptr(); +} + +const Type* ShenandoahLoadReferenceBarrierNode::Value(PhaseGVN* phase) const { + // Either input is TOP ==> the result is TOP + const Type *t2 = phase->type(in(ValueIn)); + if( t2 == Type::TOP ) return Type::TOP; + + if (t2 == TypePtr::NULL_PTR) { + return t2; + } + + const Type* type = t2->is_oopptr()/*->cast_to_nonconst()*/; + return type; +} + +Node* ShenandoahLoadReferenceBarrierNode::Identity(PhaseGVN* phase) { + Node* value = in(ValueIn); + if (!needs_barrier(phase, value)) { + return value; + } + return this; +} + +bool ShenandoahLoadReferenceBarrierNode::needs_barrier(PhaseGVN* phase, Node* n) { + Unique_Node_List visited; + return needs_barrier_impl(phase, n, visited); +} + +bool ShenandoahLoadReferenceBarrierNode::needs_barrier_impl(PhaseGVN* phase, Node* n, Unique_Node_List &visited) { + if (n == NULL) return false; + if (visited.member(n)) { + return false; // Been there. + } + visited.push(n); + + if (n->is_Allocate()) { + // tty->print_cr("optimize barrier on alloc"); + return false; + } + if (n->is_Call()) { + // tty->print_cr("optimize barrier on call"); + return false; } + + const Type* type = phase->type(n); + if (type == Type::TOP) { + return false; + } + if (type->make_ptr()->higher_equal(TypePtr::NULL_PTR)) { + // tty->print_cr("optimize barrier on null"); + return false; + } + if (type->make_oopptr() && type->make_oopptr()->const_oop() != NULL) { + // tty->print_cr("optimize barrier on constant"); + return false; + } + + switch (n->Opcode()) { + case Op_AddP: + return true; // TODO: Can refine? + case Op_LoadP: + case Op_ShenandoahCompareAndExchangeN: + case Op_ShenandoahCompareAndExchangeP: + case Op_CompareAndExchangeN: + case Op_CompareAndExchangeP: + case Op_GetAndSetN: + case Op_GetAndSetP: + return true; + case Op_Phi: { + for (uint i = 1; i < n->req(); i++) { + if (needs_barrier_impl(phase, n->in(i), visited)) return true; + } + return false; + } + case Op_CheckCastPP: + case Op_CastPP: + return needs_barrier_impl(phase, n->in(1), visited); + case Op_Proj: + return needs_barrier_impl(phase, n->in(0), visited); + case Op_ShenandoahLoadReferenceBarrier: + // tty->print_cr("optimize barrier on barrier"); + return false; + case Op_Parm: + // tty->print_cr("optimize barrier on input arg"); + return false; + case Op_DecodeN: + case Op_EncodeP: + return needs_barrier_impl(phase, n->in(1), visited); + case Op_LoadN: + return true; + case Op_CMoveP: + return needs_barrier_impl(phase, n->in(2), visited) || + needs_barrier_impl(phase, n->in(3), visited); + case Op_ShenandoahEnqueueBarrier: + return needs_barrier_impl(phase, n->in(1), visited); + default: + break; + } +#ifdef ASSERT + tty->print("need barrier on?: "); + tty->print_cr("ins:"); + n->dump(2); + tty->print_cr("outs:"); + n->dump(-2); + ShouldNotReachHere(); +#endif + return true; } +ShenandoahLoadReferenceBarrierNode::Strength ShenandoahLoadReferenceBarrierNode::get_barrier_strength() { + Unique_Node_List visited; + Node_Stack stack(0); + stack.push(this, 0); + Strength strength = NONE; + while (strength != STRONG && stack.size() > 0) { + Node* n = stack.node(); + if (visited.member(n)) { + stack.pop(); + continue; + } + visited.push(n); + bool visit_users = false; + switch (n->Opcode()) { + case Op_StoreN: + case Op_StoreP: { + strength = STRONG; + break; + } + case Op_CmpP: { + if (!n->in(1)->bottom_type()->higher_equal(TypePtr::NULL_PTR) && + !n->in(2)->bottom_type()->higher_equal(TypePtr::NULL_PTR)) { + strength = STRONG; + } + break; + } + case Op_CallStaticJava: { + strength = STRONG; + break; + } + case Op_CallDynamicJava: + case Op_CallLeaf: + case Op_CallLeafNoFP: + case Op_CompareAndSwapL: + case Op_CompareAndSwapI: + case Op_CompareAndSwapB: + case Op_CompareAndSwapS: + case Op_ShenandoahCompareAndSwapN: + case Op_ShenandoahCompareAndSwapP: + case Op_ShenandoahWeakCompareAndSwapN: + case Op_ShenandoahWeakCompareAndSwapP: + case Op_ShenandoahCompareAndExchangeN: + case Op_ShenandoahCompareAndExchangeP: + case Op_CompareAndExchangeL: + case Op_CompareAndExchangeI: + case Op_CompareAndExchangeB: + case Op_CompareAndExchangeS: + case Op_WeakCompareAndSwapL: + case Op_WeakCompareAndSwapI: + case Op_WeakCompareAndSwapB: + case Op_WeakCompareAndSwapS: + case Op_GetAndSetL: + case Op_GetAndSetI: + case Op_GetAndSetB: + case Op_GetAndSetS: + case Op_GetAndSetP: + case Op_GetAndSetN: + case Op_GetAndAddL: + case Op_GetAndAddI: + case Op_GetAndAddB: + case Op_GetAndAddS: + case Op_ShenandoahEnqueueBarrier: + case Op_FastLock: + case Op_FastUnlock: + case Op_Rethrow: + case Op_Return: + case Op_StoreB: + case Op_StoreC: + case Op_StoreD: + case Op_StoreF: + case Op_StoreL: + case Op_StoreLConditional: + case Op_StoreI: + case Op_StoreVector: + case Op_StrInflatedCopy: + case Op_StrCompressedCopy: + case Op_EncodeP: + case Op_CastP2X: + case Op_SafePoint: + case Op_EncodeISOArray: + strength = STRONG; + break; + case Op_LoadB: + case Op_LoadUB: + case Op_LoadUS: + case Op_LoadD: + case Op_LoadF: + case Op_LoadL: + case Op_LoadI: + case Op_LoadS: + case Op_LoadN: + case Op_LoadP: + case Op_LoadVector: { + const TypePtr* adr_type = n->adr_type(); + int alias_idx = Compile::current()->get_alias_index(adr_type); + Compile::AliasType* alias_type = Compile::current()->alias_type(alias_idx); + ciField* field = alias_type->field(); + bool is_static = field != NULL && field->is_static(); + bool is_final = field != NULL && field->is_final(); + bool is_stable = field != NULL && field->is_stable(); + if (ShenandoahOptimizeStaticFinals && is_static && is_final) { + // Leave strength as is. + } else if (ShenandoahOptimizeInstanceFinals && !is_static && is_final) { + // Leave strength as is. + } else if (ShenandoahOptimizeStableFinals && (is_stable || (adr_type->isa_aryptr() && adr_type->isa_aryptr()->is_stable()))) { + // Leave strength as is. + } else { + strength = WEAK; + } + break; + } + case Op_AryEq: { + Node* n1 = n->in(2); + Node* n2 = n->in(3); + if (!ShenandoahOptimizeStableFinals || + !n1->bottom_type()->isa_aryptr() || !n1->bottom_type()->isa_aryptr()->is_stable() || + !n2->bottom_type()->isa_aryptr() || !n2->bottom_type()->isa_aryptr()->is_stable()) { + strength = WEAK; + } + break; + } + case Op_StrEquals: + case Op_StrComp: + case Op_StrIndexOf: + case Op_StrIndexOfChar: + if (!ShenandoahOptimizeStableFinals) { + strength = WEAK; + } + break; + case Op_Conv2B: + case Op_LoadRange: + case Op_LoadKlass: + case Op_LoadNKlass: + // NONE, i.e. leave current strength as is + break; + case Op_AddP: + case Op_CheckCastPP: + case Op_CastPP: + case Op_CMoveP: + case Op_Phi: + case Op_ShenandoahLoadReferenceBarrier: + visit_users = true; + break; + default: { +#ifdef ASSERT + tty->print_cr("Unknown node in get_barrier_strength:"); + n->dump(1); + ShouldNotReachHere(); +#else + strength = STRONG; +#endif + } + } +#ifdef ASSERT +/* + if (strength == STRONG) { + tty->print("strengthening node: "); + n->dump(); + } + */ +#endif + stack.pop(); + if (visit_users) { + for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) { + Node* user = n->fast_out(i); + if (user != NULL) { + stack.push(user, 0); + } + } + } + } + return strength; +} + +CallStaticJavaNode* ShenandoahLoadReferenceBarrierNode::pin_and_expand_null_check(PhaseIterGVN& igvn) { + Node* val = in(ValueIn); + + const Type* val_t = igvn.type(val); + + if (val_t->meet(TypePtr::NULL_PTR) != val_t && + val->Opcode() == Op_CastPP && + val->in(0) != NULL && + val->in(0)->Opcode() == Op_IfTrue && + val->in(0)->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none) && + val->in(0)->in(0)->is_If() && + val->in(0)->in(0)->in(1)->Opcode() == Op_Bool && + val->in(0)->in(0)->in(1)->as_Bool()->_test._test == BoolTest::ne && + val->in(0)->in(0)->in(1)->in(1)->Opcode() == Op_CmpP && + val->in(0)->in(0)->in(1)->in(1)->in(1) == val->in(1) && + val->in(0)->in(0)->in(1)->in(1)->in(2)->bottom_type() == TypePtr::NULL_PTR) { + assert(val->in(0)->in(0)->in(1)->in(1)->in(1) == val->in(1), ""); + CallStaticJavaNode* unc = val->in(0)->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none); + return unc; + } + return NULL; +} diff --git a/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.hpp b/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.hpp --- a/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.hpp +++ b/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.hpp @@ -21,8 +21,8 @@ * */ -#ifndef SHARE_VM_GC_SHENANDOAH_C2_SHENANDOAH_SUPPORT_HPP -#define SHARE_VM_GC_SHENANDOAH_C2_SHENANDOAH_SUPPORT_HPP +#ifndef SHARE_GC_SHENANDOAH_C2_SHENANDOAHSUPPORT_HPP +#define SHARE_GC_SHENANDOAH_C2_SHENANDOAHSUPPORT_HPP #include "gc/shenandoah/shenandoahBrooksPointer.hpp" #include "memory/allocation.hpp" @@ -36,215 +36,61 @@ class PhaseGVN; class MemoryGraphFixer; -class ShenandoahBarrierNode : public TypeNode { +class ShenandoahBarrierC2Support : public AllStatic { private: - bool _allow_fromspace; - #ifdef ASSERT enum verify_type { ShenandoahLoad, ShenandoahStore, ShenandoahValue, ShenandoahOopStore, - ShenandoahNone + ShenandoahNone, }; static bool verify_helper(Node* in, Node_Stack& phis, VectorSet& visited, verify_type t, bool trace, Unique_Node_List& barriers_used); -#endif - -public: - enum { Control, - Memory, - ValueIn - }; - - ShenandoahBarrierNode(Node* ctrl, Node* mem, Node* obj, bool allow_fromspace) - : TypeNode(obj->bottom_type()->isa_oopptr() ? obj->bottom_type()->is_oopptr()->cast_to_nonconst() : obj->bottom_type(), 3), - _allow_fromspace(allow_fromspace) { - - init_req(Control, ctrl); - init_req(Memory, mem); - init_req(ValueIn, obj); - - init_class_id(Class_ShenandoahBarrier); - } - - static Node* skip_through_barrier(Node* n); - - static const TypeOopPtr* brooks_pointer_type(const Type* t) { - return t->is_oopptr()->cast_to_nonconst()->add_offset(ShenandoahBrooksPointer::byte_offset())->is_oopptr(); - } - - virtual const TypePtr* adr_type() const { - if (bottom_type() == Type::TOP) { - return NULL; - } - //const TypePtr* adr_type = in(MemNode::Address)->bottom_type()->is_ptr(); - const TypePtr* adr_type = brooks_pointer_type(bottom_type()); - assert(adr_type->offset() == ShenandoahBrooksPointer::byte_offset(), "sane offset"); - assert(Compile::current()->alias_type(adr_type)->is_rewritable(), "brooks ptr must be rewritable"); - return adr_type; - } - - virtual uint ideal_reg() const { return Op_RegP; } - virtual uint match_edge(uint idx) const { - return idx >= ValueIn; - } - - Node* Identity_impl(PhaseGVN* phase); - - virtual const Type* Value(PhaseGVN* phase) const; - virtual bool depends_only_on_test() const { - return true; - }; - - static bool needs_barrier(PhaseGVN* phase, ShenandoahBarrierNode* orig, Node* n, Node* rb_mem, bool allow_fromspace); - -#ifdef ASSERT static void report_verify_failure(const char* msg, Node* n1 = NULL, Node* n2 = NULL); - static void verify(RootNode* root); static void verify_raw_mem(RootNode* root); #endif -#ifndef PRODUCT - virtual void dump_spec(outputStream *st) const; -#endif - - // protected: - static Node* dom_mem(Node* mem, Node*& mem_ctrl, Node* n, Node* rep_ctrl, int alias, PhaseIdealLoop* phase); static Node* dom_mem(Node* mem, Node* ctrl, int alias, Node*& mem_ctrl, PhaseIdealLoop* phase); - static bool is_dominator(Node *d_c, Node *n_c, Node* d, Node* n, PhaseIdealLoop* phase); - static bool is_dominator_same_ctrl(Node* c, Node* d, Node* n, PhaseIdealLoop* phase); static Node* no_branches(Node* c, Node* dom, bool allow_one_proj, PhaseIdealLoop* phase); - -protected: - uint hash() const; - uint cmp(const Node& n) const; - uint size_of() const; - -private: - static bool needs_barrier_impl(PhaseGVN* phase, ShenandoahBarrierNode* orig, Node* n, Node* rb_mem, bool allow_fromspace, Unique_Node_List &visited); - - static bool dominates_memory(PhaseGVN* phase, Node* b1, Node* b2, bool linear); - static bool dominates_memory_impl(PhaseGVN* phase, Node* b1, Node* b2, Node* current, bool linear); -}; - -class ShenandoahReadBarrierNode : public ShenandoahBarrierNode { -public: - ShenandoahReadBarrierNode(Node* ctrl, Node* mem, Node* obj) - : ShenandoahBarrierNode(ctrl, mem, obj, true) { - assert(UseShenandoahGC && (ShenandoahReadBarrier || ShenandoahStoreValReadBarrier || - ShenandoahWriteBarrier || ShenandoahAcmpBarrier), - "should be enabled"); - } - ShenandoahReadBarrierNode(Node* ctrl, Node* mem, Node* obj, bool allow_fromspace) - : ShenandoahBarrierNode(ctrl, mem, obj, allow_fromspace) { - assert(UseShenandoahGC && (ShenandoahReadBarrier || ShenandoahStoreValReadBarrier || - ShenandoahWriteBarrier || ShenandoahAcmpBarrier), - "should be enabled"); - } - - virtual Node *Ideal(PhaseGVN *phase, bool can_reshape); - virtual Node* Identity(PhaseGVN* phase); - virtual int Opcode() const; - - bool is_independent(Node* mem); - - void try_move(Node *n_ctrl, PhaseIdealLoop* phase); - -private: - static bool is_independent(const Type* in_type, const Type* this_type); - static bool dominates_memory_rb(PhaseGVN* phase, Node* b1, Node* b2, bool linear); - static bool dominates_memory_rb_impl(PhaseGVN* phase, Node* b1, Node* b2, Node* current, bool linear); -}; - -class ShenandoahWriteBarrierNode : public ShenandoahBarrierNode { -public: - ShenandoahWriteBarrierNode(Compile* C, Node* ctrl, Node* mem, Node* obj); - - virtual int Opcode() const; - virtual Node *Ideal(PhaseGVN *phase, bool can_reshape); - virtual Node* Identity(PhaseGVN* phase); - virtual bool depends_only_on_test() const { return false; } - - static bool expand(Compile* C, PhaseIterGVN& igvn, int& loop_opts_cnt); - static bool is_gc_state_load(Node *n); static bool is_heap_state_test(Node* iff, int mask); - static bool is_heap_stable_test(Node* iff); static bool try_common_gc_state_load(Node *n, PhaseIdealLoop *phase); static bool has_safepoint_between(Node* start, Node* stop, PhaseIdealLoop *phase); - - static LoopNode* try_move_before_pre_loop(Node* c, Node* val_ctrl, PhaseIdealLoop* phase); - static Node* move_above_predicates(LoopNode* cl, Node* val_ctrl, PhaseIdealLoop* phase); -#ifdef ASSERT - static bool memory_dominates_all_paths(Node* mem, Node* rep_ctrl, int alias, PhaseIdealLoop* phase); - static void memory_dominates_all_paths_helper(Node* c, Node* rep_ctrl, Unique_Node_List& controls, PhaseIdealLoop* phase); -#endif - void try_move_before_loop(GrowableArray& memory_graph_fixers, PhaseIdealLoop* phase, bool include_lsm, Unique_Node_List& uses); - void try_move_before_loop_helper(LoopNode* cl, Node* val_ctrl, GrowableArray& memory_graph_fixers, PhaseIdealLoop* phase, bool include_lsm, Unique_Node_List& uses); - static void pin_and_expand(PhaseIdealLoop* phase); - CallStaticJavaNode* pin_and_expand_null_check(PhaseIterGVN& igvn); - void pin_and_expand_move_barrier(PhaseIdealLoop* phase, GrowableArray& memory_graph_fixers, Unique_Node_List& uses); - void pin_and_expand_helper(PhaseIdealLoop* phase); static Node* find_bottom_mem(Node* ctrl, PhaseIdealLoop* phase); static void follow_barrier_uses(Node* n, Node* ctrl, Unique_Node_List& uses, PhaseIdealLoop* phase); static void test_null(Node*& ctrl, Node* val, Node*& null_ctrl, PhaseIdealLoop* phase); - static void test_heap_stable(Node*& ctrl, Node* raw_mem, Node*& heap_stable_ctrl, PhaseIdealLoop* phase); - static void call_wb_stub(Node*& ctrl, Node*& val, Node*& result_mem, - Node* raw_mem, Node* wb_mem, int alias, - PhaseIdealLoop* phase); - static void heap_stable(Node* c, Node* val, Node* unc_ctrl, Node* raw_mem, Node* wb_mem, Node* region, - Node* val_phi, Node* mem_phi, Node* raw_mem_phi, Node* unc_region, PhaseIdealLoop* phase); + static void call_lrb_stub(Node*& ctrl, Node*& val, Node*& result_mem, Node* raw_mem, PhaseIdealLoop* phase); static Node* clone_null_check(Node*& c, Node* val, Node* unc_ctrl, PhaseIdealLoop* phase); static void fix_null_check(Node* unc, Node* unc_ctrl, Node* new_unc_ctrl, Unique_Node_List& uses, PhaseIdealLoop* phase); static void in_cset_fast_test(Node*& ctrl, Node*& not_cset_ctrl, Node* val, Node* raw_mem, PhaseIdealLoop* phase); static void move_heap_stable_test_out_of_loop(IfNode* iff, PhaseIdealLoop* phase); - - static void optimize_after_expansion(VectorSet &visited, Node_Stack &nstack, Node_List &old_new, PhaseIdealLoop* phase); static void merge_back_to_back_tests(Node* n, PhaseIdealLoop* phase); static bool identical_backtoback_ifs(Node *n, PhaseIdealLoop* phase); static void fix_ctrl(Node* barrier, Node* region, const MemoryGraphFixer& fixer, Unique_Node_List& uses, Unique_Node_List& uses_to_ignore, uint last, PhaseIdealLoop* phase); + static IfNode* find_unswitching_candidate(const IdealLoopTree *loop, PhaseIdealLoop* phase); - static void optimize_before_expansion(PhaseIdealLoop* phase, GrowableArray memory_graph_fixers, bool include_lsm); - Node* would_subsume(ShenandoahBarrierNode* other, PhaseIdealLoop* phase); -}; - -class ShenandoahWBMemProjNode : public Node { public: - enum { Control, - WriteBarrier }; - - ShenandoahWBMemProjNode(Node *src) : Node(NULL, src) { - assert(UseShenandoahGC && ShenandoahWriteBarrier, "should be enabled"); - assert(src->Opcode() == Op_ShenandoahWriteBarrier || src->is_Mach(), "epxect wb"); - } - virtual Node* Identity(PhaseGVN* phase); + static bool is_dominator(Node* d_c, Node* n_c, Node* d, Node* n, PhaseIdealLoop* phase); + static bool is_dominator_same_ctrl(Node* c, Node* d, Node* n, PhaseIdealLoop* phase); - virtual int Opcode() const; - virtual bool is_CFG() const { return false; } - virtual const Type *bottom_type() const {return Type::MEMORY;} - virtual const TypePtr *adr_type() const { - Node* wb = in(WriteBarrier); - if (wb == NULL || wb->is_top()) return NULL; // node is dead - assert(wb->Opcode() == Op_ShenandoahWriteBarrier || (wb->is_Mach() && wb->as_Mach()->ideal_Opcode() == Op_ShenandoahWriteBarrier) || wb->is_Phi(), "expect wb"); - return ShenandoahBarrierNode::brooks_pointer_type(wb->bottom_type()); - } + static bool is_gc_state_load(Node* n); + static bool is_heap_stable_test(Node* iff); - virtual uint ideal_reg() const { return 0;} // memory projections don't have a register - virtual const Type *Value(PhaseGVN* phase ) const { - return bottom_type(); - } -#ifndef PRODUCT - virtual void dump_spec(outputStream *st) const {}; + static bool expand(Compile* C, PhaseIterGVN& igvn); + static void pin_and_expand(PhaseIdealLoop* phase); + static void optimize_after_expansion(VectorSet& visited, Node_Stack& nstack, Node_List& old_new, PhaseIdealLoop* phase); + +#ifdef ASSERT + static void verify(RootNode* root); #endif }; class ShenandoahEnqueueBarrierNode : public Node { public: - ShenandoahEnqueueBarrierNode(Node* val) : Node(NULL, val) { - } + ShenandoahEnqueueBarrierNode(Node* val); const Type *bottom_type() const; const Type* Value(PhaseGVN* phase) const; @@ -286,7 +132,133 @@ Node* find_mem(Node* ctrl, Node* n) const; void fix_mem(Node* ctrl, Node* region, Node* mem, Node* mem_for_ctrl, Node* mem_phi, Unique_Node_List& uses); int alias() const { return _alias; } - void remove(Node* n); +}; + +class ShenandoahCompareAndSwapPNode : public CompareAndSwapPNode { +public: + ShenandoahCompareAndSwapPNode(Node *c, Node *mem, Node *adr, Node *val, Node *ex, MemNode::MemOrd mem_ord) + : CompareAndSwapPNode(c, mem, adr, val, ex, mem_ord) { } + + virtual Node *Ideal(PhaseGVN *phase, bool can_reshape) { + if (in(ExpectedIn) != NULL && phase->type(in(ExpectedIn)) == TypePtr::NULL_PTR) { + return new CompareAndSwapPNode(in(MemNode::Control), in(MemNode::Memory), in(MemNode::Address), in(MemNode::ValueIn), in(ExpectedIn), order()); + } + return NULL; + } + + virtual int Opcode() const; +}; + +class ShenandoahCompareAndSwapNNode : public CompareAndSwapNNode { +public: + ShenandoahCompareAndSwapNNode(Node *c, Node *mem, Node *adr, Node *val, Node *ex, MemNode::MemOrd mem_ord) + : CompareAndSwapNNode(c, mem, adr, val, ex, mem_ord) { } + + virtual Node *Ideal(PhaseGVN *phase, bool can_reshape) { + if (in(ExpectedIn) != NULL && phase->type(in(ExpectedIn)) == TypeNarrowOop::NULL_PTR) { + return new CompareAndSwapNNode(in(MemNode::Control), in(MemNode::Memory), in(MemNode::Address), in(MemNode::ValueIn), in(ExpectedIn), order()); + } + return NULL; + } + + virtual int Opcode() const; +}; + +class ShenandoahWeakCompareAndSwapPNode : public WeakCompareAndSwapPNode { +public: + ShenandoahWeakCompareAndSwapPNode(Node *c, Node *mem, Node *adr, Node *val, Node *ex, MemNode::MemOrd mem_ord) + : WeakCompareAndSwapPNode(c, mem, adr, val, ex, mem_ord) { } + + virtual Node *Ideal(PhaseGVN *phase, bool can_reshape) { + if (in(ExpectedIn) != NULL && phase->type(in(ExpectedIn)) == TypePtr::NULL_PTR) { + return new WeakCompareAndSwapPNode(in(MemNode::Control), in(MemNode::Memory), in(MemNode::Address), in(MemNode::ValueIn), in(ExpectedIn), order()); + } + return NULL; + } + + virtual int Opcode() const; +}; + +class ShenandoahWeakCompareAndSwapNNode : public WeakCompareAndSwapNNode { +public: + ShenandoahWeakCompareAndSwapNNode(Node *c, Node *mem, Node *adr, Node *val, Node *ex, MemNode::MemOrd mem_ord) + : WeakCompareAndSwapNNode(c, mem, adr, val, ex, mem_ord) { } + + virtual Node *Ideal(PhaseGVN *phase, bool can_reshape) { + if (in(ExpectedIn) != NULL && phase->type(in(ExpectedIn)) == TypeNarrowOop::NULL_PTR) { + return new WeakCompareAndSwapNNode(in(MemNode::Control), in(MemNode::Memory), in(MemNode::Address), in(MemNode::ValueIn), in(ExpectedIn), order()); + } + return NULL; + } + + virtual int Opcode() const; }; -#endif // SHARE_VM_GC_SHENANDOAH_C2_SHENANDOAH_SUPPORT_HPP +class ShenandoahCompareAndExchangePNode : public CompareAndExchangePNode { +public: + ShenandoahCompareAndExchangePNode(Node *c, Node *mem, Node *adr, Node *val, Node *ex, const TypePtr* at, const Type* t, MemNode::MemOrd mem_ord) + : CompareAndExchangePNode(c, mem, adr, val, ex, at, t, mem_ord) { } + + virtual Node *Ideal(PhaseGVN *phase, bool can_reshape) { + if (in(ExpectedIn) != NULL && phase->type(in(ExpectedIn)) == TypePtr::NULL_PTR) { + return new CompareAndExchangePNode(in(MemNode::Control), in(MemNode::Memory), in(MemNode::Address), in(MemNode::ValueIn), in(ExpectedIn), adr_type(), bottom_type(), order()); + } + return NULL; + } + + virtual int Opcode() const; +}; + +class ShenandoahCompareAndExchangeNNode : public CompareAndExchangeNNode { +public: + ShenandoahCompareAndExchangeNNode(Node *c, Node *mem, Node *adr, Node *val, Node *ex, const TypePtr* at, const Type* t, MemNode::MemOrd mem_ord) + : CompareAndExchangeNNode(c, mem, adr, val, ex, at, t, mem_ord) { } + + virtual Node *Ideal(PhaseGVN *phase, bool can_reshape) { + if (in(ExpectedIn) != NULL && phase->type(in(ExpectedIn)) == TypeNarrowOop::NULL_PTR) { + return new CompareAndExchangeNNode(in(MemNode::Control), in(MemNode::Memory), in(MemNode::Address), in(MemNode::ValueIn), in(ExpectedIn), adr_type(), bottom_type(), order()); + } + return NULL; + } + + virtual int Opcode() const; +}; + +class ShenandoahLoadReferenceBarrierNode : public Node { +public: + enum { + Control, + ValueIn + }; + + enum Strength { + NONE, WEAK, STRONG, NA + }; + + ShenandoahLoadReferenceBarrierNode(Node* ctrl, Node* val); + + virtual int Opcode() const; + virtual const Type* bottom_type() const; + virtual const Type* Value(PhaseGVN* phase) const; + virtual const class TypePtr *adr_type() const { return TypeOopPtr::BOTTOM; } + virtual uint match_edge(uint idx) const { + return idx >= ValueIn; + } + virtual uint ideal_reg() const { return Op_RegP; } + + virtual Node* Identity(PhaseGVN* phase); + + uint size_of() const { + return sizeof(*this); + } + + Strength get_barrier_strength(); + CallStaticJavaNode* pin_and_expand_null_check(PhaseIterGVN& igvn); + +private: + bool needs_barrier(PhaseGVN* phase, Node* n); + bool needs_barrier_impl(PhaseGVN* phase, Node* n, Unique_Node_List &visited); +}; + + +#endif // SHARE_GC_SHENANDOAH_C2_SHENANDOAHSUPPORT_HPP diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp @@ -41,13 +41,10 @@ SHENANDOAH_ERGO_ENABLE_FLAG(ShenandoahImplicitGCInvokesConcurrent); // Final configuration checks + SHENANDOAH_CHECK_FLAG_SET(ShenandoahLoadRefBarrier); SHENANDOAH_CHECK_FLAG_SET(ShenandoahSATBBarrier); - SHENANDOAH_CHECK_FLAG_SET(ShenandoahReadBarrier); - SHENANDOAH_CHECK_FLAG_SET(ShenandoahWriteBarrier); - SHENANDOAH_CHECK_FLAG_SET(ShenandoahStoreValReadBarrier); SHENANDOAH_CHECK_FLAG_SET(ShenandoahKeepAliveBarrier); SHENANDOAH_CHECK_FLAG_SET(ShenandoahCASBarrier); - SHENANDOAH_CHECK_FLAG_SET(ShenandoahAcmpBarrier); SHENANDOAH_CHECK_FLAG_SET(ShenandoahCloneBarrier); } diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAggressiveHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAggressiveHeuristics.cpp --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAggressiveHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAggressiveHeuristics.cpp @@ -47,13 +47,10 @@ } // Final configuration checks + SHENANDOAH_CHECK_FLAG_SET(ShenandoahLoadRefBarrier); SHENANDOAH_CHECK_FLAG_SET(ShenandoahSATBBarrier); - SHENANDOAH_CHECK_FLAG_SET(ShenandoahReadBarrier); - SHENANDOAH_CHECK_FLAG_SET(ShenandoahWriteBarrier); - SHENANDOAH_CHECK_FLAG_SET(ShenandoahStoreValReadBarrier); SHENANDOAH_CHECK_FLAG_SET(ShenandoahKeepAliveBarrier); SHENANDOAH_CHECK_FLAG_SET(ShenandoahCASBarrier); - SHENANDOAH_CHECK_FLAG_SET(ShenandoahAcmpBarrier); SHENANDOAH_CHECK_FLAG_SET(ShenandoahCloneBarrier); } diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahCompactHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahCompactHeuristics.cpp --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahCompactHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahCompactHeuristics.cpp @@ -42,13 +42,10 @@ SHENANDOAH_ERGO_OVERRIDE_DEFAULT(ShenandoahGarbageThreshold, 10); // Final configuration checks + SHENANDOAH_CHECK_FLAG_SET(ShenandoahLoadRefBarrier); SHENANDOAH_CHECK_FLAG_SET(ShenandoahSATBBarrier); - SHENANDOAH_CHECK_FLAG_SET(ShenandoahReadBarrier); - SHENANDOAH_CHECK_FLAG_SET(ShenandoahWriteBarrier); - SHENANDOAH_CHECK_FLAG_SET(ShenandoahStoreValReadBarrier); SHENANDOAH_CHECK_FLAG_SET(ShenandoahKeepAliveBarrier); SHENANDOAH_CHECK_FLAG_SET(ShenandoahCASBarrier); - SHENANDOAH_CHECK_FLAG_SET(ShenandoahAcmpBarrier); SHENANDOAH_CHECK_FLAG_SET(ShenandoahCloneBarrier); } diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahPassiveHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahPassiveHeuristics.cpp --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahPassiveHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahPassiveHeuristics.cpp @@ -43,14 +43,11 @@ } // Disable known barriers by default. + SHENANDOAH_ERGO_DISABLE_FLAG(ShenandoahLoadRefBarrier); SHENANDOAH_ERGO_DISABLE_FLAG(ShenandoahSATBBarrier); SHENANDOAH_ERGO_DISABLE_FLAG(ShenandoahKeepAliveBarrier); - SHENANDOAH_ERGO_DISABLE_FLAG(ShenandoahWriteBarrier); - SHENANDOAH_ERGO_DISABLE_FLAG(ShenandoahReadBarrier); SHENANDOAH_ERGO_DISABLE_FLAG(ShenandoahStoreValEnqueueBarrier); - SHENANDOAH_ERGO_DISABLE_FLAG(ShenandoahStoreValReadBarrier); SHENANDOAH_ERGO_DISABLE_FLAG(ShenandoahCASBarrier); - SHENANDOAH_ERGO_DISABLE_FLAG(ShenandoahAcmpBarrier); SHENANDOAH_ERGO_DISABLE_FLAG(ShenandoahCloneBarrier); // Final configuration checks diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahStaticHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahStaticHeuristics.cpp --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahStaticHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahStaticHeuristics.cpp @@ -40,13 +40,10 @@ SHENANDOAH_ERGO_ENABLE_FLAG(ShenandoahImplicitGCInvokesConcurrent); // Final configuration checks + SHENANDOAH_CHECK_FLAG_SET(ShenandoahLoadRefBarrier); SHENANDOAH_CHECK_FLAG_SET(ShenandoahSATBBarrier); - SHENANDOAH_CHECK_FLAG_SET(ShenandoahReadBarrier); - SHENANDOAH_CHECK_FLAG_SET(ShenandoahWriteBarrier); - SHENANDOAH_CHECK_FLAG_SET(ShenandoahStoreValReadBarrier); SHENANDOAH_CHECK_FLAG_SET(ShenandoahKeepAliveBarrier); SHENANDOAH_CHECK_FLAG_SET(ShenandoahCASBarrier); - SHENANDOAH_CHECK_FLAG_SET(ShenandoahAcmpBarrier); SHENANDOAH_CHECK_FLAG_SET(ShenandoahCloneBarrier); } diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahTraversalHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahTraversalHeuristics.cpp --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahTraversalHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahTraversalHeuristics.cpp @@ -37,7 +37,6 @@ _last_cset_select(0) { FLAG_SET_DEFAULT(ShenandoahSATBBarrier, false); - FLAG_SET_DEFAULT(ShenandoahStoreValReadBarrier, false); FLAG_SET_DEFAULT(ShenandoahStoreValEnqueueBarrier, true); FLAG_SET_DEFAULT(ShenandoahKeepAliveBarrier, false); FLAG_SET_DEFAULT(ShenandoahAllowMixedAllocs, false); @@ -46,11 +45,9 @@ SHENANDOAH_ERGO_ENABLE_FLAG(ShenandoahImplicitGCInvokesConcurrent); // Final configuration checks - SHENANDOAH_CHECK_FLAG_SET(ShenandoahReadBarrier); - SHENANDOAH_CHECK_FLAG_SET(ShenandoahWriteBarrier); + SHENANDOAH_CHECK_FLAG_SET(ShenandoahLoadRefBarrier); SHENANDOAH_CHECK_FLAG_SET(ShenandoahStoreValEnqueueBarrier); SHENANDOAH_CHECK_FLAG_SET(ShenandoahCASBarrier); - SHENANDOAH_CHECK_FLAG_SET(ShenandoahAcmpBarrier); SHENANDOAH_CHECK_FLAG_SET(ShenandoahCloneBarrier); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahArguments.cpp b/src/hotspot/share/gc/shenandoah/shenandoahArguments.cpp --- a/src/hotspot/share/gc/shenandoah/shenandoahArguments.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahArguments.cpp @@ -44,12 +44,8 @@ FLAG_SET_DEFAULT(ShenandoahSATBBarrier, false); FLAG_SET_DEFAULT(ShenandoahKeepAliveBarrier, false); - FLAG_SET_DEFAULT(ShenandoahWriteBarrier, false); - FLAG_SET_DEFAULT(ShenandoahReadBarrier, false); FLAG_SET_DEFAULT(ShenandoahStoreValEnqueueBarrier, false); - FLAG_SET_DEFAULT(ShenandoahStoreValReadBarrier, false); FLAG_SET_DEFAULT(ShenandoahCASBarrier, false); - FLAG_SET_DEFAULT(ShenandoahAcmpBarrier, false); FLAG_SET_DEFAULT(ShenandoahCloneBarrier, false); #endif @@ -98,12 +94,8 @@ if (ShenandoahVerifyOptoBarriers && (!FLAG_IS_DEFAULT(ShenandoahSATBBarrier) || !FLAG_IS_DEFAULT(ShenandoahKeepAliveBarrier) || - !FLAG_IS_DEFAULT(ShenandoahWriteBarrier) || - !FLAG_IS_DEFAULT(ShenandoahReadBarrier) || !FLAG_IS_DEFAULT(ShenandoahStoreValEnqueueBarrier) || - !FLAG_IS_DEFAULT(ShenandoahStoreValReadBarrier) || !FLAG_IS_DEFAULT(ShenandoahCASBarrier) || - !FLAG_IS_DEFAULT(ShenandoahAcmpBarrier) || !FLAG_IS_DEFAULT(ShenandoahCloneBarrier) )) { warning("Unusual barrier configuration, disabling C2 barrier verification"); @@ -168,13 +160,6 @@ FLAG_SET_DEFAULT(UseAOT, false); } - // JNI fast get field stuff is not currently supported by Shenandoah. - // It would introduce another heap memory access for reading the forwarding - // pointer, which would have to be guarded by the signal handler machinery. - // See: - // http://mail.openjdk.java.net/pipermail/hotspot-dev/2018-June/032763.html - FLAG_SET_DEFAULT(UseFastJNIAccessors, false); - // TLAB sizing policy makes resizing decisions before each GC cycle. It averages // historical data, assigning more recent data the weight according to TLABAllocationWeight. // Current default is good for generational collectors that run frequent young GCs. diff --git a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp --- a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp @@ -215,31 +215,25 @@ } } -oop ShenandoahBarrierSet::read_barrier(oop src) { - // Check for forwarded objects, because on Full GC path we might deal with - // non-trivial fwdptrs that contain Full GC specific metadata. We could check - // for is_full_gc_in_progress(), but this also covers the case of stable heap, - // which provides a bit of performance improvement. - if (ShenandoahReadBarrier && _heap->has_forwarded_objects()) { - return ShenandoahBarrierSet::resolve_forwarded(src); +oop ShenandoahBarrierSet::load_reference_barrier_not_null(oop obj) { + if (ShenandoahLoadRefBarrier && _heap->has_forwarded_objects()) { + return load_reference_barrier_impl(obj); } else { - return src; + return obj; } } -bool ShenandoahBarrierSet::obj_equals(oop obj1, oop obj2) { - bool eq = oopDesc::unsafe_equals(obj1, obj2); - if (! eq && ShenandoahAcmpBarrier) { - OrderAccess::loadload(); - obj1 = resolve_forwarded(obj1); - obj2 = resolve_forwarded(obj2); - eq = oopDesc::unsafe_equals(obj1, obj2); +oop ShenandoahBarrierSet::load_reference_barrier(oop obj) { + if (obj != NULL) { + return load_reference_barrier_not_null(obj); + } else { + return obj; } - return eq; } -oop ShenandoahBarrierSet::write_barrier_mutator(oop obj) { - assert(UseShenandoahGC && ShenandoahWriteBarrier, "should be enabled"); + +oop ShenandoahBarrierSet::load_reference_barrier_mutator(oop obj) { + assert(ShenandoahLoadRefBarrier, "should be enabled"); assert(_heap->is_gc_in_progress_mask(ShenandoahHeap::EVACUATION | ShenandoahHeap::TRAVERSAL), "evac should be in progress"); shenandoah_assert_in_cset(NULL, obj); @@ -285,8 +279,8 @@ return fwd; } -oop ShenandoahBarrierSet::write_barrier_impl(oop obj) { - assert(UseShenandoahGC && ShenandoahWriteBarrier, "should be enabled"); +oop ShenandoahBarrierSet::load_reference_barrier_impl(oop obj) { + assert(ShenandoahLoadRefBarrier, "should be enabled"); if (!CompressedOops::is_null(obj)) { bool evac_in_progress = _heap->is_gc_in_progress_mask(ShenandoahHeap::EVACUATION | ShenandoahHeap::TRAVERSAL); oop fwd = resolve_forwarded_not_null(obj); @@ -308,23 +302,10 @@ } } -oop ShenandoahBarrierSet::write_barrier(oop obj) { - if (ShenandoahWriteBarrier && _heap->has_forwarded_objects()) { - return write_barrier_impl(obj); - } else { - return obj; - } -} - -oop ShenandoahBarrierSet::storeval_barrier(oop obj) { +void ShenandoahBarrierSet::storeval_barrier(oop obj) { if (ShenandoahStoreValEnqueueBarrier && !CompressedOops::is_null(obj) && _heap->is_concurrent_traversal_in_progress()) { - obj = write_barrier(obj); enqueue(obj); } - if (ShenandoahStoreValReadBarrier) { - obj = resolve_forwarded(obj); - } - return obj; } void ShenandoahBarrierSet::keep_alive_barrier(oop obj) { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.hpp b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.hpp --- a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.hpp @@ -89,20 +89,15 @@ virtual void on_thread_attach(JavaThread* thread); virtual void on_thread_detach(JavaThread* thread); - virtual oop read_barrier(oop src); - static inline oop resolve_forwarded_not_null(oop p); static inline oop resolve_forwarded(oop p); - virtual oop write_barrier(oop obj); - - oop write_barrier_mutator(oop obj); + void storeval_barrier(oop obj); + void keep_alive_barrier(oop obj); - virtual oop storeval_barrier(oop obj); - - virtual void keep_alive_barrier(oop obj); - - bool obj_equals(oop obj1, oop obj2); + oop load_reference_barrier(oop obj); + oop load_reference_barrier_mutator(oop obj); + oop load_reference_barrier_not_null(oop obj); void enqueue(oop obj); @@ -110,7 +105,7 @@ template void write_ref_array_loop(HeapWord* start, size_t count); - oop write_barrier_impl(oop obj); + oop load_reference_barrier_impl(oop obj); static void keep_alive_if_weak(DecoratorSet decorators, oop value) { assert((decorators & ON_UNKNOWN_OOP_REF) == 0, "Reference strength must be known"); @@ -145,114 +140,31 @@ class AccessBarrier: public BarrierSet::AccessBarrier { typedef BarrierSet::AccessBarrier Raw; - public: - // Primitive heap accesses. These accessors get resolved when - // IN_HEAP is set (e.g. when using the HeapAccess API), it is - // not an oop_* overload, and the barrier strength is AS_NORMAL. template - static T load_in_heap(T* addr) { - ShouldNotReachHere(); - return Raw::template load(addr); - } - - template - static T load_in_heap_at(oop base, ptrdiff_t offset) { - base = ShenandoahBarrierSet::barrier_set()->read_barrier(base); - return Raw::template load_at(base, offset); - } - - template - static void store_in_heap(T* addr, T value) { - ShouldNotReachHere(); - Raw::store(addr, value); - } - - template - static void store_in_heap_at(oop base, ptrdiff_t offset, T value) { - base = ShenandoahBarrierSet::barrier_set()->write_barrier(base); - Raw::store_at(base, offset, value); - } + static oop oop_atomic_cmpxchg_in_heap_impl(oop new_value, T* addr, oop compare_value); template - static T atomic_cmpxchg_in_heap(T new_value, T* addr, T compare_value) { - ShouldNotReachHere(); - return Raw::atomic_cmpxchg(new_value, addr, compare_value); - } - - template - static T atomic_cmpxchg_in_heap_at(T new_value, oop base, ptrdiff_t offset, T compare_value) { - base = ShenandoahBarrierSet::barrier_set()->write_barrier(base); - return Raw::atomic_cmpxchg_at(new_value, base, offset, compare_value); - } + static oop oop_atomic_xchg_in_heap_impl(oop new_value, T* addr); - template - static T atomic_xchg_in_heap(T new_value, T* addr) { - ShouldNotReachHere(); - return Raw::atomic_xchg(new_value, addr); - } - - template - static T atomic_xchg_in_heap_at(T new_value, oop base, ptrdiff_t offset) { - base = ShenandoahBarrierSet::barrier_set()->write_barrier(base); - return Raw::atomic_xchg_at(new_value, base, offset); - } - - template - static void arraycopy_in_heap(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw, - arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw, - size_t length); - + public: // Heap oop accesses. These accessors get resolved when // IN_HEAP is set (e.g. when using the HeapAccess API), it is // an oop_* overload, and the barrier strength is AS_NORMAL. template - static oop oop_load_in_heap(T* addr) { - // ShouldNotReachHere(); - oop value = Raw::template oop_load(addr); - keep_alive_if_weak(decorators, value); - return value; - } - - static oop oop_load_in_heap_at(oop base, ptrdiff_t offset) { - base = ShenandoahBarrierSet::barrier_set()->read_barrier(base); - oop value = Raw::template oop_load_at(base, offset); - keep_alive_if_weak(AccessBarrierSupport::resolve_possibly_unknown_oop_ref_strength(base, offset), value); - return value; - } + static oop oop_load_in_heap(T* addr); + static oop oop_load_in_heap_at(oop base, ptrdiff_t offset); template - static void oop_store_in_heap(T* addr, oop value) { - const bool keep_alive = (decorators & AS_NO_KEEPALIVE) == 0; - if (keep_alive) { - ShenandoahBarrierSet::barrier_set()->write_ref_field_pre_work(addr, value); - } - Raw::oop_store(addr, value); - } - - static void oop_store_in_heap_at(oop base, ptrdiff_t offset, oop value) { - base = ShenandoahBarrierSet::barrier_set()->write_barrier(base); - value = ShenandoahBarrierSet::barrier_set()->storeval_barrier(value); - - oop_store_in_heap(AccessInternal::oop_field_addr(base, offset), value); - } + static void oop_store_in_heap(T* addr, oop value); + static void oop_store_in_heap_at(oop base, ptrdiff_t offset, oop value); template static oop oop_atomic_cmpxchg_in_heap(oop new_value, T* addr, oop compare_value); - - static oop oop_atomic_cmpxchg_in_heap_at(oop new_value, oop base, ptrdiff_t offset, oop compare_value) { - base = ShenandoahBarrierSet::barrier_set()->write_barrier(base); - new_value = ShenandoahBarrierSet::barrier_set()->storeval_barrier(new_value); - return oop_atomic_cmpxchg_in_heap(new_value, AccessInternal::oop_field_addr(base, offset), compare_value); - } + static oop oop_atomic_cmpxchg_in_heap_at(oop new_value, oop base, ptrdiff_t offset, oop compare_value); template static oop oop_atomic_xchg_in_heap(oop new_value, T* addr); - - static oop oop_atomic_xchg_in_heap_at(oop new_value, oop base, ptrdiff_t offset) { - base = ShenandoahBarrierSet::barrier_set()->write_barrier(base); - new_value = ShenandoahBarrierSet::barrier_set()->storeval_barrier(new_value); - return oop_atomic_xchg_in_heap(new_value, AccessInternal::oop_field_addr(base, offset)); - } + static oop oop_atomic_xchg_in_heap_at(oop new_value, oop base, ptrdiff_t offset); template static bool oop_arraycopy_in_heap(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw, @@ -264,19 +176,13 @@ // Needed for loads on non-heap weak references template - static oop oop_load_not_in_heap(T* addr) { - oop value = Raw::oop_load_not_in_heap(addr); - keep_alive_if_weak(decorators, value); - return value; - } + static oop oop_load_not_in_heap(T* addr); - static oop resolve(oop obj) { - return ShenandoahBarrierSet::barrier_set()->write_barrier(obj); - } + template + static oop oop_atomic_cmpxchg_not_in_heap(oop new_value, T* addr, oop compare_value); - static bool equals(oop o1, oop o2) { - return ShenandoahBarrierSet::barrier_set()->obj_equals(o1, o2); - } + template + static oop oop_atomic_xchg_not_in_heap(oop new_value, T* addr); }; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp --- a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp @@ -46,7 +46,49 @@ template template -inline oop ShenandoahBarrierSet::AccessBarrier::oop_atomic_cmpxchg_in_heap(oop new_value, T* addr, oop compare_value) { +inline oop ShenandoahBarrierSet::AccessBarrier::oop_load_in_heap(T* addr) { + oop value = Raw::oop_load_in_heap(addr); + value = ShenandoahBarrierSet::barrier_set()->load_reference_barrier(value); + keep_alive_if_weak(decorators, value); + return value; +} + +template +inline oop ShenandoahBarrierSet::AccessBarrier::oop_load_in_heap_at(oop base, ptrdiff_t offset) { + oop value = Raw::oop_load_in_heap_at(base, offset); + value = ShenandoahBarrierSet::barrier_set()->load_reference_barrier(value); + keep_alive_if_weak(AccessBarrierSupport::resolve_possibly_unknown_oop_ref_strength(base, offset), value); + return value; +} + +template +template +inline oop ShenandoahBarrierSet::AccessBarrier::oop_load_not_in_heap(T* addr) { + oop value = Raw::oop_load_not_in_heap(addr); + value = ShenandoahBarrierSet::barrier_set()->load_reference_barrier(value); + keep_alive_if_weak(decorators, value); + return value; +} + +template +template +inline void ShenandoahBarrierSet::AccessBarrier::oop_store_in_heap(T* addr, oop value) { + ShenandoahBarrierSet::barrier_set()->storeval_barrier(value); + const bool keep_alive = (decorators & AS_NO_KEEPALIVE) == 0; + if (keep_alive) { + ShenandoahBarrierSet::barrier_set()->write_ref_field_pre_work(addr, value); + } + Raw::oop_store_in_heap(addr, value); +} + +template +inline void ShenandoahBarrierSet::AccessBarrier::oop_store_in_heap_at(oop base, ptrdiff_t offset, oop value) { + oop_store_in_heap(AccessInternal::oop_field_addr(base, offset), value); +} + +template +template +inline oop ShenandoahBarrierSet::AccessBarrier::oop_atomic_cmpxchg_not_in_heap(oop new_value, T* addr, oop compare_value) { oop res; oop expected = compare_value; do { @@ -54,42 +96,79 @@ res = Raw::oop_atomic_cmpxchg(new_value, addr, compare_value); expected = res; } while ((! oopDesc::unsafe_equals(compare_value, expected)) && oopDesc::unsafe_equals(resolve_forwarded(compare_value), resolve_forwarded(expected))); - if (oopDesc::unsafe_equals(expected, compare_value)) { - const bool keep_alive = (decorators & AS_NO_KEEPALIVE) == 0; - if (keep_alive && ShenandoahSATBBarrier && !CompressedOops::is_null(compare_value) && - ShenandoahHeap::heap()->is_concurrent_mark_in_progress()) { - ShenandoahBarrierSet::barrier_set()->enqueue(compare_value); - } + if (res != NULL) { + return ShenandoahBarrierSet::barrier_set()->load_reference_barrier_not_null(res); + } else { + return res; + } +} + +template +template +inline oop ShenandoahBarrierSet::AccessBarrier::oop_atomic_cmpxchg_in_heap_impl(oop new_value, T* addr, oop compare_value) { + ShenandoahBarrierSet::barrier_set()->storeval_barrier(new_value); + oop result = oop_atomic_cmpxchg_not_in_heap(new_value, addr, compare_value); + const bool keep_alive = (decorators & AS_NO_KEEPALIVE) == 0; + if (keep_alive && ShenandoahSATBBarrier && !CompressedOops::is_null(result) && + oopDesc::unsafe_equals(result, compare_value) && + ShenandoahHeap::heap()->is_concurrent_mark_in_progress()) { + ShenandoahBarrierSet::barrier_set()->enqueue(result); } - return res; + return result; +} + +template +template +inline oop ShenandoahBarrierSet::AccessBarrier::oop_atomic_cmpxchg_in_heap(oop new_value, T* addr, oop compare_value) { + oop result = oop_atomic_cmpxchg_in_heap_impl(new_value, addr, compare_value); + keep_alive_if_weak(decorators, result); + return result; +} + +template +inline oop ShenandoahBarrierSet::AccessBarrier::oop_atomic_cmpxchg_in_heap_at(oop new_value, oop base, ptrdiff_t offset, oop compare_value) { + oop result = oop_atomic_cmpxchg_in_heap_impl(new_value, AccessInternal::oop_field_addr(base, offset), compare_value); + keep_alive_if_weak(AccessBarrierSupport::resolve_possibly_unknown_oop_ref_strength(base, offset), result); + return result; +} + +template +template +inline oop ShenandoahBarrierSet::AccessBarrier::oop_atomic_xchg_not_in_heap(oop new_value, T* addr) { + oop previous = Raw::oop_atomic_xchg(new_value, addr); + if (previous != NULL) { + return ShenandoahBarrierSet::barrier_set()->load_reference_barrier_not_null(previous); + } else { + return previous; + } +} + +template +template +inline oop ShenandoahBarrierSet::AccessBarrier::oop_atomic_xchg_in_heap_impl(oop new_value, T* addr) { + ShenandoahBarrierSet::barrier_set()->storeval_barrier(new_value); + oop result = oop_atomic_xchg_not_in_heap(new_value, addr); + const bool keep_alive = (decorators & AS_NO_KEEPALIVE) == 0; + if (keep_alive && ShenandoahSATBBarrier && !CompressedOops::is_null(result) && + ShenandoahHeap::heap()->is_concurrent_mark_in_progress()) { + ShenandoahBarrierSet::barrier_set()->enqueue(result); + } + return result; } template template inline oop ShenandoahBarrierSet::AccessBarrier::oop_atomic_xchg_in_heap(oop new_value, T* addr) { - oop previous = Raw::oop_atomic_xchg(new_value, addr); - if (ShenandoahSATBBarrier) { - const bool keep_alive = (decorators & AS_NO_KEEPALIVE) == 0; - if (keep_alive && !CompressedOops::is_null(previous) && - ShenandoahHeap::heap()->is_concurrent_mark_in_progress()) { - ShenandoahBarrierSet::barrier_set()->enqueue(previous); - } - } - return previous; + oop result = oop_atomic_xchg_in_heap_impl(new_value, addr); + keep_alive_if_weak(addr, result); + return result; } template -template -void ShenandoahBarrierSet::AccessBarrier::arraycopy_in_heap(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw, - arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw, - size_t length) { - if (!CompressedOops::is_null(src_obj)) { - src_obj = arrayOop(ShenandoahBarrierSet::barrier_set()->read_barrier(src_obj)); - } - if (!CompressedOops::is_null(dst_obj)) { - dst_obj = arrayOop(ShenandoahBarrierSet::barrier_set()->write_barrier(dst_obj)); - } - Raw::arraycopy(src_obj, src_offset_in_bytes, src_raw, dst_obj, dst_offset_in_bytes, dst_raw, length); +inline oop ShenandoahBarrierSet::AccessBarrier::oop_atomic_xchg_in_heap_at(oop new_value, oop base, ptrdiff_t offset) { + oop result = oop_atomic_xchg_in_heap_impl(new_value, AccessInternal::oop_field_addr(base, offset)); + keep_alive_if_weak(AccessBarrierSupport::resolve_possibly_unknown_oop_ref_strength(base, offset), result); + return result; } template @@ -242,8 +321,6 @@ // Clone barrier support template void ShenandoahBarrierSet::AccessBarrier::clone_in_heap(oop src, oop dst, size_t size) { - src = arrayOop(ShenandoahBarrierSet::barrier_set()->read_barrier(src)); - dst = arrayOop(ShenandoahBarrierSet::barrier_set()->write_barrier(dst)); Raw::clone(src, dst, size); ShenandoahBarrierSet::barrier_set()->write_region(MemRegion((HeapWord*) dst, size)); } @@ -254,13 +331,6 @@ arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw, size_t length) { ShenandoahHeap* heap = ShenandoahHeap::heap(); - if (!CompressedOops::is_null(src_obj)) { - src_obj = arrayOop(ShenandoahBarrierSet::barrier_set()->read_barrier(src_obj)); - } - if (!CompressedOops::is_null(dst_obj)) { - dst_obj = arrayOop(ShenandoahBarrierSet::barrier_set()->write_barrier(dst_obj)); - } - bool satb = ShenandoahSATBBarrier && heap->is_concurrent_mark_in_progress(); bool checkcast = HasDecorator::value; bool disjoint = HasDecorator::value; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahCodeRoots.cpp b/src/hotspot/share/gc/shenandoah/shenandoahCodeRoots.cpp --- a/src/hotspot/share/gc/shenandoah/shenandoahCodeRoots.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahCodeRoots.cpp @@ -119,39 +119,6 @@ } }; -class ShenandoahNMethodOopInitializer : public OopClosure { -private: - ShenandoahHeap* const _heap; - -public: - ShenandoahNMethodOopInitializer() : _heap(ShenandoahHeap::heap()) {}; - -private: - template - inline void do_oop_work(T* p) { - T o = RawAccess<>::oop_load(p); - if (! CompressedOops::is_null(o)) { - oop obj1 = CompressedOops::decode_not_null(o); - oop obj2 = ShenandoahBarrierSet::barrier_set()->write_barrier(obj1); - if (! oopDesc::unsafe_equals(obj1, obj2)) { - shenandoah_assert_not_in_cset(NULL, obj2); - RawAccess::oop_store(p, obj2); - if (_heap->is_concurrent_traversal_in_progress()) { - ShenandoahBarrierSet::barrier_set()->enqueue(obj2); - } - } - } - } - -public: - void do_oop(oop* o) { - do_oop_work(o); - } - void do_oop(narrowOop* o) { - do_oop_work(o); - } -}; - ShenandoahCodeRoots::PaddedLock ShenandoahCodeRoots::_recorded_nms_lock; GrowableArray* ShenandoahCodeRoots::_recorded_nms; @@ -163,21 +130,13 @@ void ShenandoahCodeRoots::add_nmethod(nmethod* nm) { switch (ShenandoahCodeRootsStyle) { case 0: - case 1: { - ShenandoahNMethodOopInitializer init; - nm->oops_do(&init); - nm->fix_oop_relocations(); + case 1: break; - } case 2: { ShenandoahNMethodOopDetector detector; nm->oops_do(&detector); if (detector.has_oops()) { - ShenandoahNMethodOopInitializer init; - nm->oops_do(&init); - nm->fix_oop_relocations(); - ShenandoahNMethod* nmr = new ShenandoahNMethod(nm, detector.oops()); nmr->assert_alive_and_correct(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahEvacOOMHandler.hpp b/src/hotspot/share/gc/shenandoah/shenandoahEvacOOMHandler.hpp --- a/src/hotspot/share/gc/shenandoah/shenandoahEvacOOMHandler.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahEvacOOMHandler.hpp @@ -31,8 +31,8 @@ * Provides safe handling of out-of-memory situations during evacuation. * * When a Java thread encounters out-of-memory while evacuating an object in a - * write-barrier (i.e. it cannot copy the object to to-space), it does not necessarily - * follow we can return immediately from the WB (and store to from-space). + * load-reference-barrier (i.e. it cannot copy the object to to-space), it does not + * necessarily follow we can return immediately from the LRB (and store to from-space). * * In very basic case, on such failure we may wait until the the evacuation is over, * and then resolve the forwarded copy, and to the store there. This is possible @@ -64,17 +64,17 @@ * - failure: * - if offending value is a valid counter, then try again * - if offending value is OOM-during-evac special value: loop until - * counter drops to 0, then exit with read-barrier + * counter drops to 0, then exit with resolving the ptr * * Upon exit, exiting thread will decrease the counter using atomic dec. * * Upon OOM-during-evac, any thread will attempt to CAS OOM-during-evac * special value into the counter. Depending on result: - * - success: busy-loop until counter drops to zero, then exit with RB + * - success: busy-loop until counter drops to zero, then exit with resolve * - failure: * - offender is valid counter update: try again * - offender is OOM-during-evac: busy loop until counter drops to - * zero, then exit with RB + * zero, then exit with resolve */ class ShenandoahEvacOOMHandler { private: @@ -94,7 +94,7 @@ * * When this returns true, it is safe to continue with normal evacuation. * When this method returns false, evacuation must not be entered, and caller - * may safely continue with a read-barrier (if Java thread). + * may safely continue with a simple resolve (if Java thread). */ void enter_evacuation(); @@ -106,7 +106,7 @@ /** * Signal out-of-memory during evacuation. It will prevent any other threads * from entering the evacuation path, then wait until all threads have left the - * evacuation path, and then return. It is then safe to continue with a read-barrier. + * evacuation path, and then return. It is then safe to continue with a simple resolve. */ void handle_out_of_memory_during_evacuation(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -410,10 +410,6 @@ err_msg("Heuristics \"%s\" is experimental, and must be enabled via -XX:+UnlockExperimentalVMOptions.", _heuristics->name())); } - - if (ShenandoahStoreValEnqueueBarrier && ShenandoahStoreValReadBarrier) { - vm_exit_during_initialization("Cannot use both ShenandoahStoreValEnqueueBarrier and ShenandoahStoreValReadBarrier"); - } log_info(gc, init)("Shenandoah heuristics: %s", _heuristics->name()); } else { @@ -831,7 +827,7 @@ assert(req.is_gc_alloc(), "Can only accept GC allocs here"); result = allocate_memory_under_lock(req, in_new_region); // Do not call handle_alloc_failure() here, because we cannot block. - // The allocation failure would be handled by the WB slowpath with handle_alloc_failure_evac(). + // The allocation failure would be handled by the LRB slowpath with handle_alloc_failure_evac(). } if (in_new_region) { @@ -1108,7 +1104,6 @@ ShenandoahParallelWorkerSession worker_session(worker_id); ShenandoahEvacOOMScope oom_evac_scope; ShenandoahEvacuateUpdateRootsClosure cl; - MarkingCodeBlobClosure blobsCl(&cl, CodeBlobToOopClosure::FixRelocations); _rp->process_evacuate_roots(&cl, &blobsCl, worker_id); } @@ -2057,14 +2052,12 @@ } oop ShenandoahHeap::pin_object(JavaThread* thr, oop o) { - o = ShenandoahBarrierSet::barrier_set()->write_barrier(o); ShenandoahHeapLocker locker(lock()); heap_region_containing(o)->make_pinned(); return o; } void ShenandoahHeap::unpin_object(JavaThread* thr, oop o) { - o = ShenandoahBarrierSet::barrier_set()->read_barrier(o); ShenandoahHeapLocker locker(lock()); heap_region_containing(o)->make_unpinned(); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp @@ -231,16 +231,16 @@ // public: enum GCStateBitPos { - // Heap has forwarded objects: need RB, ACMP, CAS barriers. + // Heap has forwarded objects: needs LRB barriers. HAS_FORWARDED_BITPOS = 0, // Heap is under marking: needs SATB barriers. MARKING_BITPOS = 1, - // Heap is under evacuation: needs WB barriers. (Set together with UNSTABLE) + // Heap is under evacuation: needs LRB barriers. (Set together with HAS_FORWARDED) EVACUATION_BITPOS = 2, - // Heap is under updating: needs SVRB/SVWB barriers. + // Heap is under updating: needs no additional barriers. UPDATEREFS_BITPOS = 3, // Heap is under traversal collection diff --git a/src/hotspot/share/gc/shenandoah/shenandoahMarkCompact.cpp b/src/hotspot/share/gc/shenandoah/shenandoahMarkCompact.cpp --- a/src/hotspot/share/gc/shenandoah/shenandoahMarkCompact.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahMarkCompact.cpp @@ -131,7 +131,7 @@ // Once marking is done, which may have fixed up forwarded objects, we can drop it. // Coming out of Full GC, we would not have any forwarded objects. - // This also prevents read barrier from kicking in while adjusting pointers in phase3. + // This also prevents resolves with fwdptr from kicking in while adjusting pointers in phase3. heap->set_has_forwarded_objects(false); heap->set_full_gc_move_in_progress(true); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahOopClosures.hpp b/src/hotspot/share/gc/shenandoah/shenandoahOopClosures.hpp --- a/src/hotspot/share/gc/shenandoah/shenandoahOopClosures.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahOopClosures.hpp @@ -34,7 +34,7 @@ enum UpdateRefsMode { NONE, // No reference updating - RESOLVE, // Only a read-barrier (no reference updating) + RESOLVE, // Only a resolve (no reference updating) SIMPLE, // Reference updating using simple store CONCURRENT // Reference updating using CAS }; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahRootProcessor.cpp b/src/hotspot/share/gc/shenandoah/shenandoahRootProcessor.cpp --- a/src/hotspot/share/gc/shenandoah/shenandoahRootProcessor.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahRootProcessor.cpp @@ -207,13 +207,21 @@ _evacuation_tasks(new SubTasksDone(SHENANDOAH_EVAC_NumElements)), _srs(n_workers), _phase(phase), - _coderoots_cset_iterator(ShenandoahCodeRoots::cset_iterator()) + _coderoots_cset_iterator(ShenandoahCodeRoots::cset_iterator()), + _par_state_string(StringTable::weak_storage()) + { heap->phase_timings()->record_workers_start(_phase); + if (ShenandoahStringDedup::is_enabled()) { + StringDedup::gc_prologue(false); + } } ShenandoahRootEvacuator::~ShenandoahRootEvacuator() { delete _evacuation_tasks; + if (ShenandoahStringDedup::is_enabled()) { + StringDedup::gc_epilogue(); + } ShenandoahHeap::heap()->phase_timings()->record_workers_end(_phase); } @@ -235,11 +243,38 @@ _coderoots_cset_iterator.possibly_parallel_blobs_do(blobs); } - if (_evacuation_tasks->is_task_claimed(SHENANDOAH_EVAC_jvmti_oops_do)) { + if (ShenandoahStringDedup::is_enabled()) { ShenandoahForwardedIsAliveClosure is_alive; + ShenandoahStringDedup::parallel_oops_do(&is_alive, oops, worker_id); + } + + if (_evacuation_tasks->try_claim_task(SHENANDOAH_EVAC_Universe_oops_do)) { + ShenandoahWorkerTimingsTracker timer(worker_times, ShenandoahPhaseTimings::UniverseRoots, worker_id); + Universe::oops_do(oops); + } + + if (_evacuation_tasks->try_claim_task(SHENANDOAH_EVAC_Management_oops_do)) { + ShenandoahWorkerTimingsTracker timer(worker_times, ShenandoahPhaseTimings::ManagementRoots, worker_id); + Management::oops_do(oops); + } + + if (_evacuation_tasks->try_claim_task(SHENANDOAH_EVAC_jvmti_oops_do)) { ShenandoahWorkerTimingsTracker timer(worker_times, ShenandoahPhaseTimings::JVMTIRoots, worker_id); + JvmtiExport::oops_do(oops); + ShenandoahForwardedIsAliveClosure is_alive; JvmtiExport::weak_oops_do(&is_alive, oops); } + + if (_evacuation_tasks->try_claim_task(SHENANDOAH_EVAC_SystemDictionary_oops_do)) { + ShenandoahWorkerTimingsTracker timer(worker_times, ShenandoahPhaseTimings::SystemDictionaryRoots, worker_id); + SystemDictionary::oops_do(oops); + } + + if (_evacuation_tasks->try_claim_task(SHENANDOAH_EVAC_ObjectSynchronizer_oops_do)) { + ShenandoahWorkerTimingsTracker timer(worker_times, ShenandoahPhaseTimings::ObjectSynchronizerRoots, worker_id); + ObjectSynchronizer::oops_do(oops); + } + } uint ShenandoahRootEvacuator::n_workers() const { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahRootProcessor.hpp b/src/hotspot/share/gc/shenandoah/shenandoahRootProcessor.hpp --- a/src/hotspot/share/gc/shenandoah/shenandoahRootProcessor.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahRootProcessor.hpp @@ -57,7 +57,7 @@ StrongRootsScope _srs; OopStorage::ParState _par_state_string; ShenandoahPhaseTimings::Phase _phase; - ParallelCLDRootIterator _cld_iterator; + ParallelCLDRootIterator _cld_iterator; ShenandoahAllCodeRootsIterator _coderoots_all_iterator; CodeBlobClosure* _threads_nmethods_cl; @@ -112,11 +112,16 @@ StrongRootsScope _srs; ShenandoahPhaseTimings::Phase _phase; ShenandoahCsetCodeRootsIterator _coderoots_cset_iterator; + OopStorage::ParState _par_state_string; enum Shenandoah_evacuate_roots_tasks { - SHENANDOAH_EVAC_jvmti_oops_do, - // Leave this one last. - SHENANDOAH_EVAC_NumElements + SHENANDOAH_EVAC_Universe_oops_do, + SHENANDOAH_EVAC_ObjectSynchronizer_oops_do, + SHENANDOAH_EVAC_Management_oops_do, + SHENANDOAH_EVAC_SystemDictionary_oops_do, + SHENANDOAH_EVAC_jvmti_oops_do, + // Leave this one last. + SHENANDOAH_EVAC_NumElements }; public: ShenandoahRootEvacuator(ShenandoahHeap* heap, uint n_workers, diff --git a/src/hotspot/share/gc/shenandoah/shenandoahRuntime.cpp b/src/hotspot/share/gc/shenandoah/shenandoahRuntime.cpp --- a/src/hotspot/share/gc/shenandoah/shenandoahRuntime.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahRuntime.cpp @@ -55,8 +55,8 @@ ShenandoahThreadLocalData::satb_mark_queue(thread).enqueue_known_active(orig); JRT_END -JRT_LEAF(oopDesc*, ShenandoahRuntime::write_barrier_JRT(oopDesc* src)) - oop result = ShenandoahBarrierSet::barrier_set()->write_barrier_mutator(src); +JRT_LEAF(oopDesc*, ShenandoahRuntime::load_reference_barrier_JRT(oopDesc* src)) + oop result = ShenandoahBarrierSet::barrier_set()->load_reference_barrier_mutator(src); return (oopDesc*) result; JRT_END diff --git a/src/hotspot/share/gc/shenandoah/shenandoahRuntime.hpp b/src/hotspot/share/gc/shenandoah/shenandoahRuntime.hpp --- a/src/hotspot/share/gc/shenandoah/shenandoahRuntime.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahRuntime.hpp @@ -38,7 +38,7 @@ static void write_ref_array_post_entry(HeapWord* dst, size_t length); static void write_ref_field_pre_entry(oopDesc* orig, JavaThread* thread); - static oopDesc* write_barrier_JRT(oopDesc* src); + static oopDesc* load_reference_barrier_JRT(oopDesc* src); static void shenandoah_clone_barrier(oopDesc* obj); }; diff --git a/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp b/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp --- a/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp @@ -244,7 +244,7 @@ "Time is in microseconds.") \ \ experimental(uintx, ShenandoahEvacAssist, 10, \ - "How many objects to evacuate on WB assist path. " \ + "How many objects to evacuate on LRB assist path. " \ "Use zero to disable.") \ \ experimental(bool, ShenandoahPacing, true, \ @@ -352,27 +352,18 @@ diagnostic(bool, ShenandoahKeepAliveBarrier, true, \ "Turn on/off keep alive barriers in Shenandoah") \ \ - diagnostic(bool, ShenandoahWriteBarrier, true, \ - "Turn on/off write barriers in Shenandoah") \ - \ - diagnostic(bool, ShenandoahReadBarrier, true, \ - "Turn on/off read barriers in Shenandoah") \ - \ diagnostic(bool, ShenandoahStoreValEnqueueBarrier, false, \ "Turn on/off enqueuing of oops for storeval barriers") \ \ - diagnostic(bool, ShenandoahStoreValReadBarrier, true, \ - "Turn on/off store val read barriers in Shenandoah") \ - \ diagnostic(bool, ShenandoahCASBarrier, true, \ "Turn on/off CAS barriers in Shenandoah") \ \ - diagnostic(bool, ShenandoahAcmpBarrier, true, \ - "Turn on/off acmp barriers in Shenandoah") \ - \ diagnostic(bool, ShenandoahCloneBarrier, true, \ "Turn on/off clone barriers in Shenandoah") \ \ + diagnostic(bool, ShenandoahLoadRefBarrier, true, \ + "Turn on/off load-reference barriers in Shenandoah") \ + \ diagnostic(bool, ShenandoahStoreCheck, false, \ "Emit additional code that checks objects are written to only" \ " in to-space") \ @@ -401,20 +392,13 @@ "Turn it off for maximum compatibility with reflection or JNI " \ "code that manipulates final fields.") \ \ - diagnostic(bool, ShenandoahDecreaseRegisterPressure, false, \ - "Try to reuse after-barrier values to reduce register pressure") \ - \ experimental(bool, ShenandoahCommonGCStateLoads, false, \ "Enable commonming for GC state loads in generated code.") \ \ develop(bool, ShenandoahVerifyOptoBarriers, false, \ "Verify no missing barriers in C2") \ \ - experimental(bool, ShenandoahDontIncreaseWBFreq, true, \ - "Common 2 WriteBarriers or WriteBarrier and a ReadBarrier only " \ - "if the resulting WriteBarrier isn't executed more frequently") \ - \ experimental(bool, ShenandoahLoopOptsAfterExpansion, true, \ - "Attempt more loop opts after write barrier expansion") \ + "Attempt more loop opts after barrier expansion") \ #endif // SHARE_VM_GC_SHENANDOAH_SHENANDOAH_GLOBALS_HPP diff --git a/src/hotspot/share/opto/arraycopynode.cpp b/src/hotspot/share/opto/arraycopynode.cpp --- a/src/hotspot/share/opto/arraycopynode.cpp +++ b/src/hotspot/share/opto/arraycopynode.cpp @@ -154,45 +154,8 @@ #if INCLUDE_SHENANDOAHGC Node* ArrayCopyNode::shenandoah_add_storeval_barrier(PhaseGVN *phase, bool can_reshape, Node* v, MergeMemNode* mem, Node*& ctl) { - if (ShenandoahStoreValReadBarrier) { - RegionNode* region = new RegionNode(3); - const Type* v_t = phase->type(v); - Node* phi = new PhiNode(region, v_t->isa_oopptr() ? v_t->is_oopptr()->cast_to_nonconst() : v_t); - Node* cmp = phase->transform(new CmpPNode(v, phase->zerocon(T_OBJECT))); - Node* bol = phase->transform(new BoolNode(cmp, BoolTest::ne)); - IfNode* iff = new IfNode(ctl, bol, PROB_LIKELY_MAG(3), COUNT_UNKNOWN); - - phase->transform(iff); - if (can_reshape) { - phase->is_IterGVN()->_worklist.push(iff); - } else { - phase->record_for_igvn(iff); - } - - Node* null_true = phase->transform(new IfFalseNode(iff)); - Node* null_false = phase->transform(new IfTrueNode(iff)); - region->init_req(1, null_true); - region->init_req(2, null_false); - phi->init_req(1, phase->zerocon(T_OBJECT)); - Node* cast = new CastPPNode(v, phase->type(v)->join_speculative(TypePtr::NOTNULL)); - cast->set_req(0, null_false); - cast = phase->transform(cast); - Node* rb = phase->transform(new ShenandoahReadBarrierNode(null_false, phase->C->immutable_memory(), cast, false)); - phi->init_req(2, rb); - ctl = phase->transform(region); - return phase->transform(phi); - } if (ShenandoahStoreValEnqueueBarrier) { - const TypePtr* adr_type = ShenandoahBarrierNode::brooks_pointer_type(phase->type(v)); - int alias = phase->C->get_alias_index(adr_type); - Node* wb = new ShenandoahWriteBarrierNode(phase->C, ctl, mem->memory_at(alias), v); - Node* wb_transformed = phase->transform(wb); - Node* enqueue = phase->transform(new ShenandoahEnqueueBarrierNode(wb_transformed)); - if (wb_transformed == wb) { - Node* proj = phase->transform(new ShenandoahWBMemProjNode(wb)); - mem->set_memory_at(alias, proj); - } - return enqueue; + return phase->transform(new ShenandoahEnqueueBarrierNode(v)); } return v; } diff --git a/src/hotspot/share/opto/cfgnode.cpp b/src/hotspot/share/opto/cfgnode.cpp --- a/src/hotspot/share/opto/cfgnode.cpp +++ b/src/hotspot/share/opto/cfgnode.cpp @@ -1314,9 +1314,8 @@ // Build int->bool concfgversion Node *in1 = cmp->in(1); -#if INCLUDE_SHENANDOAHGC - in1 = ShenandoahBarrierNode::skip_through_barrier(in1); -#endif + BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2(); + in1 = bs->step_over_gc_barrier(in1); Node *n = new Conv2BNode(in1); if( flipped ) n = new XorINode( phase->transform(n), phase->intcon(1) ); diff --git a/src/hotspot/share/opto/classes.hpp b/src/hotspot/share/opto/classes.hpp --- a/src/hotspot/share/opto/classes.hpp +++ b/src/hotspot/share/opto/classes.hpp @@ -270,9 +270,7 @@ #define shmacro(x) optionalmacro(x) #endif shmacro(ShenandoahEnqueueBarrier) -shmacro(ShenandoahReadBarrier) -shmacro(ShenandoahWriteBarrier) -shmacro(ShenandoahWBMemProj) +shmacro(ShenandoahLoadReferenceBarrier) macro(SCMemProj) macro(SqrtD) macro(SqrtF) diff --git a/src/hotspot/share/opto/compile.cpp b/src/hotspot/share/opto/compile.cpp --- a/src/hotspot/share/opto/compile.cpp +++ b/src/hotspot/share/opto/compile.cpp @@ -2410,7 +2410,7 @@ print_method(PHASE_BEFORE_BARRIER_EXPAND, 2); #if INCLUDE_SHENANDOAHGC - if (UseShenandoahGC && !ShenandoahWriteBarrierNode::expand(this, igvn, loop_opts_cnt)) { + if (UseShenandoahGC && !((ShenandoahBarrierSetC2*)BarrierSet::barrier_set()->barrier_set_c2())->expand_barriers(this, igvn)) { assert(failing(), "must bail out w/ explicit message"); return; } @@ -3058,7 +3058,7 @@ Node *m = wq.at(next); for (DUIterator_Fast imax, i = m->fast_outs(imax); i < imax; i++) { Node* use = m->fast_out(i); - if (use->is_Mem() || use->is_EncodeNarrowPtr() || use->is_ShenandoahBarrier()) { + if (use->is_Mem() || use->is_EncodeNarrowPtr()) { use->ensure_control_or_add_prec(n->in(0)); } else { switch(use->Opcode()) { @@ -3395,9 +3395,7 @@ } break; #if INCLUDE_SHENANDOAHGC - case Op_ShenandoahReadBarrier: - break; - case Op_ShenandoahWriteBarrier: + case Op_ShenandoahLoadReferenceBarrier: assert(false, "should have been expanded already"); break; #endif diff --git a/src/hotspot/share/opto/lcm.cpp b/src/hotspot/share/opto/lcm.cpp --- a/src/hotspot/share/opto/lcm.cpp +++ b/src/hotspot/share/opto/lcm.cpp @@ -178,7 +178,6 @@ case Op_LoadRange: case Op_LoadD_unaligned: case Op_LoadL_unaligned: - case Op_ShenandoahReadBarrier: assert(mach->in(2) == val, "should be address"); break; case Op_StoreB: diff --git a/src/hotspot/share/opto/library_call.cpp b/src/hotspot/share/opto/library_call.cpp --- a/src/hotspot/share/opto/library_call.cpp +++ b/src/hotspot/share/opto/library_call.cpp @@ -4536,7 +4536,7 @@ for (MergeMemStream mms(merged_memory(), mem->as_MergeMem()); mms.next_non_empty2(); ) { Node* n = mms.memory(); if (n != mms.memory2() && !(n->is_Proj() && n->in(0) == alloc->initialization())) { - assert(n->is_Store() || n->Opcode() == Op_ShenandoahWBMemProj, "what else?"); + assert(n->is_Store(), "what else?"); no_interfering_store = false; break; } @@ -4545,7 +4545,7 @@ for (MergeMemStream mms(merged_memory()); mms.next_non_empty(); ) { Node* n = mms.memory(); if (n != mem && !(n->is_Proj() && n->in(0) == alloc->initialization())) { - assert(n->is_Store() || n->Opcode() == Op_ShenandoahWBMemProj, "what else?"); + assert(n->is_Store(), "what else?"); no_interfering_store = false; break; } diff --git a/src/hotspot/share/opto/loopPredicate.cpp b/src/hotspot/share/opto/loopPredicate.cpp --- a/src/hotspot/share/opto/loopPredicate.cpp +++ b/src/hotspot/share/opto/loopPredicate.cpp @@ -539,9 +539,6 @@ if (_lpt->is_invariant(n)) { // known invariant _invariant.set(n->_idx); } else if (!n->is_CFG()) { - if (n->Opcode() == Op_ShenandoahWriteBarrier) { - return; - } Node *n_ctrl = _phase->ctrl_or_self(n); Node *u_ctrl = _phase->ctrl_or_self(use); // self if use is a CFG if (_phase->is_dominator(n_ctrl, u_ctrl)) { diff --git a/src/hotspot/share/opto/loopopts.cpp b/src/hotspot/share/opto/loopopts.cpp --- a/src/hotspot/share/opto/loopopts.cpp +++ b/src/hotspot/share/opto/loopopts.cpp @@ -3227,7 +3227,7 @@ // if not pinned and not a load (which maybe anti-dependent on a store) // and not a CMove (Matcher expects only bool->cmove). - if (n->in(0) == NULL && !n->is_Load() && !n->is_CMove() && n->Opcode() != Op_ShenandoahWBMemProj) { + if (n->in(0) == NULL && !n->is_Load() && !n->is_CMove()) { cloned_for_outside_use += clone_for_use_outside_loop( loop, n, worklist ); sink_list.push(n); peel >>= n->_idx; // delete n from peel set. diff --git a/src/hotspot/share/opto/node.hpp b/src/hotspot/share/opto/node.hpp --- a/src/hotspot/share/opto/node.hpp +++ b/src/hotspot/share/opto/node.hpp @@ -142,7 +142,6 @@ class RootNode; class SafePointNode; class SafePointScalarObjectNode; -class ShenandoahBarrierNode; class StartNode; class State; class StoreNode; @@ -676,7 +675,6 @@ DEFINE_CLASS_ID(EncodeNarrowPtr, Type, 6) DEFINE_CLASS_ID(EncodeP, EncodeNarrowPtr, 0) DEFINE_CLASS_ID(EncodePKlass, EncodeNarrowPtr, 1) - DEFINE_CLASS_ID(ShenandoahBarrier, Type, 7) DEFINE_CLASS_ID(Proj, Node, 3) DEFINE_CLASS_ID(CatchProj, Proj, 0) @@ -875,7 +873,6 @@ DEFINE_CLASS_QUERY(Root) DEFINE_CLASS_QUERY(SafePoint) DEFINE_CLASS_QUERY(SafePointScalarObject) - DEFINE_CLASS_QUERY(ShenandoahBarrier) DEFINE_CLASS_QUERY(Start) DEFINE_CLASS_QUERY(Store) DEFINE_CLASS_QUERY(Sub) diff --git a/test/hotspot/jtreg/gc/shenandoah/options/TestSelectiveBarrierFlags.java b/test/hotspot/jtreg/gc/shenandoah/options/TestSelectiveBarrierFlags.java --- a/test/hotspot/jtreg/gc/shenandoah/options/TestSelectiveBarrierFlags.java +++ b/test/hotspot/jtreg/gc/shenandoah/options/TestSelectiveBarrierFlags.java @@ -43,13 +43,9 @@ public static void main(String[] args) throws Exception { String[][] opts = { new String[] { "ShenandoahKeepAliveBarrier" }, - new String[] { "ShenandoahWriteBarrier" }, - new String[] { "ShenandoahReadBarrier" }, - // StoreValRead+SATB are actually compatible, but we need to protect against - // StorveValEnqueue+SATB. TODO: Make it better. - new String[] { "ShenandoahSATBBarrier", "ShenandoahStoreValReadBarrier", "ShenandoahStoreValEnqueueBarrier" }, + new String[] { "ShenandoahLoadRefBarrier" }, + new String[] { "ShenandoahSATBBarrier", "ShenandoahStoreValEnqueueBarrier" }, new String[] { "ShenandoahCASBarrier" }, - new String[] { "ShenandoahAcmpBarrier" }, new String[] { "ShenandoahCloneBarrier" }, }; diff --git a/test/hotspot/jtreg/gc/shenandoah/options/TestWrongBarrierDisable.java b/test/hotspot/jtreg/gc/shenandoah/options/TestWrongBarrierDisable.java --- a/test/hotspot/jtreg/gc/shenandoah/options/TestWrongBarrierDisable.java +++ b/test/hotspot/jtreg/gc/shenandoah/options/TestWrongBarrierDisable.java @@ -38,21 +38,16 @@ public static void main(String[] args) throws Exception { String[] concurrent = { - "ShenandoahReadBarrier", - "ShenandoahWriteBarrier", + "ShenandoahLoadRefBarrier", "ShenandoahCASBarrier", - "ShenandoahAcmpBarrier", "ShenandoahCloneBarrier", "ShenandoahSATBBarrier", "ShenandoahKeepAliveBarrier", - "ShenandoahStoreValReadBarrier", }; String[] traversal = { - "ShenandoahReadBarrier", - "ShenandoahWriteBarrier", + "ShenandoahLoadRefBarrier", "ShenandoahCASBarrier", - "ShenandoahAcmpBarrier", "ShenandoahCloneBarrier", };