# HG changeset patch # User rkennke # Date 1554238822 -7200 # Tue Apr 02 23:00:22 2019 +0200 # Node ID ee9a3958b336abec385aaf75f055fd034cfb4e45 # Parent 9559ba212c1889e0c77e7bf6cc7a689632b87bcd 8221766: Load-reference barriers for Shenandoah diff --git a/make/hotspot/lib/JvmFeatures.gmk b/make/hotspot/lib/JvmFeatures.gmk --- a/make/hotspot/lib/JvmFeatures.gmk +++ b/make/hotspot/lib/JvmFeatures.gmk @@ -172,8 +172,6 @@ ifneq ($(call check-jvm-feature, shenandoahgc), true) JVM_CFLAGS_FEATURES += -DINCLUDE_SHENANDOAHGC=0 JVM_EXCLUDE_PATTERNS += gc/shenandoah -else - JVM_CFLAGS_FEATURES += -DSUPPORT_BARRIER_ON_PRIMITIVES -DSUPPORT_NOT_TO_SPACE_INVARIANT endif ifneq ($(call check-jvm-feature, jfr), true) diff --git a/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp --- a/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp @@ -40,7 +40,7 @@ #define __ masm-> -address ShenandoahBarrierSetAssembler::_shenandoah_wb = NULL; +address ShenandoahBarrierSetAssembler::_shenandoah_lrb = NULL; void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop, Register addr, Register count, RegSet saved_regs) { @@ -198,59 +198,43 @@ __ bind(done); } -void ShenandoahBarrierSetAssembler::read_barrier(MacroAssembler* masm, Register dst) { - if (ShenandoahReadBarrier) { - read_barrier_impl(masm, dst); - } -} - -void ShenandoahBarrierSetAssembler::read_barrier_impl(MacroAssembler* masm, Register dst) { - assert(UseShenandoahGC && (ShenandoahReadBarrier || ShenandoahStoreValReadBarrier || ShenandoahCASBarrier), "should be enabled"); +void ShenandoahBarrierSetAssembler::resolve_forward_pointer(MacroAssembler* masm, Register dst) { + assert(ShenandoahLoadRefBarrier || ShenandoahCASBarrier, "Should be enabled"); Label is_null; __ cbz(dst, is_null); - read_barrier_not_null_impl(masm, dst); + resolve_forward_pointer_not_null(masm, dst); __ bind(is_null); } -void ShenandoahBarrierSetAssembler::read_barrier_not_null(MacroAssembler* masm, Register dst) { - if (ShenandoahReadBarrier) { - read_barrier_not_null_impl(masm, dst); - } -} - - -void ShenandoahBarrierSetAssembler::read_barrier_not_null_impl(MacroAssembler* masm, Register dst) { - assert(UseShenandoahGC && (ShenandoahReadBarrier || ShenandoahStoreValReadBarrier || ShenandoahCASBarrier), "should be enabled"); +void ShenandoahBarrierSetAssembler::resolve_forward_pointer_not_null(MacroAssembler* masm, Register dst) { + assert(ShenandoahLoadRefBarrier || ShenandoahCASBarrier, "Should be enabled"); __ ldr(dst, Address(dst, ShenandoahBrooksPointer::byte_offset())); } -void ShenandoahBarrierSetAssembler::write_barrier(MacroAssembler* masm, Register dst) { - if (ShenandoahWriteBarrier) { - write_barrier_impl(masm, dst); +void ShenandoahBarrierSetAssembler::load_reference_barrier_not_null(MacroAssembler* masm, Register dst, Register tmp) { + assert(ShenandoahLoadRefBarrier, "Should be enabled"); + assert(dst != rscratch2, "need rscratch2"); + if (tmp == noreg) { + assert(dst != rscratch1, "need rscratch1"); + tmp = rscratch1; } -} - -void ShenandoahBarrierSetAssembler::write_barrier_impl(MacroAssembler* masm, Register dst) { - assert(UseShenandoahGC && (ShenandoahWriteBarrier || ShenandoahStoreValEnqueueBarrier), "Should be enabled"); - assert(dst != rscratch1, "need rscratch1"); - assert(dst != rscratch2, "need rscratch2"); Label done; Address gc_state(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset())); - __ ldrb(rscratch1, gc_state); + __ ldrb(tmp, gc_state); // Check for heap stability __ mov(rscratch2, ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::EVACUATION | ShenandoahHeap::TRAVERSAL); - __ tst(rscratch1, rscratch2); + __ tst(tmp, rscratch2); __ br(Assembler::EQ, done); - // Heap is unstable, need to perform the read-barrier even if WB is inactive - __ ldr(dst, Address(dst, ShenandoahBrooksPointer::byte_offset())); + // Heap is unstable, need to perform the resolve even if LRB is inactive + resolve_forward_pointer_not_null(masm, dst); - // Check for evacuation-in-progress and jump to WB slow-path if needed + // Check for evacuation-in-progress and jump to LRB slow-path if needed __ mov(rscratch2, ShenandoahHeap::EVACUATION | ShenandoahHeap::TRAVERSAL); - __ tst(rscratch1, rscratch2); + __ tst(tmp, rscratch2); __ br(Assembler::EQ, done); RegSet to_save = RegSet::of(r0); @@ -259,7 +243,7 @@ __ mov(r0, dst); } - __ far_call(RuntimeAddress(CAST_FROM_FN_PTR(address, ShenandoahBarrierSetAssembler::shenandoah_wb()))); + __ far_call(RuntimeAddress(CAST_FROM_FN_PTR(address, ShenandoahBarrierSetAssembler::shenandoah_lrb()))); if (dst != r0) { __ mov(dst, r0); @@ -271,10 +255,6 @@ void ShenandoahBarrierSetAssembler::storeval_barrier(MacroAssembler* masm, Register dst, Register tmp) { if (ShenandoahStoreValEnqueueBarrier) { - Label is_null; - __ cbz(dst, is_null); - write_barrier_impl(masm, dst); - __ bind(is_null); // Save possibly live regs. RegSet live_regs = RegSet::range(r0, r4) - dst; __ push(live_regs, sp); @@ -286,44 +266,45 @@ __ ldrd(v0, __ post(sp, 2 * wordSize)); __ pop(live_regs, sp); } - if (ShenandoahStoreValReadBarrier) { - read_barrier_impl(masm, dst); +} + +void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler* masm, Register dst, Register tmp) { + if (ShenandoahLoadRefBarrier) { + Label is_null; + __ cbz(dst, is_null); + load_reference_barrier_not_null(masm, dst, tmp); + __ bind(is_null); } } void ShenandoahBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Register dst, Address src, Register tmp1, Register tmp_thread) { bool on_oop = type == T_OBJECT || type == T_ARRAY; - bool in_heap = (decorators & IN_HEAP) != 0; bool on_weak = (decorators & ON_WEAK_OOP_REF) != 0; bool on_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0; bool on_reference = on_weak || on_phantom; - if (in_heap) { - read_barrier_not_null(masm, src.base()); - } + BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread); + if (on_oop) { + load_reference_barrier(masm, dst, tmp1); - BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread); - if (ShenandoahKeepAliveBarrier && on_oop && on_reference) { - __ enter(); - satb_write_barrier_pre(masm /* masm */, - noreg /* obj */, - dst /* pre_val */, - rthread /* thread */, - tmp1 /* tmp */, - true /* tosca_live */, - true /* expand_call */); - __ leave(); + if (ShenandoahKeepAliveBarrier && on_reference) { + __ enter(); + satb_write_barrier_pre(masm /* masm */, + noreg /* obj */, + dst /* pre_val */, + rthread /* thread */, + tmp1 /* tmp */, + true /* tosca_live */, + true /* expand_call */); + __ leave(); + } } } void ShenandoahBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Address dst, Register val, Register tmp1, Register tmp2) { bool on_oop = type == T_OBJECT || type == T_ARRAY; - bool in_heap = (decorators & IN_HEAP) != 0; - if (in_heap) { - write_barrier(masm, dst.base()); - } if (!on_oop) { BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2); return; @@ -361,21 +342,6 @@ } -void ShenandoahBarrierSetAssembler::obj_equals(MacroAssembler* masm, Register op1, Register op2) { - __ cmp(op1, op2); - if (ShenandoahAcmpBarrier) { - Label done; - __ br(Assembler::EQ, done); - // The object may have been evacuated, but we won't see it without a - // membar here. - __ membar(Assembler::LoadStore| Assembler::LoadLoad); - read_barrier(masm, op1); - read_barrier(masm, op2); - __ cmp(op1, op2); - __ bind(done); - } -} - void ShenandoahBarrierSetAssembler::tlab_allocate(MacroAssembler* masm, Register obj, Register var_size_in_bytes, int con_size_in_bytes, @@ -410,27 +376,6 @@ } } -void ShenandoahBarrierSetAssembler::resolve(MacroAssembler* masm, DecoratorSet decorators, Register obj) { - bool oop_not_null = (decorators & IS_NOT_NULL) != 0; - bool is_write = (decorators & ACCESS_WRITE) != 0; - if (is_write) { - if (oop_not_null) { - write_barrier(masm, obj); - } else { - Label done; - __ cbz(obj, done); - write_barrier(masm, obj); - __ bind(done); - } - } else { - if (oop_not_null) { - read_barrier_not_null(masm, obj); - } else { - read_barrier(masm, obj); - } - } -} - void ShenandoahBarrierSetAssembler::cmpxchg_oop(MacroAssembler* masm, Register addr, Register expected, Register new_val, bool acquire, bool release, bool weak, bool is_cae, Register result) { @@ -469,8 +414,8 @@ __ decode_heap_oop(tmp1, tmp1); __ decode_heap_oop(tmp2, tmp2); } - read_barrier_impl(masm, tmp1); - read_barrier_impl(masm, tmp2); + resolve_forward_pointer(masm, tmp1); + resolve_forward_pointer(masm, tmp2); __ cmp(tmp1, tmp2); // Retry with expected now being the value we just loaded from addr. __ br(Assembler::EQ, retry); @@ -515,7 +460,7 @@ __ b(*stub->continuation()); } -void ShenandoahBarrierSetAssembler::gen_write_barrier_stub(LIR_Assembler* ce, ShenandoahWriteBarrierStub* stub) { +void ShenandoahBarrierSetAssembler::gen_load_reference_barrier_stub(LIR_Assembler* ce, ShenandoahLoadReferenceBarrierStub* stub) { Register obj = stub->obj()->as_register(); Register res = stub->result()->as_register(); @@ -532,7 +477,7 @@ __ cbz(res, done); } - write_barrier(ce->masm(), res); + load_reference_barrier_not_null(ce->masm(), res, rscratch1); __ bind(done); __ b(*stub->continuation()); @@ -592,14 +537,14 @@ #endif // COMPILER1 -address ShenandoahBarrierSetAssembler::shenandoah_wb() { - assert(_shenandoah_wb != NULL, "need write barrier stub"); - return _shenandoah_wb; +address ShenandoahBarrierSetAssembler::shenandoah_lrb() { + assert(_shenandoah_lrb != NULL, "need load reference barrier stub"); + return _shenandoah_lrb; } #define __ cgen->assembler()-> -// Shenandoah write barrier. +// Shenandoah load reference barrier. // // Input: // r0: OOP to evacuate. Not null. @@ -608,10 +553,10 @@ // r0: Pointer to evacuated OOP. // // Trash rscratch1, rscratch2. Preserve everything else. -address ShenandoahBarrierSetAssembler::generate_shenandoah_wb(StubCodeGenerator* cgen) { +address ShenandoahBarrierSetAssembler::generate_shenandoah_lrb(StubCodeGenerator* cgen) { __ align(6); - StubCodeMark mark(cgen, "StubRoutines", "shenandoah_wb"); + StubCodeMark mark(cgen, "StubRoutines", "shenandoah_lrb"); address start = __ pc(); Label work; @@ -628,7 +573,7 @@ __ push_call_clobbered_registers(); - __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_JRT)); + __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_JRT)); __ blrt(lr, 1, 0, MacroAssembler::ret_type_integral); __ mov(rscratch1, obj); __ pop_call_clobbered_registers(); @@ -643,12 +588,12 @@ #undef __ void ShenandoahBarrierSetAssembler::barrier_stubs_init() { - if (ShenandoahWriteBarrier || ShenandoahStoreValEnqueueBarrier) { + if (ShenandoahLoadRefBarrier) { int stub_code_size = 2048; ResourceMark rm; BufferBlob* bb = BufferBlob::create("shenandoah_barrier_stubs", stub_code_size); CodeBuffer buf(bb); StubCodeGenerator cgen(&buf); - _shenandoah_wb = generate_shenandoah_wb(&cgen); + _shenandoah_lrb = generate_shenandoah_lrb(&cgen); } } diff --git a/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.hpp b/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.hpp --- a/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.hpp +++ b/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.hpp @@ -29,7 +29,7 @@ #ifdef COMPILER1 class LIR_Assembler; class ShenandoahPreBarrierStub; -class ShenandoahWriteBarrierStub; +class ShenandoahLoadReferenceBarrierStub; class StubAssembler; class StubCodeGenerator; #endif @@ -37,7 +37,7 @@ class ShenandoahBarrierSetAssembler: public BarrierSetAssembler { private: - static address _shenandoah_wb; + static address _shenandoah_lrb; void satb_write_barrier_pre(MacroAssembler* masm, Register obj, @@ -54,24 +54,21 @@ bool tosca_live, bool expand_call); - void read_barrier(MacroAssembler* masm, Register dst); - void read_barrier_impl(MacroAssembler* masm, Register dst); - void read_barrier_not_null(MacroAssembler* masm, Register dst); - void read_barrier_not_null_impl(MacroAssembler* masm, Register dst); - void write_barrier(MacroAssembler* masm, Register dst); - void write_barrier_impl(MacroAssembler* masm, Register dst); - void asm_acmp_barrier(MacroAssembler* masm, Register op1, Register op2); + void resolve_forward_pointer(MacroAssembler* masm, Register dst); + void resolve_forward_pointer_not_null(MacroAssembler* masm, Register dst); + void load_reference_barrier(MacroAssembler* masm, Register dst, Register tmp); + void load_reference_barrier_not_null(MacroAssembler* masm, Register dst, Register tmp); - address generate_shenandoah_wb(StubCodeGenerator* cgen); + address generate_shenandoah_lrb(StubCodeGenerator* cgen); public: - static address shenandoah_wb(); + static address shenandoah_lrb(); void storeval_barrier(MacroAssembler* masm, Register dst, Register tmp); #ifdef COMPILER1 void gen_pre_barrier_stub(LIR_Assembler* ce, ShenandoahPreBarrierStub* stub); - void gen_write_barrier_stub(LIR_Assembler* ce, ShenandoahWriteBarrierStub* stub); + void gen_load_reference_barrier_stub(LIR_Assembler* ce, ShenandoahLoadReferenceBarrierStub* stub); void generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm); #endif @@ -83,8 +80,6 @@ Register dst, Address src, Register tmp1, Register tmp_thread); virtual void store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Address dst, Register val, Register tmp1, Register tmp2); - virtual void obj_equals(MacroAssembler* masm, Register src1, Register src2); - virtual void resolve(MacroAssembler* masm, DecoratorSet decorators, Register obj); virtual void tlab_allocate(MacroAssembler* masm, Register obj, Register var_size_in_bytes, int con_size_in_bytes, diff --git a/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetC1_aarch64.cpp b/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetC1_aarch64.cpp --- a/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetC1_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetC1_aarch64.cpp @@ -99,6 +99,7 @@ __ xchg(access.resolved_addr(), value_opr, result, tmp); if (access.is_oop()) { + result = load_reference_barrier(access.gen(), result, access.access_emit_info(), true); if (ShenandoahSATBBarrier) { pre_barrier(access.gen(), access.access_emit_info(), access.decorators(), LIR_OprFact::illegalOpr, result /* pre_val */); diff --git a/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoah_aarch64.ad b/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoah_aarch64.ad --- a/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoah_aarch64.ad +++ b/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoah_aarch64.ad @@ -45,18 +45,6 @@ %} %} -instruct shenandoahRB(iRegPNoSp dst, iRegP src, rFlagsReg cr) %{ - match(Set dst (ShenandoahReadBarrier src)); - format %{ "shenandoah_rb $dst,$src" %} - ins_encode %{ - Register s = $src$$Register; - Register d = $dst$$Register; - __ ldr(d, Address(s, ShenandoahBrooksPointer::byte_offset())); - %} - ins_pipe(pipe_class_memory); -%} - - instruct compareAndSwapP_shenandoah(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp, rFlagsReg cr) %{ match(Set res (ShenandoahCompareAndSwapP mem (Binary oldval newval))); diff --git a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp --- a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp +++ b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp @@ -41,7 +41,7 @@ #define __ masm-> -address ShenandoahBarrierSetAssembler::_shenandoah_wb = NULL; +address ShenandoahBarrierSetAssembler::_shenandoah_lrb = NULL; void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Register src, Register dst, Register count) { @@ -293,41 +293,23 @@ __ bind(done); } -void ShenandoahBarrierSetAssembler::read_barrier(MacroAssembler* masm, Register dst) { - if (ShenandoahReadBarrier) { - read_barrier_impl(masm, dst); - } -} - -void ShenandoahBarrierSetAssembler::read_barrier_impl(MacroAssembler* masm, Register dst) { - assert(UseShenandoahGC && (ShenandoahReadBarrier || ShenandoahStoreValReadBarrier || ShenandoahCASBarrier), "should be enabled"); +void ShenandoahBarrierSetAssembler::resolve_forward_pointer(MacroAssembler* masm, Register dst) { + assert(ShenandoahCASBarrier, "should be enabled"); Label is_null; __ testptr(dst, dst); __ jcc(Assembler::zero, is_null); - read_barrier_not_null_impl(masm, dst); + resolve_forward_pointer_not_null(masm, dst); __ bind(is_null); } -void ShenandoahBarrierSetAssembler::read_barrier_not_null(MacroAssembler* masm, Register dst) { - if (ShenandoahReadBarrier) { - read_barrier_not_null_impl(masm, dst); - } -} - -void ShenandoahBarrierSetAssembler::read_barrier_not_null_impl(MacroAssembler* masm, Register dst) { - assert(UseShenandoahGC && (ShenandoahReadBarrier || ShenandoahStoreValReadBarrier || ShenandoahCASBarrier), "should be enabled"); +void ShenandoahBarrierSetAssembler::resolve_forward_pointer_not_null(MacroAssembler* masm, Register dst) { + assert(ShenandoahCASBarrier || ShenandoahLoadRefBarrier, "should be enabled"); __ movptr(dst, Address(dst, ShenandoahBrooksPointer::byte_offset())); } -void ShenandoahBarrierSetAssembler::write_barrier(MacroAssembler* masm, Register dst) { - if (ShenandoahWriteBarrier) { - write_barrier_impl(masm, dst); - } -} - -void ShenandoahBarrierSetAssembler::write_barrier_impl(MacroAssembler* masm, Register dst) { - assert(UseShenandoahGC && (ShenandoahWriteBarrier || ShenandoahStoreValEnqueueBarrier), "Should be enabled"); +void ShenandoahBarrierSetAssembler::load_reference_barrier_not_null(MacroAssembler* masm, Register dst) { + assert(ShenandoahLoadRefBarrier, "Should be enabled"); #ifdef _LP64 Label done; @@ -335,8 +317,8 @@ __ testb(gc_state, ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::EVACUATION | ShenandoahHeap::TRAVERSAL); __ jccb(Assembler::zero, done); - // Heap is unstable, need to perform the read-barrier even if WB is inactive - read_barrier_not_null(masm, dst); + // Heap is unstable, need to perform the resolve even if LRB is inactive + resolve_forward_pointer_not_null(masm, dst); __ testb(gc_state, ShenandoahHeap::EVACUATION | ShenandoahHeap::TRAVERSAL); __ jccb(Assembler::zero, done); @@ -345,7 +327,7 @@ __ xchgptr(dst, rax); // Move obj into rax and save rax into obj. } - __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, ShenandoahBarrierSetAssembler::shenandoah_wb()))); + __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, ShenandoahBarrierSetAssembler::shenandoah_lrb()))); if (dst != rax) { __ xchgptr(rax, dst); // Swap back obj with rax. @@ -358,24 +340,18 @@ } void ShenandoahBarrierSetAssembler::storeval_barrier(MacroAssembler* masm, Register dst, Register tmp) { - if (ShenandoahStoreValReadBarrier || ShenandoahStoreValEnqueueBarrier) { + if (ShenandoahStoreValEnqueueBarrier) { storeval_barrier_impl(masm, dst, tmp); } } void ShenandoahBarrierSetAssembler::storeval_barrier_impl(MacroAssembler* masm, Register dst, Register tmp) { - assert(UseShenandoahGC && (ShenandoahStoreValReadBarrier || ShenandoahStoreValEnqueueBarrier), "should be enabled"); + assert(ShenandoahStoreValEnqueueBarrier, "should be enabled"); if (dst == noreg) return; #ifdef _LP64 if (ShenandoahStoreValEnqueueBarrier) { - Label is_null; - __ testptr(dst, dst); - __ jcc(Assembler::zero, is_null); - write_barrier_impl(masm, dst); - __ bind(is_null); - // The set of registers to be saved+restored is the same as in the write-barrier above. // Those are the commonly used registers in the interpreter. __ pusha(); @@ -389,50 +365,54 @@ //__ pop_callee_saved_registers(); __ popa(); } - if (ShenandoahStoreValReadBarrier) { - read_barrier_impl(masm, dst); - } #else Unimplemented(); #endif } +void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler* masm, Register dst) { + if (ShenandoahLoadRefBarrier) { + Label done; + __ testptr(dst, dst); + __ jcc(Assembler::zero, done); + load_reference_barrier_not_null(masm, dst); + __ bind(done); + } +} + void ShenandoahBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Register dst, Address src, Register tmp1, Register tmp_thread) { bool on_oop = type == T_OBJECT || type == T_ARRAY; - bool in_heap = (decorators & IN_HEAP) != 0; bool on_weak = (decorators & ON_WEAK_OOP_REF) != 0; bool on_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0; bool on_reference = on_weak || on_phantom; - if (in_heap) { - read_barrier_not_null(masm, src.base()); - } - BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread); - if (ShenandoahKeepAliveBarrier && on_oop && on_reference) { - const Register thread = NOT_LP64(tmp_thread) LP64_ONLY(r15_thread); - NOT_LP64(__ get_thread(thread)); + BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread); + if (on_oop) { + load_reference_barrier(masm, dst); - // Generate the SATB pre-barrier code to log the value of - // the referent field in an SATB buffer. - shenandoah_write_barrier_pre(masm /* masm */, - noreg /* obj */, - dst /* pre_val */, - thread /* thread */, - tmp1 /* tmp */, - true /* tosca_live */, - true /* expand_call */); + if (ShenandoahKeepAliveBarrier && on_reference) { + const Register thread = NOT_LP64(tmp_thread) LP64_ONLY(r15_thread); + NOT_LP64(__ get_thread(thread)); + // Generate the SATB pre-barrier code to log the value of + // the referent field in an SATB buffer. + shenandoah_write_barrier_pre(masm /* masm */, + noreg /* obj */, + dst /* pre_val */, + thread /* thread */, + tmp1 /* tmp */, + true /* tosca_live */, + true /* expand_call */); + } } } void ShenandoahBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Address dst, Register val, Register tmp1, Register tmp2) { + bool on_oop = type == T_OBJECT || type == T_ARRAY; bool in_heap = (decorators & IN_HEAP) != 0; bool as_normal = (decorators & AS_NORMAL) != 0; - if (in_heap) { - write_barrier(masm, dst.base()); - } - if (type == T_OBJECT || type == T_ARRAY) { + if (on_oop && in_heap) { bool needs_pre_barrier = as_normal; Register tmp3 = LP64_ONLY(r8) NOT_LP64(rsi); @@ -475,44 +455,6 @@ } } -#ifndef _LP64 -void ShenandoahBarrierSetAssembler::obj_equals(MacroAssembler* masm, - Address obj1, jobject obj2) { - Unimplemented(); -} - -void ShenandoahBarrierSetAssembler::obj_equals(MacroAssembler* masm, - Register obj1, jobject obj2) { - Unimplemented(); -} -#endif - - -void ShenandoahBarrierSetAssembler::obj_equals(MacroAssembler* masm, Register op1, Register op2) { - __ cmpptr(op1, op2); - if (ShenandoahAcmpBarrier) { - Label done; - __ jccb(Assembler::equal, done); - read_barrier(masm, op1); - read_barrier(masm, op2); - __ cmpptr(op1, op2); - __ bind(done); - } -} - -void ShenandoahBarrierSetAssembler::obj_equals(MacroAssembler* masm, Register src1, Address src2) { - __ cmpptr(src1, src2); - if (ShenandoahAcmpBarrier) { - Label done; - __ jccb(Assembler::equal, done); - __ movptr(rscratch2, src2); - read_barrier(masm, src1); - read_barrier(masm, rscratch2); - __ cmpptr(src1, rscratch2); - __ bind(done); - } -} - void ShenandoahBarrierSetAssembler::tlab_allocate(MacroAssembler* masm, Register thread, Register obj, Register var_size_in_bytes, @@ -562,28 +504,6 @@ __ verify_tlab(); } -void ShenandoahBarrierSetAssembler::resolve(MacroAssembler* masm, DecoratorSet decorators, Register obj) { - bool oop_not_null = (decorators & IS_NOT_NULL) != 0; - bool is_write = (decorators & ACCESS_WRITE) != 0; - if (is_write) { - if (oop_not_null) { - write_barrier(masm, obj); - } else { - Label done; - __ testptr(obj, obj); - __ jcc(Assembler::zero, done); - write_barrier(masm, obj); - __ bind(done); - } - } else { - if (oop_not_null) { - read_barrier_not_null(masm, obj); - } else { - read_barrier(masm, obj); - } - } -} - // Special Shenandoah CAS implementation that handles false negatives // due to concurrent evacuation. #ifndef _LP64 @@ -622,14 +542,14 @@ // Step 2. CAS had failed. This may be a false negative. // // The trouble comes when we compare the to-space pointer with the from-space - // pointer to the same object. To resolve this, it will suffice to read both - // oldval and the value from memory through the read barriers -- this will give - // both to-space pointers. If they mismatch, then it was a legitimate failure. + // pointer to the same object. To resolve this, it will suffice to resolve both + // oldval and the value from memory -- this will give both to-space pointers. + // If they mismatch, then it was a legitimate failure. // if (UseCompressedOops) { __ decode_heap_oop(tmp1); } - read_barrier_impl(masm, tmp1); + resolve_forward_pointer(masm, tmp1); if (UseCompressedOops) { __ movl(tmp2, oldval); @@ -637,7 +557,7 @@ } else { __ movptr(tmp2, oldval); } - read_barrier_impl(masm, tmp2); + resolve_forward_pointer(masm, tmp2); __ cmpptr(tmp1, tmp2); __ jcc(Assembler::notEqual, done, true); @@ -646,8 +566,8 @@ // // Corner case: it may happen that somebody stored the from-space pointer // to memory while we were preparing for retry. Therefore, we can fail again - // on retry, and so need to do this in loop, always re-reading the failure - // witness through the read barrier. + // on retry, and so need to do this in loop, always resolving the failure + // witness. __ bind(retry); if (os::is_MP()) __ lock(); if (UseCompressedOops) { @@ -663,7 +583,7 @@ } else { __ movptr(tmp2, oldval); } - read_barrier_impl(masm, tmp2); + resolve_forward_pointer(masm, tmp2); __ cmpptr(tmp1, tmp2); __ jcc(Assembler::equal, retry, true); @@ -811,7 +731,7 @@ } -void ShenandoahBarrierSetAssembler::gen_write_barrier_stub(LIR_Assembler* ce, ShenandoahWriteBarrierStub* stub) { +void ShenandoahBarrierSetAssembler::gen_load_reference_barrier_stub(LIR_Assembler* ce, ShenandoahLoadReferenceBarrierStub* stub) { __ bind(*stub->entry()); Label done; @@ -828,7 +748,7 @@ __ jcc(Assembler::zero, done); } - write_barrier(ce->masm(), res); + load_reference_barrier_not_null(ce->masm(), res); __ bind(done); __ jmp(*stub->continuation()); @@ -898,16 +818,16 @@ #endif // COMPILER1 -address ShenandoahBarrierSetAssembler::shenandoah_wb() { - assert(_shenandoah_wb != NULL, "need write barrier stub"); - return _shenandoah_wb; +address ShenandoahBarrierSetAssembler::shenandoah_lrb() { + assert(_shenandoah_lrb != NULL, "need load reference barrier stub"); + return _shenandoah_lrb; } #define __ cgen->assembler()-> -address ShenandoahBarrierSetAssembler::generate_shenandoah_wb(StubCodeGenerator* cgen) { +address ShenandoahBarrierSetAssembler::generate_shenandoah_lrb(StubCodeGenerator* cgen) { __ align(CodeEntryAlignment); - StubCodeMark mark(cgen, "StubRoutines", "shenandoah_wb"); + StubCodeMark mark(cgen, "StubRoutines", "shenandoah_lrb"); address start = __ pc(); #ifdef _LP64 @@ -955,7 +875,7 @@ __ push(r15); save_vector_registers(cgen->assembler()); __ movptr(rdi, rax); - __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_JRT), rdi); + __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_JRT), rdi); restore_vector_registers(cgen->assembler()); __ pop(r15); __ pop(r14); @@ -982,12 +902,12 @@ #undef __ void ShenandoahBarrierSetAssembler::barrier_stubs_init() { - if (ShenandoahWriteBarrier || ShenandoahStoreValEnqueueBarrier) { + if (ShenandoahLoadRefBarrier) { int stub_code_size = 4096; ResourceMark rm; BufferBlob* bb = BufferBlob::create("shenandoah_barrier_stubs", stub_code_size); CodeBuffer buf(bb); StubCodeGenerator cgen(&buf); - _shenandoah_wb = generate_shenandoah_wb(&cgen); + _shenandoah_lrb = generate_shenandoah_lrb(&cgen); } } diff --git a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.hpp b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.hpp --- a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.hpp +++ b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.hpp @@ -29,7 +29,7 @@ #ifdef COMPILER1 class LIR_Assembler; class ShenandoahPreBarrierStub; -class ShenandoahWriteBarrierStub; +class ShenandoahLoadReferenceBarrierStub; class StubAssembler; class StubCodeGenerator; #endif @@ -37,7 +37,7 @@ class ShenandoahBarrierSetAssembler: public BarrierSetAssembler { private: - static address _shenandoah_wb; + static address _shenandoah_lrb; void satb_write_barrier_pre(MacroAssembler* masm, Register obj, @@ -55,32 +55,30 @@ bool tosca_live, bool expand_call); - void read_barrier(MacroAssembler* masm, Register dst); - void read_barrier_impl(MacroAssembler* masm, Register dst); + void resolve_forward_pointer(MacroAssembler* masm, Register dst); + void resolve_forward_pointer_not_null(MacroAssembler* masm, Register dst); - void read_barrier_not_null(MacroAssembler* masm, Register dst); - void read_barrier_not_null_impl(MacroAssembler* masm, Register dst); - - void write_barrier(MacroAssembler* masm, Register dst); - void write_barrier_impl(MacroAssembler* masm, Register dst); + void load_reference_barrier_not_null(MacroAssembler* masm, Register dst); void storeval_barrier_impl(MacroAssembler* masm, Register dst, Register tmp); - address generate_shenandoah_wb(StubCodeGenerator* cgen); + address generate_shenandoah_lrb(StubCodeGenerator* cgen); void save_vector_registers(MacroAssembler* masm); void restore_vector_registers(MacroAssembler* masm); public: - static address shenandoah_wb(); + static address shenandoah_lrb(); void storeval_barrier(MacroAssembler* masm, Register dst, Register tmp); #ifdef COMPILER1 void gen_pre_barrier_stub(LIR_Assembler* ce, ShenandoahPreBarrierStub* stub); - void gen_write_barrier_stub(LIR_Assembler* ce, ShenandoahWriteBarrierStub* stub); + void gen_load_reference_barrier_stub(LIR_Assembler* ce, ShenandoahLoadReferenceBarrierStub* stub); void generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm); #endif + void load_reference_barrier(MacroAssembler* masm, Register dst); + void cmpxchg_oop(MacroAssembler* masm, Register res, Address addr, Register oldval, Register newval, bool exchange, Register tmp1, Register tmp2); @@ -93,16 +91,6 @@ virtual void store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Address dst, Register val, Register tmp1, Register tmp2); -#ifndef _LP64 - virtual void obj_equals(MacroAssembler* masm, - Address obj1, jobject obj2); - virtual void obj_equals(MacroAssembler* masm, - Register obj1, jobject obj2); -#endif - - virtual void obj_equals(MacroAssembler* masm, Register src1, Register src2); - virtual void obj_equals(MacroAssembler* masm, Register src1, Address src2); - virtual void tlab_allocate(MacroAssembler* masm, Register thread, Register obj, Register var_size_in_bytes, @@ -110,8 +98,6 @@ Register t1, Register t2, Label& slow_case); - virtual void resolve(MacroAssembler* masm, DecoratorSet decorators, Register obj); - virtual void barrier_stubs_init(); }; diff --git a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetC1_x86.cpp b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetC1_x86.cpp --- a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetC1_x86.cpp +++ b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetC1_x86.cpp @@ -107,6 +107,7 @@ __ xchg(access.resolved_addr(), result, result, LIR_OprFact::illegalOpr); if (access.is_oop()) { + result = load_reference_barrier(access.gen(), result, access.access_emit_info(), true); if (ShenandoahSATBBarrier) { pre_barrier(access.gen(), access.access_emit_info(), access.decorators(), LIR_OprFact::illegalOpr, result /* pre_val */); diff --git a/src/hotspot/cpu/x86/gc/shenandoah/shenandoah_x86_64.ad b/src/hotspot/cpu/x86/gc/shenandoah/shenandoah_x86_64.ad --- a/src/hotspot/cpu/x86/gc/shenandoah/shenandoah_x86_64.ad +++ b/src/hotspot/cpu/x86/gc/shenandoah/shenandoah_x86_64.ad @@ -23,47 +23,7 @@ source_hpp %{ #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp" -%} - -instruct shenandoahRB(rRegP dst, rRegP src, rFlagsReg cr) %{ - match(Set dst (ShenandoahReadBarrier src)); - effect(DEF dst, USE src); - ins_cost(125); // XXX - format %{ "shenandoah_rb $dst, $src" %} - ins_encode %{ - Register d = $dst$$Register; - Register s = $src$$Register; - __ movptr(d, Address(s, ShenandoahBrooksPointer::byte_offset())); - %} - ins_pipe(ialu_reg_mem); -%} - -instruct shenandoahRBNarrow(rRegP dst, rRegN src) %{ - predicate(UseCompressedOops && (Universe::narrow_oop_shift() == 0)); - match(Set dst (ShenandoahReadBarrier (DecodeN src))); - effect(DEF dst, USE src); - ins_cost(125); // XXX - format %{ "shenandoah_rb $dst, $src" %} - ins_encode %{ - Register d = $dst$$Register; - Register s = $src$$Register; - __ movptr(d, Address(r12, s, Address::times_1, ShenandoahBrooksPointer::byte_offset())); - %} - ins_pipe(ialu_reg_mem); -%} - -instruct shenandoahRBNarrowShift(rRegP dst, rRegN src) %{ - predicate(UseCompressedOops && (Universe::narrow_oop_shift() == Address::times_8)); - match(Set dst (ShenandoahReadBarrier (DecodeN src))); - effect(DEF dst, USE src); - ins_cost(125); // XXX - format %{ "shenandoah_rb $dst, $src" %} - ins_encode %{ - Register d = $dst$$Register; - Register s = $src$$Register; - __ movptr(d, Address(r12, s, Address::times_8, ShenandoahBrooksPointer::byte_offset())); - %} - ins_pipe(ialu_reg_mem); +#include "gc/shenandoah/c2/shenandoahSupport.hpp" %} instruct compareAndSwapP_shenandoah(rRegI res, diff --git a/src/hotspot/share/adlc/formssel.cpp b/src/hotspot/share/adlc/formssel.cpp --- a/src/hotspot/share/adlc/formssel.cpp +++ b/src/hotspot/share/adlc/formssel.cpp @@ -777,8 +777,7 @@ !strcmp(_matrule->_rChild->_opType,"CompareAndExchangeP") || !strcmp(_matrule->_rChild->_opType,"CompareAndExchangeN") || !strcmp(_matrule->_rChild->_opType,"ShenandoahCompareAndExchangeP") || - !strcmp(_matrule->_rChild->_opType,"ShenandoahCompareAndExchangeN") || - !strcmp(_matrule->_rChild->_opType,"ShenandoahReadBarrier"))) return true; + !strcmp(_matrule->_rChild->_opType,"ShenandoahCompareAndExchangeN"))) return true; else if ( is_ideal_load() == Form::idealP ) return true; else if ( is_ideal_store() != Form::none ) return true; @@ -3506,7 +3505,6 @@ "ClearArray", "GetAndSetB", "GetAndSetS", "GetAndAddI", "GetAndSetI", "GetAndSetP", "GetAndAddB", "GetAndAddS", "GetAndAddL", "GetAndSetL", "GetAndSetN", - "ShenandoahReadBarrier", "LoadBarrierSlowReg", "LoadBarrierWeakSlowReg" }; int cnt = sizeof(needs_ideal_memory_list)/sizeof(char*); diff --git a/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.cpp b/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.cpp --- a/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.cpp +++ b/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.cpp @@ -46,9 +46,9 @@ bs->gen_pre_barrier_stub(ce, this); } -void ShenandoahWriteBarrierStub::emit_code(LIR_Assembler* ce) { +void ShenandoahLoadReferenceBarrierStub::emit_code(LIR_Assembler* ce) { ShenandoahBarrierSetAssembler* bs = (ShenandoahBarrierSetAssembler*)BarrierSet::barrier_set()->barrier_set_assembler(); - bs->gen_write_barrier_stub(ce, this); + bs->gen_load_reference_barrier_stub(ce, this); } void ShenandoahBarrierSetC1::pre_barrier(LIRGenerator* gen, CodeEmitInfo* info, DecoratorSet decorators, LIR_Opr addr_opr, LIR_Opr pre_val) { @@ -105,40 +105,16 @@ __ branch_destination(slow->continuation()); } -LIR_Opr ShenandoahBarrierSetC1::read_barrier(LIRGenerator* gen, LIR_Opr obj, CodeEmitInfo* info, bool need_null_check) { - if (UseShenandoahGC && ShenandoahReadBarrier) { - return read_barrier_impl(gen, obj, info, need_null_check); +LIR_Opr ShenandoahBarrierSetC1::load_reference_barrier(LIRGenerator* gen, LIR_Opr obj, CodeEmitInfo* info, bool need_null_check) { + if (ShenandoahLoadRefBarrier) { + return load_reference_barrier_impl(gen, obj, info, need_null_check); } else { return obj; } } -LIR_Opr ShenandoahBarrierSetC1::read_barrier_impl(LIRGenerator* gen, LIR_Opr obj, CodeEmitInfo* info, bool need_null_check) { - assert(UseShenandoahGC && (ShenandoahReadBarrier || ShenandoahStoreValReadBarrier), "Should be enabled"); - LabelObj* done = new LabelObj(); - LIR_Opr result = gen->new_register(T_OBJECT); - __ move(obj, result); - if (need_null_check) { - __ cmp(lir_cond_equal, result, LIR_OprFact::oopConst(NULL)); - __ branch(lir_cond_equal, T_LONG, done->label()); - } - LIR_Address* brooks_ptr_address = gen->generate_address(result, ShenandoahBrooksPointer::byte_offset(), T_ADDRESS); - __ load(brooks_ptr_address, result, info ? new CodeEmitInfo(info) : NULL, lir_patch_none); - - __ branch_destination(done->label()); - return result; -} - -LIR_Opr ShenandoahBarrierSetC1::write_barrier(LIRGenerator* gen, LIR_Opr obj, CodeEmitInfo* info, bool need_null_check) { - if (UseShenandoahGC && ShenandoahWriteBarrier) { - return write_barrier_impl(gen, obj, info, need_null_check); - } else { - return obj; - } -} - -LIR_Opr ShenandoahBarrierSetC1::write_barrier_impl(LIRGenerator* gen, LIR_Opr obj, CodeEmitInfo* info, bool need_null_check) { - assert(UseShenandoahGC && (ShenandoahWriteBarrier || ShenandoahStoreValEnqueueBarrier), "Should be enabled"); +LIR_Opr ShenandoahBarrierSetC1::load_reference_barrier_impl(LIRGenerator* gen, LIR_Opr obj, CodeEmitInfo* info, bool need_null_check) { + assert(ShenandoahLoadRefBarrier, "Should be enabled"); obj = ensure_in_register(gen, obj); assert(obj->is_register(), "must be a register at this point"); @@ -168,7 +144,7 @@ } __ cmp(lir_cond_notEqual, flag_val, LIR_OprFact::intConst(0)); - CodeStub* slow = new ShenandoahWriteBarrierStub(obj, result, info ? new CodeEmitInfo(info) : NULL, need_null_check); + CodeStub* slow = new ShenandoahLoadReferenceBarrierStub(obj, result, info ? new CodeEmitInfo(info) : NULL, need_null_check); __ branch(lir_cond_notEqual, T_INT, slow); __ branch_destination(slow->continuation()); @@ -189,58 +165,13 @@ } LIR_Opr ShenandoahBarrierSetC1::storeval_barrier(LIRGenerator* gen, LIR_Opr obj, CodeEmitInfo* info, DecoratorSet decorators) { - bool need_null_check = (decorators & IS_NOT_NULL) == 0; if (ShenandoahStoreValEnqueueBarrier) { - obj = write_barrier_impl(gen, obj, info, need_null_check); + obj = ensure_in_register(gen, obj); pre_barrier(gen, info, decorators, LIR_OprFact::illegalOpr, obj); } - if (ShenandoahStoreValReadBarrier) { - obj = read_barrier_impl(gen, obj, info, true /*need_null_check*/); - } return obj; } -LIR_Opr ShenandoahBarrierSetC1::resolve_address(LIRAccess& access, bool resolve_in_register) { - DecoratorSet decorators = access.decorators(); - bool is_array = (decorators & IS_ARRAY) != 0; - bool needs_patching = (decorators & C1_NEEDS_PATCHING) != 0; - - bool is_write = (decorators & ACCESS_WRITE) != 0; - bool needs_null_check = (decorators & IS_NOT_NULL) == 0; - - LIR_Opr base = access.base().item().result(); - LIR_Opr offset = access.offset().opr(); - LIRGenerator* gen = access.gen(); - - if (is_write) { - base = write_barrier(gen, base, access.access_emit_info(), needs_null_check); - } else { - base = read_barrier(gen, base, access.access_emit_info(), needs_null_check); - } - - LIR_Opr addr_opr; - if (is_array) { - addr_opr = LIR_OprFact::address(gen->emit_array_address(base, offset, access.type())); - } else if (needs_patching) { - // we need to patch the offset in the instruction so don't allow - // generate_address to try to be smart about emitting the -1. - // Otherwise the patching code won't know how to find the - // instruction to patch. - addr_opr = LIR_OprFact::address(new LIR_Address(base, PATCHED_ADDR, access.type())); - } else { - addr_opr = LIR_OprFact::address(gen->generate_address(base, offset, 0, 0, access.type())); - } - - if (resolve_in_register) { - LIR_Opr resolved_addr = gen->new_pointer_register(); - __ leal(addr_opr, resolved_addr); - resolved_addr = LIR_OprFact::address(new LIR_Address(resolved_addr, access.type())); - return resolved_addr; - } else { - return addr_opr; - } -} - void ShenandoahBarrierSetC1::store_at_resolved(LIRAccess& access, LIR_Opr value) { if (access.is_oop()) { if (ShenandoahSATBBarrier) { @@ -252,15 +183,28 @@ } void ShenandoahBarrierSetC1::load_at_resolved(LIRAccess& access, LIR_Opr result) { - BarrierSetC1::load_at_resolved(access, result); + if (!access.is_oop()) { + BarrierSetC1::load_at_resolved(access, result); + return; + } + + LIRGenerator *gen = access.gen(); + + if (ShenandoahLoadRefBarrier) { + LIR_Opr tmp = gen->new_register(T_OBJECT); + BarrierSetC1::load_at_resolved(access, tmp); + tmp = load_reference_barrier(access.gen(), tmp, access.access_emit_info(), true); + __ move(tmp, result); + } else { + BarrierSetC1::load_at_resolved(access, result); + } if (ShenandoahKeepAliveBarrier) { DecoratorSet decorators = access.decorators(); bool is_weak = (decorators & ON_WEAK_OOP_REF) != 0; bool is_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0; bool is_anonymous = (decorators & ON_UNKNOWN_OOP_REF) != 0; - LIRGenerator *gen = access.gen(); - if (access.is_oop() && (is_weak || is_phantom || is_anonymous)) { + if (is_weak || is_phantom || is_anonymous) { // Register the value in the referent field with the pre-barrier LabelObj *Lcont_anonymous; if (is_anonymous) { @@ -276,19 +220,6 @@ } } -LIR_Opr ShenandoahBarrierSetC1::atomic_add_at_resolved(LIRAccess& access, LIRItem& value) { - return BarrierSetC1::atomic_add_at_resolved(access, value); -} - -LIR_Opr ShenandoahBarrierSetC1::resolve(LIRGenerator* gen, DecoratorSet decorators, LIR_Opr obj) { - bool is_write = decorators & ACCESS_WRITE; - if (is_write) { - return write_barrier(gen, obj, NULL, (decorators & IS_NOT_NULL) == 0); - } else { - return read_barrier(gen, obj, NULL, (decorators & IS_NOT_NULL) == 0); - } -} - class C1ShenandoahPreBarrierCodeGenClosure : public StubAssemblerCodeGenClosure { virtual OopMapSet* generate_code(StubAssembler* sasm) { ShenandoahBarrierSetAssembler* bs = (ShenandoahBarrierSetAssembler*)BarrierSet::barrier_set()->barrier_set_assembler(); diff --git a/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.hpp b/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.hpp --- a/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.hpp +++ b/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.hpp @@ -85,7 +85,7 @@ #endif // PRODUCT }; -class ShenandoahWriteBarrierStub: public CodeStub { +class ShenandoahLoadReferenceBarrierStub: public CodeStub { friend class ShenandoahBarrierSetC1; private: LIR_Opr _obj; @@ -94,7 +94,7 @@ bool _needs_null_check; public: - ShenandoahWriteBarrierStub(LIR_Opr obj, LIR_Opr result, CodeEmitInfo* info, bool needs_null_check) : + ShenandoahLoadReferenceBarrierStub(LIR_Opr obj, LIR_Opr result, CodeEmitInfo* info, bool needs_null_check) : _obj(obj), _result(result), _info(info), _needs_null_check(needs_null_check) { assert(_obj->is_register(), "should be register"); @@ -113,7 +113,7 @@ visitor->do_temp(_result); } #ifndef PRODUCT - virtual void print_name(outputStream* out) const { out->print("ShenandoahWritePreBarrierStub"); } + virtual void print_name(outputStream* out) const { out->print("ShenandoahLoadReferenceBarrierStub"); } #endif // PRODUCT }; @@ -181,12 +181,10 @@ void pre_barrier(LIRGenerator* gen, CodeEmitInfo* info, DecoratorSet decorators, LIR_Opr addr_opr, LIR_Opr pre_val); - LIR_Opr read_barrier(LIRGenerator* gen, LIR_Opr obj, CodeEmitInfo* info, bool need_null_check); - LIR_Opr write_barrier(LIRGenerator* gen, LIR_Opr obj, CodeEmitInfo* info, bool need_null_check); + LIR_Opr load_reference_barrier(LIRGenerator* gen, LIR_Opr obj, CodeEmitInfo* info, bool need_null_check); LIR_Opr storeval_barrier(LIRGenerator* gen, LIR_Opr obj, CodeEmitInfo* info, DecoratorSet decorators); - LIR_Opr read_barrier_impl(LIRGenerator* gen, LIR_Opr obj, CodeEmitInfo* info, bool need_null_check); - LIR_Opr write_barrier_impl(LIRGenerator* gen, LIR_Opr obj, CodeEmitInfo* info, bool need_null_check); + LIR_Opr load_reference_barrier_impl(LIRGenerator* gen, LIR_Opr obj, CodeEmitInfo* info, bool need_null_check); LIR_Opr ensure_in_register(LIRGenerator* gen, LIR_Opr obj); @@ -194,7 +192,6 @@ CodeBlob* pre_barrier_c1_runtime_code_blob() { return _pre_barrier_c1_runtime_code_blob; } protected: - virtual LIR_Opr resolve_address(LIRAccess& access, bool resolve_in_register); virtual void store_at_resolved(LIRAccess& access, LIR_Opr value); virtual void load_at_resolved(LIRAccess& access, LIR_Opr result); @@ -202,10 +199,8 @@ virtual LIR_Opr atomic_cmpxchg_at_resolved(LIRAccess& access, LIRItem& cmp_value, LIRItem& new_value); virtual LIR_Opr atomic_xchg_at_resolved(LIRAccess& access, LIRItem& value); - virtual LIR_Opr atomic_add_at_resolved(LIRAccess& access, LIRItem& value); public: - virtual LIR_Opr resolve(LIRGenerator* gen, DecoratorSet decorators, LIR_Opr obj); virtual void generate_c1_runtime_stubs(BufferBlob* buffer_blob); }; diff --git a/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp b/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp --- a/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp +++ b/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp @@ -43,121 +43,56 @@ } ShenandoahBarrierSetC2State::ShenandoahBarrierSetC2State(Arena* comp_arena) - : _shenandoah_barriers(new (comp_arena) GrowableArray(comp_arena, 8, 0, NULL)) { + : _enqueue_barriers(new (comp_arena) GrowableArray(comp_arena, 8, 0, NULL)), + _load_reference_barriers(new (comp_arena) GrowableArray(comp_arena, 8, 0, NULL)) { } -int ShenandoahBarrierSetC2State::shenandoah_barriers_count() const { - return _shenandoah_barriers->length(); +int ShenandoahBarrierSetC2State::enqueue_barriers_count() const { + return _enqueue_barriers->length(); } -ShenandoahWriteBarrierNode* ShenandoahBarrierSetC2State::shenandoah_barrier(int idx) const { - return _shenandoah_barriers->at(idx); +ShenandoahEnqueueBarrierNode* ShenandoahBarrierSetC2State::enqueue_barrier(int idx) const { + return _enqueue_barriers->at(idx); } -void ShenandoahBarrierSetC2State::add_shenandoah_barrier(ShenandoahWriteBarrierNode * n) { - assert(!_shenandoah_barriers->contains(n), "duplicate entry in barrier list"); - _shenandoah_barriers->append(n); +void ShenandoahBarrierSetC2State::add_enqueue_barrier(ShenandoahEnqueueBarrierNode * n) { + assert(!_enqueue_barriers->contains(n), "duplicate entry in barrier list"); + _enqueue_barriers->append(n); } -void ShenandoahBarrierSetC2State::remove_shenandoah_barrier(ShenandoahWriteBarrierNode * n) { - if (_shenandoah_barriers->contains(n)) { - _shenandoah_barriers->remove(n); +void ShenandoahBarrierSetC2State::remove_enqueue_barrier(ShenandoahEnqueueBarrierNode * n) { + if (_enqueue_barriers->contains(n)) { + _enqueue_barriers->remove(n); } } -#define __ kit-> +int ShenandoahBarrierSetC2State::load_reference_barriers_count() const { + return _load_reference_barriers->length(); +} -Node* ShenandoahBarrierSetC2::shenandoah_read_barrier(GraphKit* kit, Node* obj) const { - if (ShenandoahReadBarrier) { - obj = shenandoah_read_barrier_impl(kit, obj, false, true, true); +ShenandoahLoadReferenceBarrierNode* ShenandoahBarrierSetC2State::load_reference_barrier(int idx) const { + return _load_reference_barriers->at(idx); +} + +void ShenandoahBarrierSetC2State::add_load_reference_barrier(ShenandoahLoadReferenceBarrierNode * n) { + assert(!_load_reference_barriers->contains(n), "duplicate entry in barrier list"); + _load_reference_barriers->append(n); +} + +void ShenandoahBarrierSetC2State::remove_load_reference_barrier(ShenandoahLoadReferenceBarrierNode * n) { + if (_load_reference_barriers->contains(n)) { + _load_reference_barriers->remove(n); + } +} + +Node* ShenandoahBarrierSetC2::shenandoah_storeval_barrier(GraphKit* kit, Node* obj) const { + if (ShenandoahStoreValEnqueueBarrier) { + obj = shenandoah_enqueue_barrier(kit, obj); } return obj; } -Node* ShenandoahBarrierSetC2::shenandoah_storeval_barrier(GraphKit* kit, Node* obj) const { - if (ShenandoahStoreValEnqueueBarrier) { - obj = shenandoah_write_barrier(kit, obj); - obj = shenandoah_enqueue_barrier(kit, obj); - } - if (ShenandoahStoreValReadBarrier) { - obj = shenandoah_read_barrier_impl(kit, obj, true, false, false); - } - return obj; -} - -Node* ShenandoahBarrierSetC2::shenandoah_read_barrier_impl(GraphKit* kit, Node* obj, bool use_ctrl, bool use_mem, bool allow_fromspace) const { - const Type* obj_type = obj->bottom_type(); - if (obj_type->higher_equal(TypePtr::NULL_PTR)) { - return obj; - } - const TypePtr* adr_type = ShenandoahBarrierNode::brooks_pointer_type(obj_type); - Node* mem = use_mem ? __ memory(adr_type) : __ immutable_memory(); - - if (! ShenandoahBarrierNode::needs_barrier(&__ gvn(), NULL, obj, mem, allow_fromspace)) { - // We know it is null, no barrier needed. - return obj; - } - - if (obj_type->meet(TypePtr::NULL_PTR) == obj_type->remove_speculative()) { - - // We don't know if it's null or not. Need null-check. - enum { _not_null_path = 1, _null_path, PATH_LIMIT }; - RegionNode* region = new RegionNode(PATH_LIMIT); - Node* phi = new PhiNode(region, obj_type); - Node* null_ctrl = __ top(); - Node* not_null_obj = __ null_check_oop(obj, &null_ctrl); - - region->init_req(_null_path, null_ctrl); - phi ->init_req(_null_path, __ zerocon(T_OBJECT)); - - Node* ctrl = use_ctrl ? __ control() : NULL; - ShenandoahReadBarrierNode* rb = new ShenandoahReadBarrierNode(ctrl, mem, not_null_obj, allow_fromspace); - Node* n = __ gvn().transform(rb); - - region->init_req(_not_null_path, __ control()); - phi ->init_req(_not_null_path, n); - - __ set_control(__ gvn().transform(region)); - __ record_for_igvn(region); - return __ gvn().transform(phi); - - } else { - // We know it is not null. Simple barrier is sufficient. - Node* ctrl = use_ctrl ? __ control() : NULL; - ShenandoahReadBarrierNode* rb = new ShenandoahReadBarrierNode(ctrl, mem, obj, allow_fromspace); - Node* n = __ gvn().transform(rb); - __ record_for_igvn(n); - return n; - } -} - -Node* ShenandoahBarrierSetC2::shenandoah_write_barrier_helper(GraphKit* kit, Node* obj, const TypePtr* adr_type) const { - ShenandoahWriteBarrierNode* wb = new ShenandoahWriteBarrierNode(kit->C, kit->control(), kit->memory(adr_type), obj); - Node* n = __ gvn().transform(wb); - if (n == wb) { // New barrier needs memory projection. - Node* proj = __ gvn().transform(new ShenandoahWBMemProjNode(n)); - __ set_memory(proj, adr_type); - } - return n; -} - -Node* ShenandoahBarrierSetC2::shenandoah_write_barrier(GraphKit* kit, Node* obj) const { - if (ShenandoahWriteBarrier) { - obj = shenandoah_write_barrier_impl(kit, obj); - } - return obj; -} - -Node* ShenandoahBarrierSetC2::shenandoah_write_barrier_impl(GraphKit* kit, Node* obj) const { - if (! ShenandoahBarrierNode::needs_barrier(&__ gvn(), NULL, obj, NULL, true)) { - return obj; - } - const Type* obj_type = obj->bottom_type(); - const TypePtr* adr_type = ShenandoahBarrierNode::brooks_pointer_type(obj_type); - Node* n = shenandoah_write_barrier_helper(kit, obj, adr_type); - __ record_for_igvn(n); - return n; -} +#define __ kit-> bool ShenandoahBarrierSetC2::satb_can_remove_pre_barrier(GraphKit* kit, PhaseTransform* phase, Node* adr, BasicType bt, uint adr_idx) const { @@ -304,7 +239,7 @@ Node* gc_state = __ AddP(no_base, tls, __ ConX(in_bytes(ShenandoahThreadLocalData::gc_state_offset()))); Node* ld = __ load(__ ctrl(), gc_state, TypeInt::BYTE, T_BYTE, Compile::AliasIdxRaw); marking = __ AndI(ld, __ ConI(ShenandoahHeap::MARKING)); - assert(ShenandoahWriteBarrierNode::is_gc_state_load(ld), "Should match the shape"); + assert(ShenandoahBarrierC2Support::is_gc_state_load(ld), "Should match the shape"); // if (!marking) __ if_then(marking, BoolTest::ne, zero, unlikely); { @@ -361,7 +296,7 @@ bool ShenandoahBarrierSetC2::is_shenandoah_wb_call(Node* call) { return call->is_CallLeaf() && - call->as_CallLeaf()->entry_point() == CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_JRT); + call->as_CallLeaf()->entry_point() == CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_JRT); } bool ShenandoahBarrierSetC2::is_shenandoah_marking_if(PhaseTransform *phase, Node* n) { @@ -549,88 +484,6 @@ return TypeFunc::make(domain, range); } -void ShenandoahBarrierSetC2::resolve_address(C2Access& access) const { - const TypePtr* adr_type = access.addr().type(); - - if ((access.decorators() & IN_NATIVE) == 0 && (adr_type->isa_instptr() || adr_type->isa_aryptr())) { - int off = adr_type->is_ptr()->offset(); - int base_off = adr_type->isa_instptr() ? instanceOopDesc::base_offset_in_bytes() : - arrayOopDesc::base_offset_in_bytes(adr_type->is_aryptr()->elem()->array_element_basic_type()); - assert(off != Type::OffsetTop, "unexpected offset"); - if (off == Type::OffsetBot || off >= base_off) { - DecoratorSet decorators = access.decorators(); - bool is_write = (decorators & C2_WRITE_ACCESS) != 0; - GraphKit* kit = NULL; - if (access.is_parse_access()) { - C2ParseAccess& parse_access = static_cast(access); - kit = parse_access.kit(); - } - Node* adr = access.addr().node(); - assert(adr->is_AddP(), "unexpected address shape"); - Node* base = adr->in(AddPNode::Base); - - if (is_write) { - if (kit != NULL) { - base = shenandoah_write_barrier(kit, base); - } else { - assert(access.is_opt_access(), "either parse or opt access"); - assert((access.decorators() & C2_ARRAY_COPY) != 0, "can be skipped for clone"); - } - } else { - if (adr_type->isa_instptr()) { - Compile* C = access.gvn().C; - ciField* field = C->alias_type(adr_type)->field(); - - // Insert read barrier for Shenandoah. - if (field != NULL && - ((ShenandoahOptimizeStaticFinals && field->is_static() && field->is_final()) || - (ShenandoahOptimizeInstanceFinals && !field->is_static() && field->is_final()) || - (ShenandoahOptimizeStableFinals && field->is_stable()))) { - // Skip the barrier for special fields - } else { - if (kit != NULL) { - base = shenandoah_read_barrier(kit, base); - } else { - assert(access.is_opt_access(), "either parse or opt access"); - assert((access.decorators() & C2_ARRAY_COPY) != 0, "can be skipped for arraycopy"); - } - } - } else { - if (kit != NULL) { - base = shenandoah_read_barrier(kit, base); - } else { - assert(access.is_opt_access(), "either parse or opt access"); - assert((access.decorators() & C2_ARRAY_COPY) != 0, "can be skipped for arraycopy"); - } - } - } - if (base != adr->in(AddPNode::Base)) { - assert(kit != NULL, "no barrier should have been added"); - - Node* address = adr->in(AddPNode::Address); - - if (address->is_AddP()) { - assert(address->in(AddPNode::Base) == adr->in(AddPNode::Base), "unexpected address shape"); - assert(!address->in(AddPNode::Address)->is_AddP(), "unexpected address shape"); - assert(address->in(AddPNode::Address) == adr->in(AddPNode::Base), "unexpected address shape"); - address = address->clone(); - address->set_req(AddPNode::Base, base); - address->set_req(AddPNode::Address, base); - address = kit->gvn().transform(address); - } else { - assert(address == adr->in(AddPNode::Base), "unexpected address shape"); - address = base; - } - adr = adr->clone(); - adr->set_req(AddPNode::Base, base); - adr->set_req(AddPNode::Address, address); - adr = kit->gvn().transform(adr); - access.addr().set_node(adr); - } - } - } -} - Node* ShenandoahBarrierSetC2::store_at_resolved(C2Access& access, C2AccessValue& val) const { DecoratorSet decorators = access.decorators(); @@ -662,44 +515,8 @@ PhaseGVN& gvn = opt_access.gvn(); MergeMemNode* mm = opt_access.mem(); - if (ShenandoahStoreValReadBarrier) { - RegionNode* region = new RegionNode(3); - const Type* v_t = gvn.type(val.node()); - Node* phi = new PhiNode(region, v_t->isa_oopptr() ? v_t->is_oopptr()->cast_to_nonconst() : v_t); - Node* cmp = gvn.transform(new CmpPNode(val.node(), gvn.zerocon(T_OBJECT))); - Node* bol = gvn.transform(new BoolNode(cmp, BoolTest::ne)); - IfNode* iff = new IfNode(opt_access.ctl(), bol, PROB_LIKELY_MAG(3), COUNT_UNKNOWN); - - gvn.transform(iff); - if (gvn.is_IterGVN()) { - gvn.is_IterGVN()->_worklist.push(iff); - } else { - gvn.record_for_igvn(iff); - } - - Node* null_true = gvn.transform(new IfFalseNode(iff)); - Node* null_false = gvn.transform(new IfTrueNode(iff)); - region->init_req(1, null_true); - region->init_req(2, null_false); - phi->init_req(1, gvn.zerocon(T_OBJECT)); - Node* cast = new CastPPNode(val.node(), gvn.type(val.node())->join_speculative(TypePtr::NOTNULL)); - cast->set_req(0, null_false); - cast = gvn.transform(cast); - Node* rb = gvn.transform(new ShenandoahReadBarrierNode(null_false, gvn.C->immutable_memory(), cast, false)); - phi->init_req(2, rb); - opt_access.set_ctl(gvn.transform(region)); - val.set_node(gvn.transform(phi)); - } if (ShenandoahStoreValEnqueueBarrier) { - const TypePtr* adr_type = ShenandoahBarrierNode::brooks_pointer_type(gvn.type(val.node())); - int alias = gvn.C->get_alias_index(adr_type); - Node* wb = new ShenandoahWriteBarrierNode(gvn.C, opt_access.ctl(), mm->memory_at(alias), val.node()); - Node* wb_transformed = gvn.transform(wb); - Node* enqueue = gvn.transform(new ShenandoahEnqueueBarrierNode(wb_transformed)); - if (wb_transformed == wb) { - Node* proj = gvn.transform(new ShenandoahWBMemProjNode(wb)); - mm->set_memory_at(alias, proj); - } + Node* enqueue = gvn.transform(new ShenandoahEnqueueBarrierNode(val.node())); val.set_node(enqueue); } } @@ -724,6 +541,17 @@ Node* offset = adr->is_AddP() ? adr->in(AddPNode::Offset) : top; Node* load = BarrierSetC2::load_at_resolved(access, val_type); + if (access.is_oop()) { + if (ShenandoahLoadRefBarrier) { + load = new ShenandoahLoadReferenceBarrierNode(NULL, load); + if (access.is_parse_access()) { + load = static_cast(access).kit()->gvn().transform(load); + } else { + load = static_cast(access).gvn().transform(load); + } + } + } + // If we are reading the value of the referent field of a Reference // object (either by using Unsafe directly or through reflection) // then, if SATB is enabled, we need to record the referent in an @@ -797,9 +625,10 @@ #ifdef _LP64 if (adr->bottom_type()->is_ptr_to_narrowoop()) { - return kit->gvn().transform(new DecodeNNode(load_store, load_store->get_ptr_type())); + load_store = kit->gvn().transform(new DecodeNNode(load_store, load_store->get_ptr_type())); } #endif + load_store = kit->gvn().transform(new ShenandoahLoadReferenceBarrierNode(NULL, load_store)); return load_store; } return BarrierSetC2::atomic_cmpxchg_val_at_resolved(access, expected_val, new_val, value_type); @@ -867,6 +696,7 @@ } Node* result = BarrierSetC2::atomic_xchg_at_resolved(access, val, value_type); if (access.is_oop()) { + result = kit->gvn().transform(new ShenandoahLoadReferenceBarrierNode(NULL, result)); shenandoah_write_barrier_pre(kit, false /* do_load */, NULL, NULL, max_juint, NULL, NULL, result /* pre_val */, T_OBJECT); @@ -876,19 +706,9 @@ void ShenandoahBarrierSetC2::clone(GraphKit* kit, Node* src, Node* dst, Node* size, bool is_array) const { assert(!src->is_AddP(), "unexpected input"); - src = shenandoah_read_barrier(kit, src); BarrierSetC2::clone(kit, src, dst, size, is_array); } -Node* ShenandoahBarrierSetC2::resolve(GraphKit* kit, Node* n, DecoratorSet decorators) const { - bool is_write = decorators & ACCESS_WRITE; - if (is_write) { - return shenandoah_write_barrier(kit, n); - } else { - return shenandoah_read_barrier(kit, n); - } -} - Node* ShenandoahBarrierSetC2::obj_allocate(PhaseMacroExpand* macro, Node* ctrl, Node* mem, Node* toobig_false, Node* size_in_bytes, Node*& i_o, Node*& needgc_ctrl, Node*& fast_oop_ctrl, Node*& fast_oop_rawmem, @@ -915,6 +735,7 @@ // Support for GC barriers emitted during parsing bool ShenandoahBarrierSetC2::is_gc_barrier_node(Node* node) const { + if (node->Opcode() == Op_ShenandoahLoadReferenceBarrier) return true; if (node->Opcode() != Op_CallLeaf && node->Opcode() != Op_CallLeafNoFP) { return false; } @@ -929,26 +750,30 @@ } Node* ShenandoahBarrierSetC2::step_over_gc_barrier(Node* c) const { - return ShenandoahBarrierNode::skip_through_barrier(c); + if (c->Opcode() == Op_ShenandoahLoadReferenceBarrier) { + return c->in(ShenandoahLoadReferenceBarrierNode::ValueIn); + } + if (c->Opcode() == Op_ShenandoahEnqueueBarrier) { + c = c->in(1); + } + return c; } bool ShenandoahBarrierSetC2::expand_barriers(Compile* C, PhaseIterGVN& igvn) const { - return !ShenandoahWriteBarrierNode::expand(C, igvn); + return !ShenandoahBarrierC2Support::expand(C, igvn); } bool ShenandoahBarrierSetC2::optimize_loops(PhaseIdealLoop* phase, LoopOptsMode mode, VectorSet& visited, Node_Stack& nstack, Node_List& worklist) const { if (mode == LoopOptsShenandoahExpand) { assert(UseShenandoahGC, "only for shenandoah"); - ShenandoahWriteBarrierNode::pin_and_expand(phase); + ShenandoahBarrierC2Support::pin_and_expand(phase); return true; } else if (mode == LoopOptsShenandoahPostExpand) { assert(UseShenandoahGC, "only for shenandoah"); visited.Clear(); - ShenandoahWriteBarrierNode::optimize_after_expansion(visited, nstack, worklist, phase); + ShenandoahBarrierC2Support::optimize_after_expansion(visited, nstack, worklist, phase); return true; } - GrowableArray memory_graph_fixers; - ShenandoahWriteBarrierNode::optimize_before_expansion(phase, memory_graph_fixers, false); return false; } @@ -957,7 +782,6 @@ if (!is_oop) { return false; } - if (tightly_coupled_alloc) { if (phase == Optimization) { return false; @@ -985,7 +809,7 @@ } } else { return true; - } + } } else if (src_type->isa_aryptr()) { BasicType src_elem = src_type->klass()->as_array_klass()->element_type()->basic_type(); if (src_elem == T_OBJECT || src_elem == T_ARRAY) { @@ -1038,14 +862,20 @@ // Support for macro expanded GC barriers void ShenandoahBarrierSetC2::register_potential_barrier_node(Node* node) const { - if (node->Opcode() == Op_ShenandoahWriteBarrier) { - state()->add_shenandoah_barrier((ShenandoahWriteBarrierNode*) node); + if (node->Opcode() == Op_ShenandoahEnqueueBarrier) { + state()->add_enqueue_barrier((ShenandoahEnqueueBarrierNode*) node); + } + if (node->Opcode() == Op_ShenandoahLoadReferenceBarrier) { + state()->add_load_reference_barrier((ShenandoahLoadReferenceBarrierNode*) node); } } void ShenandoahBarrierSetC2::unregister_potential_barrier_node(Node* node) const { - if (node->Opcode() == Op_ShenandoahWriteBarrier) { - state()->remove_shenandoah_barrier((ShenandoahWriteBarrierNode*) node); + if (node->Opcode() == Op_ShenandoahEnqueueBarrier) { + state()->remove_enqueue_barrier((ShenandoahEnqueueBarrierNode*) node); + } + if (node->Opcode() == Op_ShenandoahLoadReferenceBarrier) { + state()->remove_load_reference_barrier((ShenandoahLoadReferenceBarrierNode*) node); } } @@ -1091,19 +921,18 @@ } } } - for (int i = state()->shenandoah_barriers_count()-1; i >= 0; i--) { - ShenandoahWriteBarrierNode* n = state()->shenandoah_barrier(i); + for (int i = state()->enqueue_barriers_count() - 1; i >= 0; i--) { + ShenandoahEnqueueBarrierNode* n = state()->enqueue_barrier(i); if (!useful.member(n)) { - state()->remove_shenandoah_barrier(n); + state()->remove_enqueue_barrier(n); } } - -} - -bool ShenandoahBarrierSetC2::has_special_unique_user(const Node* node) const { - assert(node->outcnt() == 1, "match only for unique out"); - Node* n = node->unique_out(); - return node->Opcode() == Op_ShenandoahWriteBarrier && n->Opcode() == Op_ShenandoahWBMemProj; + for (int i = state()->load_reference_barriers_count() - 1; i >= 0; i--) { + ShenandoahLoadReferenceBarrierNode* n = state()->load_reference_barrier(i); + if (!useful.member(n)) { + state()->remove_load_reference_barrier(n); + } + } } void ShenandoahBarrierSetC2::add_users_to_worklist(Unique_Node_List* worklist) const {} @@ -1123,7 +952,7 @@ #ifdef ASSERT void ShenandoahBarrierSetC2::verify_gc_barriers(Compile* compile, CompilePhase phase) const { if (ShenandoahVerifyOptoBarriers && phase == BarrierSetC2::BeforeExpand) { - ShenandoahBarrierNode::verify(Compile::current()->root()); + ShenandoahBarrierC2Support::verify(Compile::current()->root()); } else if (phase == BarrierSetC2::BeforeCodeGen) { // Verify G1 pre-barriers const int marking_offset = in_bytes(ShenandoahThreadLocalData::satb_mark_queue_active_offset()); @@ -1229,7 +1058,7 @@ } } else if (can_reshape && n->Opcode() == Op_If && - ShenandoahWriteBarrierNode::is_heap_stable_test(n) && + ShenandoahBarrierC2Support::is_heap_stable_test(n) && n->in(0) != NULL) { Node* dom = n->in(0); Node* prev_dom = n; @@ -1237,7 +1066,7 @@ int dist = 16; // Search up the dominator tree for another heap stable test while (dom->Opcode() != op || // Not same opcode? - !ShenandoahWriteBarrierNode::is_heap_stable_test(dom) || // Not same input 1? + !ShenandoahBarrierC2Support::is_heap_stable_test(dom) || // Not same input 1? prev_dom->in(0) != dom) { // One path of test does not dominate? if (dist < 0) return NULL; @@ -1258,46 +1087,6 @@ return NULL; } -Node* ShenandoahBarrierSetC2::identity_node(PhaseGVN* phase, Node* n) const { - if (n->is_Load()) { - Node *mem = n->in(MemNode::Memory); - Node *value = n->as_Load()->can_see_stored_value(mem, phase); - if (value) { - PhaseIterGVN *igvn = phase->is_IterGVN(); - if (igvn != NULL && - value->is_Phi() && - value->req() > 2 && - value->in(1) != NULL && - value->in(1)->is_ShenandoahBarrier()) { - if (igvn->_worklist.member(value) || - igvn->_worklist.member(value->in(0)) || - (value->in(0)->in(1) != NULL && - value->in(0)->in(1)->is_IfProj() && - (igvn->_worklist.member(value->in(0)->in(1)) || - (value->in(0)->in(1)->in(0) != NULL && - igvn->_worklist.member(value->in(0)->in(1)->in(0)))))) { - igvn->_worklist.push(n); - return n; - } - } - // (This works even when value is a Con, but LoadNode::Value - // usually runs first, producing the singleton type of the Con.) - Node *value_no_barrier = step_over_gc_barrier(value->Opcode() == Op_EncodeP ? value->in(1) : value); - if (value->Opcode() == Op_EncodeP) { - if (value_no_barrier != value->in(1)) { - Node *encode = value->clone(); - encode->set_req(1, value_no_barrier); - encode = phase->transform(encode); - return encode; - } - } else { - return value_no_barrier; - } - } - } - return n; -} - bool ShenandoahBarrierSetC2::has_only_shenandoah_wb_pre_uses(Node* n) { for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) { Node* u = n->fast_out(i); @@ -1308,20 +1097,6 @@ return n->outcnt() > 0; } -bool ShenandoahBarrierSetC2::flatten_gc_alias_type(const TypePtr*& adr_type) const { - int offset = adr_type->offset(); - if (offset == ShenandoahBrooksPointer::byte_offset()) { - if (adr_type->isa_aryptr()) { - adr_type = TypeAryPtr::make(adr_type->ptr(), adr_type->isa_aryptr()->ary(), adr_type->isa_aryptr()->klass(), false, offset); - } else if (adr_type->isa_instptr()) { - adr_type = TypeInstPtr::make(adr_type->ptr(), ciEnv::current()->Object_klass(), false, NULL, offset); - } - return true; - } else { - return false; - } -} - bool ShenandoahBarrierSetC2::final_graph_reshaping(Compile* compile, Node* n, uint opcode) const { switch (opcode) { case Op_CallLeaf: @@ -1356,9 +1131,7 @@ } #endif return true; - case Op_ShenandoahReadBarrier: - return true; - case Op_ShenandoahWriteBarrier: + case Op_ShenandoahLoadReferenceBarrier: assert(false, "should have been expanded already"); return true; default: @@ -1366,17 +1139,6 @@ } } -#ifdef ASSERT -bool ShenandoahBarrierSetC2::verify_gc_alias_type(const TypePtr* adr_type, int offset) const { - if (offset == ShenandoahBrooksPointer::byte_offset() && - (adr_type->base() == Type::AryPtr || adr_type->base() == Type::OopPtr)) { - return true; - } else { - return false; - } -} -#endif - bool ShenandoahBarrierSetC2::escape_add_to_con_graph(ConnectionGraph* conn_graph, PhaseGVN* gvn, Unique_Node_List* delayed_worklist, Node* n, uint opcode) const { switch (opcode) { case Op_ShenandoahCompareAndExchangeP: @@ -1412,15 +1174,12 @@ } return false; } - case Op_ShenandoahReadBarrier: - case Op_ShenandoahWriteBarrier: - // Barriers 'pass through' its arguments. I.e. what goes in, comes out. - // It doesn't escape. - conn_graph->add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(ShenandoahBarrierNode::ValueIn), delayed_worklist); - break; case Op_ShenandoahEnqueueBarrier: conn_graph->add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(1), delayed_worklist); break; + case Op_ShenandoahLoadReferenceBarrier: + conn_graph->add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(ShenandoahLoadReferenceBarrierNode::ValueIn), delayed_worklist); + return true; default: // Nothing break; @@ -1441,15 +1200,12 @@ case Op_ShenandoahWeakCompareAndSwapP: case Op_ShenandoahWeakCompareAndSwapN: return conn_graph->add_final_edges_unsafe_access(n, opcode); - case Op_ShenandoahReadBarrier: - case Op_ShenandoahWriteBarrier: - // Barriers 'pass through' its arguments. I.e. what goes in, comes out. - // It doesn't escape. - conn_graph->add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(ShenandoahBarrierNode::ValueIn), NULL); - return true; case Op_ShenandoahEnqueueBarrier: conn_graph->add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(1), NULL); return true; + case Op_ShenandoahLoadReferenceBarrier: + conn_graph->add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(ShenandoahLoadReferenceBarrierNode::ValueIn), NULL); + return true; default: // Nothing break; @@ -1464,21 +1220,7 @@ } bool ShenandoahBarrierSetC2::escape_is_barrier_node(Node* n) const { - return n->is_ShenandoahBarrier(); -} - -bool ShenandoahBarrierSetC2::matcher_find_shared_visit(Matcher* matcher, Matcher::MStack& mstack, Node* n, uint opcode, bool& mem_op, int& mem_addr_idx) const { - switch (opcode) { - case Op_ShenandoahReadBarrier: - if (n->in(ShenandoahBarrierNode::ValueIn)->is_DecodeNarrowPtr()) { - matcher->set_shared(n->in(ShenandoahBarrierNode::ValueIn)->in(1)); - } - matcher->set_shared(n); - return true; - default: - break; - } - return false; + return n->Opcode() == Op_ShenandoahLoadReferenceBarrier; } bool ShenandoahBarrierSetC2::matcher_find_shared_post_visit(Matcher* matcher, Node* n, uint opcode) const { @@ -1510,62 +1252,3 @@ xop == Op_ShenandoahCompareAndSwapN || xop == Op_ShenandoahCompareAndSwapP; } - -void ShenandoahBarrierSetC2::igvn_add_users_to_worklist(PhaseIterGVN* igvn, Node* use) const { - if (use->is_ShenandoahBarrier()) { - for (DUIterator_Fast i2max, i2 = use->fast_outs(i2max); i2 < i2max; i2++) { - Node* u = use->fast_out(i2); - Node* cmp = use->find_out_with(Op_CmpP); - if (u->Opcode() == Op_CmpP) { - igvn->_worklist.push(cmp); - } - } - } -} - -void ShenandoahBarrierSetC2::ccp_analyze(PhaseCCP* ccp, Unique_Node_List& worklist, Node* use) const { - if (use->is_ShenandoahBarrier()) { - for (DUIterator_Fast i2max, i2 = use->fast_outs(i2max); i2 < i2max; i2++) { - Node* p = use->fast_out(i2); - if (p->Opcode() == Op_AddP) { - for (DUIterator_Fast i3max, i3 = p->fast_outs(i3max); i3 < i3max; i3++) { - Node* q = p->fast_out(i3); - if (q->is_Load()) { - if(q->bottom_type() != ccp->type(q)) { - worklist.push(q); - } - } - } - } - } - } -} - -Node* ShenandoahBarrierSetC2::split_if_pre(PhaseIdealLoop* phase, Node* n) const { - if (n->Opcode() == Op_ShenandoahReadBarrier) { - ((ShenandoahReadBarrierNode*)n)->try_move(phase); - } else if (n->Opcode() == Op_ShenandoahWriteBarrier) { - return ((ShenandoahWriteBarrierNode*)n)->try_split_thru_phi(phase); - } - - return NULL; -} - -bool ShenandoahBarrierSetC2::build_loop_late_post(PhaseIdealLoop* phase, Node* n) const { - return ShenandoahBarrierNode::build_loop_late_post(phase, n); -} - -bool ShenandoahBarrierSetC2::sink_node(PhaseIdealLoop* phase, Node* n, Node* x, Node* x_ctrl, Node* n_ctrl) const { - if (n->is_ShenandoahBarrier()) { - return x->as_ShenandoahBarrier()->sink_node(phase, x_ctrl, n_ctrl); - } - if (n->is_MergeMem()) { - // PhaseIdealLoop::split_if_with_blocks_post() would: - // _igvn._worklist.yank(x); - // which sometimes causes chains of MergeMem which some of - // shenandoah specific code doesn't support - phase->register_new_node(x, x_ctrl); - return true; - } - return false; -} diff --git a/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.hpp b/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.hpp --- a/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.hpp +++ b/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.hpp @@ -30,14 +30,21 @@ class ShenandoahBarrierSetC2State : public ResourceObj { private: - GrowableArray* _shenandoah_barriers; + GrowableArray* _enqueue_barriers; + GrowableArray* _load_reference_barriers; public: ShenandoahBarrierSetC2State(Arena* comp_arena); - int shenandoah_barriers_count() const; - ShenandoahWriteBarrierNode* shenandoah_barrier(int idx) const; - void add_shenandoah_barrier(ShenandoahWriteBarrierNode * n); - void remove_shenandoah_barrier(ShenandoahWriteBarrierNode * n); + + int enqueue_barriers_count() const; + ShenandoahEnqueueBarrierNode* enqueue_barrier(int idx) const; + void add_enqueue_barrier(ShenandoahEnqueueBarrierNode* n); + void remove_enqueue_barrier(ShenandoahEnqueueBarrierNode * n); + + int load_reference_barriers_count() const; + ShenandoahLoadReferenceBarrierNode* load_reference_barrier(int idx) const; + void add_load_reference_barrier(ShenandoahLoadReferenceBarrierNode* n); + void remove_load_reference_barrier(ShenandoahLoadReferenceBarrierNode * n); }; class ShenandoahBarrierSetC2 : public BarrierSetC2 { @@ -66,12 +73,7 @@ BasicType bt) const; Node* shenandoah_enqueue_barrier(GraphKit* kit, Node* val) const; - Node* shenandoah_read_barrier(GraphKit* kit, Node* obj) const; Node* shenandoah_storeval_barrier(GraphKit* kit, Node* obj) const; - Node* shenandoah_write_barrier(GraphKit* kit, Node* obj) const; - Node* shenandoah_read_barrier_impl(GraphKit* kit, Node* obj, bool use_ctrl, bool use_mem, bool allow_fromspace) const; - Node* shenandoah_write_barrier_impl(GraphKit* kit, Node* obj) const; - Node* shenandoah_write_barrier_helper(GraphKit* kit, Node* obj, const TypePtr* adr_type) const; void insert_pre_barrier(GraphKit* kit, Node* base_oop, Node* offset, Node* pre_val, bool need_mem_bar) const; @@ -79,7 +81,6 @@ static bool clone_needs_postbarrier(ArrayCopyNode *ac, PhaseIterGVN& igvn); protected: - virtual void resolve_address(C2Access& access) const; virtual Node* load_at_resolved(C2Access& access, const Type* val_type) const; virtual Node* store_at_resolved(C2Access& access, C2AccessValue& val) const; virtual Node* atomic_cmpxchg_val_at_resolved(C2AtomicParseAccess& access, Node* expected_val, @@ -102,12 +103,11 @@ static const TypeFunc* write_ref_field_pre_entry_Type(); static const TypeFunc* shenandoah_clone_barrier_Type(); static const TypeFunc* shenandoah_write_barrier_Type(); + virtual bool has_load_barriers() const { return true; } // This is the entry-point for the backend to perform accesses through the Access API. virtual void clone(GraphKit* kit, Node* src, Node* dst, Node* size, bool is_array) const; - virtual Node* resolve(GraphKit* kit, Node* n, DecoratorSet decorators) const; - virtual Node* obj_allocate(PhaseMacroExpand* macro, Node* ctrl, Node* mem, Node* toobig_false, Node* size_in_bytes, Node*& i_o, Node*& needgc_ctrl, Node*& fast_oop_ctrl, Node*& fast_oop_rawmem, @@ -144,13 +144,7 @@ virtual void verify_gc_barriers(Compile* compile, CompilePhase phase) const; #endif - virtual bool flatten_gc_alias_type(const TypePtr*& adr_type) const; -#ifdef ASSERT - virtual bool verify_gc_alias_type(const TypePtr* adr_type, int offset) const; -#endif - virtual Node* ideal_node(PhaseGVN* phase, Node* n, bool can_reshape) const; - virtual Node* identity_node(PhaseGVN* phase, Node* n) const; virtual bool final_graph_reshaping(Compile* compile, Node* n, uint opcode) const; virtual bool escape_add_to_con_graph(ConnectionGraph* conn_graph, PhaseGVN* gvn, Unique_Node_List* delayed_worklist, Node* n, uint opcode) const; @@ -158,17 +152,8 @@ virtual bool escape_has_out_with_unsafe_object(Node* n) const; virtual bool escape_is_barrier_node(Node* n) const; - virtual bool matcher_find_shared_visit(Matcher* matcher, Matcher::MStack& mstack, Node* n, uint opcode, bool& mem_op, int& mem_addr_idx) const; virtual bool matcher_find_shared_post_visit(Matcher* matcher, Node* n, uint opcode) const; virtual bool matcher_is_store_load_barrier(Node* x, uint xop) const; - - virtual void igvn_add_users_to_worklist(PhaseIterGVN* igvn, Node* use) const; - virtual void ccp_analyze(PhaseCCP* ccp, Unique_Node_List& worklist, Node* use) const; - - virtual bool has_special_unique_user(const Node* node) const; - virtual Node* split_if_pre(PhaseIdealLoop* phase, Node* n) const; - virtual bool build_loop_late_post(PhaseIdealLoop* phase, Node* n) const; - virtual bool sink_node(PhaseIdealLoop* phase, Node* n, Node* x, Node* x_ctrl, Node* n_ctrl) const; }; #endif // SHARE_GC_SHENANDOAH_C2_SHENANDOAHBARRIERSETC2_HPP diff --git a/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp b/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp --- a/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp +++ b/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp @@ -41,383 +41,28 @@ #include "opto/runtime.hpp" #include "opto/subnode.hpp" -Node* ShenandoahBarrierNode::skip_through_barrier(Node* n) { - if (n == NULL) { - return NULL; - } - if (n->Opcode() == Op_ShenandoahEnqueueBarrier) { - n = n->in(1); - } - - if (n->is_ShenandoahBarrier()) { - return n->in(ValueIn); - } else if (n->is_Phi() && - n->req() == 3 && - n->in(1) != NULL && - n->in(1)->is_ShenandoahBarrier() && - n->in(2) != NULL && - n->in(2)->bottom_type() == TypePtr::NULL_PTR && - n->in(0) != NULL && - n->in(0)->in(1) != NULL && - n->in(0)->in(1)->is_IfProj() && - n->in(0)->in(2) != NULL && - n->in(0)->in(2)->is_IfProj() && - n->in(0)->in(1)->in(0) != NULL && - n->in(0)->in(1)->in(0) == n->in(0)->in(2)->in(0) && - n->in(1)->in(ValueIn)->Opcode() == Op_CastPP) { - Node* iff = n->in(0)->in(1)->in(0); - Node* res = n->in(1)->in(ValueIn)->in(1); - if (iff->is_If() && - iff->in(1) != NULL && - iff->in(1)->is_Bool() && - iff->in(1)->as_Bool()->_test._test == BoolTest::ne && - iff->in(1)->in(1) != NULL && - iff->in(1)->in(1)->Opcode() == Op_CmpP && - iff->in(1)->in(1)->in(1) != NULL && - iff->in(1)->in(1)->in(1) == res && - iff->in(1)->in(1)->in(2) != NULL && - iff->in(1)->in(1)->in(2)->bottom_type() == TypePtr::NULL_PTR) { - return res; - } - } - return n; -} - -bool ShenandoahBarrierNode::needs_barrier(PhaseGVN* phase, ShenandoahBarrierNode* orig, Node* n, Node* rb_mem, bool allow_fromspace) { - Unique_Node_List visited; - return needs_barrier_impl(phase, orig, n, rb_mem, allow_fromspace, visited); -} - -bool ShenandoahBarrierNode::needs_barrier_impl(PhaseGVN* phase, ShenandoahBarrierNode* orig, Node* n, Node* rb_mem, bool allow_fromspace, Unique_Node_List &visited) { - if (visited.member(n)) { - return false; // Been there. - } - visited.push(n); - - if (n->is_Allocate()) { - return false; - } - - if (n->is_Call()) { - return true; - } - - const Type* type = phase->type(n); - if (type == Type::TOP) { - return false; - } - if (type->make_ptr()->higher_equal(TypePtr::NULL_PTR)) { - return false; - } - if (type->make_oopptr() && type->make_oopptr()->const_oop() != NULL) { - return false; - } - - if (ShenandoahOptimizeStableFinals) { - const TypeAryPtr* ary = type->isa_aryptr(); - if (ary && ary->is_stable() && allow_fromspace) { - return false; - } - } - - if (n->is_CheckCastPP() || n->is_ConstraintCast() || n->Opcode() == Op_ShenandoahEnqueueBarrier) { - return needs_barrier_impl(phase, orig, n->in(1), rb_mem, allow_fromspace, visited); - } - if (n->is_Parm()) { - return true; - } - if (n->is_Proj()) { - return needs_barrier_impl(phase, orig, n->in(0), rb_mem, allow_fromspace, visited); - } - - if (n->Opcode() == Op_ShenandoahWBMemProj) { - return needs_barrier_impl(phase, orig, n->in(ShenandoahWBMemProjNode::WriteBarrier), rb_mem, allow_fromspace, visited); - } - if (n->is_Phi()) { - bool need_barrier = false; - for (uint i = 1; i < n->req() && ! need_barrier; i++) { - Node* input = n->in(i); - if (input == NULL) { - need_barrier = true; // Phi not complete yet? - } else if (needs_barrier_impl(phase, orig, input, rb_mem, allow_fromspace, visited)) { - need_barrier = true; - } - } - return need_barrier; - } - if (n->is_CMove()) { - return needs_barrier_impl(phase, orig, n->in(CMoveNode::IfFalse), rb_mem, allow_fromspace, visited) || - needs_barrier_impl(phase, orig, n->in(CMoveNode::IfTrue ), rb_mem, allow_fromspace, visited); - } - if (n->Opcode() == Op_CreateEx) { - return true; - } - if (n->Opcode() == Op_ShenandoahWriteBarrier) { - return false; - } - if (n->Opcode() == Op_ShenandoahReadBarrier) { - if (rb_mem == n->in(Memory)) { - return false; - } else { - return true; - } - } - - if (n->Opcode() == Op_LoadP || - n->Opcode() == Op_LoadN || - n->Opcode() == Op_GetAndSetP || - n->Opcode() == Op_CompareAndExchangeP || - n->Opcode() == Op_ShenandoahCompareAndExchangeP || - n->Opcode() == Op_GetAndSetN || - n->Opcode() == Op_CompareAndExchangeN || - n->Opcode() == Op_ShenandoahCompareAndExchangeN) { - return true; - } - if (n->Opcode() == Op_DecodeN || - n->Opcode() == Op_EncodeP) { - return needs_barrier_impl(phase, orig, n->in(1), rb_mem, allow_fromspace, visited); - } - -#ifdef ASSERT - tty->print("need barrier on?: "); n->dump(); - ShouldNotReachHere(); -#endif - return true; -} - -bool ShenandoahReadBarrierNode::dominates_memory_rb_impl(PhaseGVN* phase, - Node* b1, - Node* b2, - Node* current, - bool linear) { - ResourceMark rm; - VectorSet visited(Thread::current()->resource_area()); - Node_Stack phis(0); - - for(int i = 0; i < 10; i++) { - if (current == NULL) { - return false; - } else if (visited.test_set(current->_idx) || current->is_top() || current == b1) { - current = NULL; - while (phis.is_nonempty() && current == NULL) { - uint idx = phis.index(); - Node* phi = phis.node(); - if (idx >= phi->req()) { - phis.pop(); - } else { - current = phi->in(idx); - phis.set_index(idx+1); - } - } - if (current == NULL) { - return true; - } - } else if (current == phase->C->immutable_memory()) { - return false; - } else if (current->isa_Phi()) { - if (!linear) { +bool ShenandoahBarrierC2Support::expand(Compile* C, PhaseIterGVN& igvn) { + ShenandoahBarrierSetC2State* state = ShenandoahBarrierSetC2::bsc2()->state(); + if ((state->enqueue_barriers_count() + + state->load_reference_barriers_count()) > 0) { + bool attempt_more_loopopts = ShenandoahLoopOptsAfterExpansion; + C->clear_major_progress(); + PhaseIdealLoop ideal_loop(igvn, LoopOptsShenandoahExpand); + if (C->failing()) return false; + PhaseIdealLoop::verify(igvn); + DEBUG_ONLY(verify_raw_mem(C->root());) + if (attempt_more_loopopts) { + C->set_major_progress(); + if (!C->optimize_loops(igvn, LoopOptsShenandoahPostExpand)) { return false; } - phis.push(current, 2); - current = current->in(1); - } else if (current->Opcode() == Op_ShenandoahWriteBarrier) { - const Type* in_type = current->bottom_type(); - const Type* this_type = b2->bottom_type(); - if (is_independent(in_type, this_type)) { - current = current->in(Memory); - } else { - return false; - } - } else if (current->Opcode() == Op_ShenandoahWBMemProj) { - current = current->in(ShenandoahWBMemProjNode::WriteBarrier); - } else if (current->is_Proj()) { - current = current->in(0); - } else if (current->is_Call()) { - return false; // TODO: Maybe improve by looking at the call's memory effects? - } else if (current->is_MemBar()) { - return false; // TODO: Do we need to stop at *any* membar? - } else if (current->is_MergeMem()) { - const TypePtr* adr_type = brooks_pointer_type(phase->type(b2)); - uint alias_idx = phase->C->get_alias_index(adr_type); - current = current->as_MergeMem()->memory_at(alias_idx); - } else { -#ifdef ASSERT - current->dump(); -#endif - ShouldNotReachHere(); - return false; - } - } - return false; -} - -bool ShenandoahReadBarrierNode::is_independent(Node* mem) { - if (mem->is_Phi() || mem->is_Proj() || mem->is_MergeMem()) { - return true; - } else if (mem->Opcode() == Op_ShenandoahWBMemProj) { - return true; - } else if (mem->Opcode() == Op_ShenandoahWriteBarrier) { - const Type* mem_type = mem->bottom_type(); - const Type* this_type = bottom_type(); - if (is_independent(mem_type, this_type)) { - return true; - } else { - return false; - } - } else if (mem->is_Call() || mem->is_MemBar()) { - return false; - } -#ifdef ASSERT - mem->dump(); -#endif - ShouldNotReachHere(); - return true; -} - -bool ShenandoahReadBarrierNode::dominates_memory_rb(PhaseGVN* phase, Node* b1, Node* b2, bool linear) { - return dominates_memory_rb_impl(phase, b1->in(Memory), b2, b2->in(Memory), linear); -} - -bool ShenandoahReadBarrierNode::is_independent(const Type* in_type, const Type* this_type) { - assert(in_type->isa_oopptr(), "expect oop ptr"); - assert(this_type->isa_oopptr(), "expect oop ptr"); - - ciKlass* in_kls = in_type->is_oopptr()->klass(); - ciKlass* this_kls = this_type->is_oopptr()->klass(); - if (in_kls != NULL && this_kls != NULL && - in_kls->is_loaded() && this_kls->is_loaded() && - (!in_kls->is_subclass_of(this_kls)) && - (!this_kls->is_subclass_of(in_kls))) { - return true; - } - return false; -} - -Node* ShenandoahReadBarrierNode::Ideal(PhaseGVN *phase, bool can_reshape) { - if (! can_reshape) { - return NULL; - } - - if (in(Memory) == phase->C->immutable_memory()) return NULL; - - // If memory input is a MergeMem, take the appropriate slice out of it. - Node* mem_in = in(Memory); - if (mem_in->isa_MergeMem()) { - const TypePtr* adr_type = brooks_pointer_type(bottom_type()); - uint alias_idx = phase->C->get_alias_index(adr_type); - mem_in = mem_in->as_MergeMem()->memory_at(alias_idx); - set_req(Memory, mem_in); - return this; - } - - Node* input = in(Memory); - if (input->Opcode() == Op_ShenandoahWBMemProj) { - ResourceMark rm; - VectorSet seen(Thread::current()->resource_area()); - Node* n = in(Memory); - while (n->Opcode() == Op_ShenandoahWBMemProj && - n->in(ShenandoahWBMemProjNode::WriteBarrier) != NULL && - n->in(ShenandoahWBMemProjNode::WriteBarrier)->Opcode() == Op_ShenandoahWriteBarrier && - n->in(ShenandoahWBMemProjNode::WriteBarrier)->in(Memory) != NULL) { - if (seen.test_set(n->_idx)) { - return NULL; // loop - } - n = n->in(ShenandoahWBMemProjNode::WriteBarrier)->in(Memory); - } - - Node* wb = input->in(ShenandoahWBMemProjNode::WriteBarrier); - const Type* in_type = phase->type(wb); - // is_top() test not sufficient here: we can come here after CCP - // in a dead branch of the graph that has not yet been removed. - if (in_type == Type::TOP) return NULL; // Dead path. - assert(wb->Opcode() == Op_ShenandoahWriteBarrier, "expect write barrier"); - if (is_independent(in_type, _type)) { - phase->igvn_rehash_node_delayed(wb); - set_req(Memory, wb->in(Memory)); - if (can_reshape && input->outcnt() == 0) { - phase->is_IterGVN()->_worklist.push(input); - } - return this; - } - } - return NULL; -} - -ShenandoahWriteBarrierNode::ShenandoahWriteBarrierNode(Compile* C, Node* ctrl, Node* mem, Node* obj) - : ShenandoahBarrierNode(ctrl, mem, obj, false) { - assert(UseShenandoahGC && ShenandoahWriteBarrier, "should be enabled"); - ShenandoahBarrierSetC2::bsc2()->state()->add_shenandoah_barrier(this); -} - -Node* ShenandoahWriteBarrierNode::Identity(PhaseGVN* phase) { - assert(in(0) != NULL, "should have control"); - PhaseIterGVN* igvn = phase->is_IterGVN(); - Node* mem_in = in(Memory); - Node* mem_proj = NULL; - - if (igvn != NULL) { - mem_proj = find_out_with(Op_ShenandoahWBMemProj); - if (mem_in == mem_proj) { - return this; - } - } - - Node* replacement = Identity_impl(phase); - if (igvn != NULL) { - if (replacement != NULL && replacement != this && mem_proj != NULL) { - igvn->replace_node(mem_proj, mem_in); - } - } - return replacement; -} - -Node* ShenandoahWriteBarrierNode::Ideal(PhaseGVN *phase, bool can_reshape) { - assert(in(0) != NULL, "should have control"); - if (!can_reshape) { - return NULL; - } - - Node* mem_in = in(Memory); - - if (mem_in->isa_MergeMem()) { - const TypePtr* adr_type = brooks_pointer_type(bottom_type()); - uint alias_idx = phase->C->get_alias_index(adr_type); - mem_in = mem_in->as_MergeMem()->memory_at(alias_idx); - set_req(Memory, mem_in); - return this; - } - - Node* val = in(ValueIn); - if (val->is_ShenandoahBarrier()) { - set_req(ValueIn, val->in(ValueIn)); - return this; - } - - return NULL; -} - -bool ShenandoahWriteBarrierNode::expand(Compile* C, PhaseIterGVN& igvn) { - if (UseShenandoahGC) { - if (ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_barriers_count() > 0 || (!ShenandoahWriteBarrier && ShenandoahStoreValEnqueueBarrier)) { - bool attempt_more_loopopts = ShenandoahLoopOptsAfterExpansion; C->clear_major_progress(); - PhaseIdealLoop ideal_loop(igvn, LoopOptsShenandoahExpand); - if (C->failing()) return false; - PhaseIdealLoop::verify(igvn); - DEBUG_ONLY(ShenandoahBarrierNode::verify_raw_mem(C->root());) - if (attempt_more_loopopts) { - C->set_major_progress(); - if (!C->optimize_loops(igvn, LoopOptsShenandoahPostExpand)) { - return false; - } - C->clear_major_progress(); - } } } return true; } -bool ShenandoahWriteBarrierNode::is_heap_state_test(Node* iff, int mask) { +bool ShenandoahBarrierC2Support::is_heap_state_test(Node* iff, int mask) { if (!UseShenandoahGC) { return false; } @@ -450,11 +95,11 @@ return is_gc_state_load(in1); } -bool ShenandoahWriteBarrierNode::is_heap_stable_test(Node* iff) { +bool ShenandoahBarrierC2Support::is_heap_stable_test(Node* iff) { return is_heap_state_test(iff, ShenandoahHeap::HAS_FORWARDED); } -bool ShenandoahWriteBarrierNode::is_gc_state_load(Node *n) { +bool ShenandoahBarrierC2Support::is_gc_state_load(Node *n) { if (!UseShenandoahGC) { return false; } @@ -476,7 +121,7 @@ return true; } -bool ShenandoahWriteBarrierNode::has_safepoint_between(Node* start, Node* stop, PhaseIdealLoop *phase) { +bool ShenandoahBarrierC2Support::has_safepoint_between(Node* start, Node* stop, PhaseIdealLoop *phase) { assert(phase->is_dominator(stop, start), "bad inputs"); ResourceMark rm; Unique_Node_List wq; @@ -500,7 +145,7 @@ return false; } -bool ShenandoahWriteBarrierNode::try_common_gc_state_load(Node *n, PhaseIdealLoop *phase) { +bool ShenandoahBarrierC2Support::try_common_gc_state_load(Node *n, PhaseIdealLoop *phase) { assert(is_gc_state_load(n), "inconsistent"); Node* addp = n->in(MemNode::Address); Node* dominator = NULL; @@ -525,193 +170,8 @@ return true; } -bool ShenandoahBarrierNode::dominates_memory_impl(PhaseGVN* phase, - Node* b1, - Node* b2, - Node* current, - bool linear) { - ResourceMark rm; - VectorSet visited(Thread::current()->resource_area()); - Node_Stack phis(0); - - for(int i = 0; i < 10; i++) { - if (current == NULL) { - return false; - } else if (visited.test_set(current->_idx) || current->is_top() || current == b1) { - current = NULL; - while (phis.is_nonempty() && current == NULL) { - uint idx = phis.index(); - Node* phi = phis.node(); - if (idx >= phi->req()) { - phis.pop(); - } else { - current = phi->in(idx); - phis.set_index(idx+1); - } - } - if (current == NULL) { - return true; - } - } else if (current == b2) { - return false; - } else if (current == phase->C->immutable_memory()) { - return false; - } else if (current->isa_Phi()) { - if (!linear) { - return false; - } - phis.push(current, 2); - current = current->in(1); - } else if (current->Opcode() == Op_ShenandoahWriteBarrier) { - current = current->in(Memory); - } else if (current->Opcode() == Op_ShenandoahWBMemProj) { - current = current->in(ShenandoahWBMemProjNode::WriteBarrier); - } else if (current->is_Proj()) { - current = current->in(0); - } else if (current->is_Call()) { - current = current->in(TypeFunc::Memory); - } else if (current->is_MemBar()) { - current = current->in(TypeFunc::Memory); - } else if (current->is_MergeMem()) { - const TypePtr* adr_type = brooks_pointer_type(phase->type(b2)); - uint alias_idx = phase->C->get_alias_index(adr_type); - current = current->as_MergeMem()->memory_at(alias_idx); - } else { #ifdef ASSERT - current->dump(); -#endif - ShouldNotReachHere(); - return false; - } - } - return false; -} - -/** - * Determines if b1 dominates b2 through memory inputs. It returns true if: - * - b1 can be reached by following each branch in b2's memory input (through phis, etc) - * - or we get back to b2 (i.e. through a loop) without seeing b1 - * In all other cases, (in particular, if we reach immutable_memory without having seen b1) - * we return false. - */ -bool ShenandoahBarrierNode::dominates_memory(PhaseGVN* phase, Node* b1, Node* b2, bool linear) { - return dominates_memory_impl(phase, b1, b2, b2->in(Memory), linear); -} - -Node* ShenandoahBarrierNode::Identity_impl(PhaseGVN* phase) { - Node* n = in(ValueIn); - - Node* rb_mem = Opcode() == Op_ShenandoahReadBarrier ? in(Memory) : NULL; - if (! needs_barrier(phase, this, n, rb_mem, _allow_fromspace)) { - return n; - } - - // Try to find a write barrier sibling with identical inputs that we can fold into. - for (DUIterator i = n->outs(); n->has_out(i); i++) { - Node* sibling = n->out(i); - if (sibling == this) { - continue; - } - if (sibling->Opcode() != Op_ShenandoahWriteBarrier) { - continue; - } - - assert(sibling->in(ValueIn) == in(ValueIn), "sanity"); - assert(sibling->Opcode() == Op_ShenandoahWriteBarrier, "sanity"); - - if (dominates_memory(phase, sibling, this, phase->is_IterGVN() == NULL)) { - return sibling; - } - } - return this; -} - -#ifndef PRODUCT -void ShenandoahBarrierNode::dump_spec(outputStream *st) const { - const TypePtr* adr = adr_type(); - if (adr == NULL) { - return; - } - st->print(" @"); - adr->dump_on(st); - st->print(" ("); - Compile::current()->alias_type(adr)->adr_type()->dump_on(st); - st->print(") "); -} -#endif - -Node* ShenandoahReadBarrierNode::Identity(PhaseGVN* phase) { - Node* id = Identity_impl(phase); - - if (id == this && phase->is_IterGVN()) { - Node* n = in(ValueIn); - // No success in super call. Try to combine identical read barriers. - for (DUIterator i = n->outs(); n->has_out(i); i++) { - Node* sibling = n->out(i); - if (sibling == this || sibling->Opcode() != Op_ShenandoahReadBarrier) { - continue; - } - assert(sibling->in(ValueIn) == in(ValueIn), "sanity"); - if (phase->is_IterGVN()->hash_find(sibling) && - sibling->bottom_type() == bottom_type() && - sibling->in(Control) == in(Control) && - dominates_memory_rb(phase, sibling, this, phase->is_IterGVN() == NULL)) { - return sibling; - } - } - } - return id; -} - -const Type* ShenandoahBarrierNode::Value(PhaseGVN* phase) const { - // Either input is TOP ==> the result is TOP - const Type *t1 = phase->type(in(Memory)); - if (t1 == Type::TOP) return Type::TOP; - const Type *t2 = phase->type(in(ValueIn)); - if( t2 == Type::TOP ) return Type::TOP; - - if (t2 == TypePtr::NULL_PTR) { - return _type; - } - - const Type* type = t2->is_oopptr()->cast_to_nonconst(); - return type; -} - -uint ShenandoahBarrierNode::hash() const { - return TypeNode::hash() + _allow_fromspace; -} - -bool ShenandoahBarrierNode::cmp(const Node& n) const { - return _allow_fromspace == ((ShenandoahBarrierNode&) n)._allow_fromspace - && TypeNode::cmp(n); -} - -uint ShenandoahBarrierNode::size_of() const { - return sizeof(*this); -} - -Node* ShenandoahWBMemProjNode::Identity(PhaseGVN* phase) { - Node* wb = in(WriteBarrier); - if (wb->is_top()) return phase->C->top(); // Dead path. - - assert(wb->Opcode() == Op_ShenandoahWriteBarrier, "expect write barrier"); - PhaseIterGVN* igvn = phase->is_IterGVN(); - // We can't do the below unless the graph is fully constructed. - if (igvn == NULL) { - return this; - } - - // If the mem projection has no barrier users, it's not needed anymore. - if (wb->outcnt() == 1) { - return wb->in(ShenandoahBarrierNode::Memory); - } - - return this; -} - -#ifdef ASSERT -bool ShenandoahBarrierNode::verify_helper(Node* in, Node_Stack& phis, VectorSet& visited, verify_type t, bool trace, Unique_Node_List& barriers_used) { +bool ShenandoahBarrierC2Support::verify_helper(Node* in, Node_Stack& phis, VectorSet& visited, verify_type t, bool trace, Unique_Node_List& barriers_used) { assert(phis.size() == 0, ""); while (true) { @@ -732,12 +192,24 @@ in = in->in(AddPNode::Address); continue; } else if (in->is_Con()) { - if (trace) {tty->print("Found constant"); in->dump();} - } else if (in->is_ShenandoahBarrier()) { + if (trace) { + tty->print("Found constant"); + in->dump(); + } + } else if (in->Opcode() == Op_Parm) { + if (trace) { + tty->print("Found argument"); + } + } else if (in->Opcode() == Op_CreateEx) { + if (trace) { + tty->print("Found create-exception"); + } + } else if (in->Opcode() == Op_LoadP && in->adr_type() == TypeRawPtr::BOTTOM) { + if (trace) { + tty->print("Found raw LoadP (OSR argument?)"); + } + } else if (in->Opcode() == Op_ShenandoahLoadReferenceBarrier) { if (t == ShenandoahOopStore) { - if (in->Opcode() != Op_ShenandoahWriteBarrier) { - return false; - } uint i = 0; for (; i < phis.size(); i++) { Node* n = phis.node_at(i); @@ -748,8 +220,6 @@ if (i == phis.size()) { return false; } - } else if (t == ShenandoahStore && in->Opcode() != Op_ShenandoahWriteBarrier) { - return false; } barriers_used.push(in); if (trace) {tty->print("Found barrier"); in->dump();} @@ -763,7 +233,14 @@ in = in->in(1); continue; } else if (in->is_Proj() && in->in(0)->is_Allocate()) { - if (trace) {tty->print("Found alloc"); in->in(0)->dump();} + if (trace) { + tty->print("Found alloc"); + in->in(0)->dump(); + } + } else if (in->is_Proj() && (in->in(0)->Opcode() == Op_CallStaticJava || in->in(0)->Opcode() == Op_CallDynamicJava)) { + if (trace) { + tty->print("Found Java call"); + } } else if (in->is_Phi()) { if (!visited.test_set(in->_idx)) { if (trace) {tty->print("Pushed phi:"); in->dump();} @@ -809,7 +286,7 @@ return true; } -void ShenandoahBarrierNode::report_verify_failure(const char *msg, Node *n1, Node *n2) { +void ShenandoahBarrierC2Support::report_verify_failure(const char* msg, Node* n1, Node* n2) { if (n1 != NULL) { n1->dump(+10); } @@ -819,7 +296,7 @@ fatal("%s", msg); } -void ShenandoahBarrierNode::verify(RootNode* root) { +void ShenandoahBarrierC2Support::verify(RootNode* root) { ResourceMark rm; Unique_Node_List wq; GrowableArray barriers; @@ -871,7 +348,7 @@ } } - if (verify && !ShenandoahBarrierNode::verify_helper(n->in(MemNode::Address), phis, visited, ShenandoahLoad, trace, barriers_used)) { + if (verify && !verify_helper(n->in(MemNode::Address), phis, visited, ShenandoahLoad, trace, barriers_used)) { report_verify_failure("Shenandoah verification: Load should have barriers", n); } } @@ -899,11 +376,11 @@ } } - if (verify && !ShenandoahBarrierNode::verify_helper(n->in(MemNode::ValueIn), phis, visited, ShenandoahStoreValEnqueueBarrier ? ShenandoahOopStore : ShenandoahValue, trace, barriers_used)) { + if (verify && !verify_helper(n->in(MemNode::ValueIn), phis, visited, ShenandoahStoreValEnqueueBarrier ? ShenandoahOopStore : ShenandoahValue, trace, barriers_used)) { report_verify_failure("Shenandoah verification: Store should have barriers", n); } } - if (!ShenandoahBarrierNode::verify_helper(n->in(MemNode::Address), phis, visited, ShenandoahStore, trace, barriers_used)) { + if (!verify_helper(n->in(MemNode::Address), phis, visited, ShenandoahStore, trace, barriers_used)) { report_verify_failure("Shenandoah verification: Store (address) should have barriers", n); } } else if (n->Opcode() == Op_CmpP) { @@ -926,26 +403,26 @@ } else { assert(in2->bottom_type()->isa_oopptr(), ""); - if (!ShenandoahBarrierNode::verify_helper(in1, phis, visited, ShenandoahStore, trace, barriers_used) || - !ShenandoahBarrierNode::verify_helper(in2, phis, visited, ShenandoahStore, trace, barriers_used)) { + if (!verify_helper(in1, phis, visited, ShenandoahStore, trace, barriers_used) || + !verify_helper(in2, phis, visited, ShenandoahStore, trace, barriers_used)) { report_verify_failure("Shenandoah verification: Cmp should have barriers", n); } } if (verify_no_useless_barrier && mark_inputs && - (!ShenandoahBarrierNode::verify_helper(in1, phis, visited, ShenandoahValue, trace, barriers_used) || - !ShenandoahBarrierNode::verify_helper(in2, phis, visited, ShenandoahValue, trace, barriers_used))) { + (!verify_helper(in1, phis, visited, ShenandoahValue, trace, barriers_used) || + !verify_helper(in2, phis, visited, ShenandoahValue, trace, barriers_used))) { phis.clear(); visited.Reset(); } } } else if (n->is_LoadStore()) { if (n->in(MemNode::ValueIn)->bottom_type()->make_ptr() && - !ShenandoahBarrierNode::verify_helper(n->in(MemNode::ValueIn), phis, visited, ShenandoahStoreValEnqueueBarrier ? ShenandoahOopStore : ShenandoahValue, trace, barriers_used)) { + !verify_helper(n->in(MemNode::ValueIn), phis, visited, ShenandoahStoreValEnqueueBarrier ? ShenandoahOopStore : ShenandoahValue, trace, barriers_used)) { report_verify_failure("Shenandoah verification: LoadStore (value) should have barriers", n); } - if (n->in(MemNode::Address)->bottom_type()->make_oopptr() && !ShenandoahBarrierNode::verify_helper(n->in(MemNode::Address), phis, visited, ShenandoahStore, trace, barriers_used)) { + if (n->in(MemNode::Address)->bottom_type()->make_oopptr() && !verify_helper(n->in(MemNode::Address), phis, visited, ShenandoahStore, trace, barriers_used)) { report_verify_failure("Shenandoah verification: LoadStore (address) should have barriers", n); } } else if (n->Opcode() == Op_CallLeafNoFP || n->Opcode() == Op_CallLeaf) { @@ -1041,13 +518,13 @@ } } } - if (!ShenandoahBarrierNode::verify_helper(n->in(TypeFunc::Parms), phis, visited, ShenandoahLoad, trace, barriers_used) || - !ShenandoahBarrierNode::verify_helper(dest, phis, visited, ShenandoahStore, trace, barriers_used)) { + if (!verify_helper(n->in(TypeFunc::Parms), phis, visited, ShenandoahLoad, trace, barriers_used) || + !verify_helper(dest, phis, visited, ShenandoahStore, trace, barriers_used)) { report_verify_failure("Shenandoah verification: ArrayCopy should have barriers", n); } } else if (strlen(call->_name) > 5 && !strcmp(call->_name + strlen(call->_name) - 5, "_fill")) { - if (!ShenandoahBarrierNode::verify_helper(n->in(TypeFunc::Parms), phis, visited, ShenandoahStore, trace, barriers_used)) { + if (!verify_helper(n->in(TypeFunc::Parms), phis, visited, ShenandoahStore, trace, barriers_used)) { report_verify_failure("Shenandoah verification: _fill should have barriers", n); } } else if (!strcmp(call->_name, "shenandoah_wb_pre")) { @@ -1067,7 +544,7 @@ if (pos == -1) { break; } - if (!ShenandoahBarrierNode::verify_helper(call->in(pos), phis, visited, calls[i].args[j].t, trace, barriers_used)) { + if (!verify_helper(call->in(pos), phis, visited, calls[i].args[j].t, trace, barriers_used)) { report_verify_failure("Shenandoah verification: intrinsic calls should have barriers", n); } } @@ -1090,15 +567,8 @@ } } } - } else if (n->is_ShenandoahBarrier()) { - assert(!barriers.contains(n), ""); - assert(n->Opcode() != Op_ShenandoahWriteBarrier || n->find_out_with(Op_ShenandoahWBMemProj) != NULL, "bad shenandoah write barrier"); - assert(n->Opcode() != Op_ShenandoahWriteBarrier || n->outcnt() > 1, "bad shenandoah write barrier"); - barriers.push(n); - } else if (n->Opcode() == Op_ShenandoahEnqueueBarrier) { + } else if (n->Opcode() == Op_ShenandoahEnqueueBarrier || n->Opcode() == Op_ShenandoahLoadReferenceBarrier) { // skip - } else if (n->Opcode() == Op_ShenandoahWBMemProj) { - assert(n->in(0) == NULL && n->in(ShenandoahWBMemProjNode::WriteBarrier)->Opcode() == Op_ShenandoahWriteBarrier, "strange ShenandoahWBMemProj"); } else if (n->is_AddP() || n->is_Phi() || n->is_ConstraintCast() @@ -1165,7 +635,7 @@ if (pos == -1) { break; } - if (!ShenandoahBarrierNode::verify_helper(n->in(pos), phis, visited, others[i].inputs[j].t, trace, barriers_used)) { + if (!verify_helper(n->in(pos), phis, visited, others[i].inputs[j].t, trace, barriers_used)) { report_verify_failure("Shenandoah verification: intrinsic calls should have barriers", n); } } @@ -1193,7 +663,7 @@ SafePointNode* sfpt = n->as_SafePoint(); if (verify_no_useless_barrier && sfpt->jvms() != NULL) { for (uint i = sfpt->jvms()->scloff(); i < sfpt->jvms()->endoff(); i++) { - if (!ShenandoahBarrierNode::verify_helper(sfpt->in(i), phis, visited, ShenandoahLoad, trace, barriers_used)) { + if (!verify_helper(sfpt->in(i), phis, visited, ShenandoahLoad, trace, barriers_used)) { phis.clear(); visited.Reset(); } @@ -1227,9 +697,8 @@ n->Opcode() == Op_SCMemProj || n->Opcode() == Op_EncodeP || n->Opcode() == Op_DecodeN || - n->Opcode() == Op_ShenandoahWriteBarrier || - n->Opcode() == Op_ShenandoahWBMemProj || - n->Opcode() == Op_ShenandoahEnqueueBarrier)) { + n->Opcode() == Op_ShenandoahEnqueueBarrier || + n->Opcode() == Op_ShenandoahLoadReferenceBarrier)) { if (m->bottom_type()->make_oopptr() && m->bottom_type()->make_oopptr()->meet(TypePtr::NULL_PTR) == m->bottom_type()) { report_verify_failure("Shenandoah verification: null input", n, m); } @@ -1251,7 +720,7 @@ } #endif -bool ShenandoahBarrierNode::is_dominator_same_ctrl(Node*c, Node* d, Node* n, PhaseIdealLoop* phase) { +bool ShenandoahBarrierC2Support::is_dominator_same_ctrl(Node* c, Node* d, Node* n, PhaseIdealLoop* phase) { // That both nodes have the same control is not sufficient to prove // domination, verify that there's no path from d to n ResourceMark rm; @@ -1275,7 +744,7 @@ return true; } -bool ShenandoahBarrierNode::is_dominator(Node *d_c, Node *n_c, Node* d, Node* n, PhaseIdealLoop* phase) { +bool ShenandoahBarrierC2Support::is_dominator(Node* d_c, Node* n_c, Node* d, Node* n, PhaseIdealLoop* phase) { if (d_c != n_c) { return phase->is_dominator(d_c, n_c); } @@ -1290,15 +759,11 @@ res = mem->in(TypeFunc::Memory); } else if (mem->is_Phi()) { res = mem->in(1); - } else if (mem->is_ShenandoahBarrier()) { - res = mem->in(ShenandoahBarrierNode::Memory); } else if (mem->is_MergeMem()) { res = mem->as_MergeMem()->memory_at(alias); } else if (mem->is_Store() || mem->is_LoadStore() || mem->is_ClearArray()) { assert(alias = Compile::AliasIdxRaw, "following raw memory can't lead to a barrier"); res = mem->in(MemNode::Memory); - } else if (mem->Opcode() == Op_ShenandoahWBMemProj) { - res = mem->in(ShenandoahWBMemProjNode::WriteBarrier); } else { #ifdef ASSERT mem->dump(); @@ -1308,7 +773,7 @@ return res; } -Node* ShenandoahBarrierNode::no_branches(Node* c, Node* dom, bool allow_one_proj, PhaseIdealLoop* phase) { +Node* ShenandoahBarrierC2Support::no_branches(Node* c, Node* dom, bool allow_one_proj, PhaseIdealLoop* phase) { Node* iffproj = NULL; while (c != dom) { Node* next = phase->idom(c); @@ -1373,270 +838,7 @@ return iffproj; } -bool ShenandoahBarrierNode::build_loop_late_post(PhaseIdealLoop* phase, Node* n) { - if (n->Opcode() == Op_ShenandoahReadBarrier || - n->Opcode() == Op_ShenandoahWriteBarrier || - n->Opcode() == Op_ShenandoahWBMemProj) { - - phase->build_loop_late_post_work(n, false); - - if (n->Opcode() == Op_ShenandoahWriteBarrier) { - // The write barrier and its memory proj must have the same - // control otherwise some loop opts could put nodes (Phis) between - // them - Node* proj = n->find_out_with(Op_ShenandoahWBMemProj); - if (proj != NULL) { - phase->set_ctrl_and_loop(proj, phase->get_ctrl(n)); - } - } - return true; - } - return false; -} - -bool ShenandoahBarrierNode::sink_node(PhaseIdealLoop* phase, Node* ctrl, Node* n_ctrl) { - ctrl = phase->find_non_split_ctrl(ctrl); - assert(phase->dom_depth(n_ctrl) <= phase->dom_depth(ctrl), "n is later than its clone"); - set_req(0, ctrl); - phase->register_new_node(this, ctrl); - return true; -} - -#ifdef ASSERT -void ShenandoahWriteBarrierNode::memory_dominates_all_paths_helper(Node* c, Node* rep_ctrl, Unique_Node_List& controls, PhaseIdealLoop* phase) { - const bool trace = false; - if (trace) { tty->print("X control is"); c->dump(); } - - uint start = controls.size(); - controls.push(c); - for (uint i = start; i < controls.size(); i++) { - Node *n = controls.at(i); - - if (trace) { tty->print("X from"); n->dump(); } - - if (n == rep_ctrl) { - continue; - } - - if (n->is_Proj()) { - Node* n_dom = n->in(0); - IdealLoopTree* n_dom_loop = phase->get_loop(n_dom); - if (n->is_IfProj() && n_dom->outcnt() == 2) { - n_dom_loop = phase->get_loop(n_dom->as_If()->proj_out(n->as_Proj()->_con == 0 ? 1 : 0)); - } - if (n_dom_loop != phase->ltree_root()) { - Node* tail = n_dom_loop->tail(); - if (tail->is_Region()) { - for (uint j = 1; j < tail->req(); j++) { - if (phase->is_dominator(n_dom, tail->in(j)) && !phase->is_dominator(n, tail->in(j))) { - assert(phase->is_dominator(rep_ctrl, tail->in(j)), "why are we here?"); - // entering loop from below, mark backedge - if (trace) { tty->print("X pushing backedge"); tail->in(j)->dump(); } - controls.push(tail->in(j)); - //assert(n->in(0) == n_dom, "strange flow control"); - } - } - } else if (phase->get_loop(n) != n_dom_loop && phase->is_dominator(n_dom, tail)) { - // entering loop from below, mark backedge - if (trace) { tty->print("X pushing backedge"); tail->dump(); } - controls.push(tail); - //assert(n->in(0) == n_dom, "strange flow control"); - } - } - } - - if (n->is_Loop()) { - Node* c = n->in(LoopNode::EntryControl); - if (trace) { tty->print("X pushing"); c->dump(); } - controls.push(c); - } else if (n->is_Region()) { - for (uint i = 1; i < n->req(); i++) { - Node* c = n->in(i); - if (trace) { tty->print("X pushing"); c->dump(); } - controls.push(c); - } - } else { - Node* c = n->in(0); - if (trace) { tty->print("X pushing"); c->dump(); } - controls.push(c); - } - } -} - -bool ShenandoahWriteBarrierNode::memory_dominates_all_paths(Node* mem, Node* rep_ctrl, int alias, PhaseIdealLoop* phase) { - const bool trace = false; - if (trace) { - tty->print("XXX mem is"); mem->dump(); - tty->print("XXX rep ctrl is"); rep_ctrl->dump(); - tty->print_cr("XXX alias is %d", alias); - } - ResourceMark rm; - Unique_Node_List wq; - Unique_Node_List controls; - wq.push(mem); - for (uint next = 0; next < wq.size(); next++) { - Node *nn = wq.at(next); - if (trace) { tty->print("XX from mem"); nn->dump(); } - assert(nn->bottom_type() == Type::MEMORY, "memory only"); - - if (nn->is_Phi()) { - Node* r = nn->in(0); - for (DUIterator_Fast jmax, j = r->fast_outs(jmax); j < jmax; j++) { - Node* u = r->fast_out(j); - if (u->is_Phi() && u->bottom_type() == Type::MEMORY && u != nn && - (u->adr_type() == TypePtr::BOTTOM || phase->C->get_alias_index(u->adr_type()) == alias)) { - if (trace) { tty->print("XX Next mem (other phi)"); u->dump(); } - wq.push(u); - } - } - } - - for (DUIterator_Fast imax, i = nn->fast_outs(imax); i < imax; i++) { - Node* use = nn->fast_out(i); - - if (trace) { tty->print("XX use %p", use->adr_type()); use->dump(); } - if (use->is_CFG() && use->in(TypeFunc::Memory) == nn) { - Node* c = use->in(0); - if (phase->is_dominator(rep_ctrl, c)) { - memory_dominates_all_paths_helper(c, rep_ctrl, controls, phase); - } else if (use->is_CallStaticJava() && use->as_CallStaticJava()->uncommon_trap_request() != 0 && c->is_Region()) { - Node* region = c; - if (trace) { tty->print("XX unc region"); region->dump(); } - for (uint j = 1; j < region->req(); j++) { - if (phase->is_dominator(rep_ctrl, region->in(j))) { - if (trace) { tty->print("XX unc follows"); region->in(j)->dump(); } - memory_dominates_all_paths_helper(region->in(j), rep_ctrl, controls, phase); - } - } - } - //continue; - } else if (use->is_Phi()) { - assert(use->bottom_type() == Type::MEMORY, "bad phi"); - if ((use->adr_type() == TypePtr::BOTTOM) || - phase->C->get_alias_index(use->adr_type()) == alias) { - for (uint j = 1; j < use->req(); j++) { - if (use->in(j) == nn) { - Node* c = use->in(0)->in(j); - if (phase->is_dominator(rep_ctrl, c)) { - memory_dominates_all_paths_helper(c, rep_ctrl, controls, phase); - } - } - } - } - // continue; - } - - if (use->is_MergeMem()) { - if (use->as_MergeMem()->memory_at(alias) == nn) { - if (trace) { tty->print("XX Next mem"); use->dump(); } - // follow the memory edges - wq.push(use); - } - } else if (use->is_Phi()) { - assert(use->bottom_type() == Type::MEMORY, "bad phi"); - if ((use->adr_type() == TypePtr::BOTTOM) || - phase->C->get_alias_index(use->adr_type()) == alias) { - if (trace) { tty->print("XX Next mem"); use->dump(); } - // follow the memory edges - wq.push(use); - } - } else if (use->bottom_type() == Type::MEMORY && - (use->adr_type() == TypePtr::BOTTOM || phase->C->get_alias_index(use->adr_type()) == alias)) { - if (trace) { tty->print("XX Next mem"); use->dump(); } - // follow the memory edges - wq.push(use); - } else if ((use->is_SafePoint() || use->is_MemBar()) && - (use->adr_type() == TypePtr::BOTTOM || phase->C->get_alias_index(use->adr_type()) == alias)) { - for (DUIterator_Fast jmax, j = use->fast_outs(jmax); j < jmax; j++) { - Node* u = use->fast_out(j); - if (u->bottom_type() == Type::MEMORY) { - if (trace) { tty->print("XX Next mem"); u->dump(); } - // follow the memory edges - wq.push(u); - } - } - } else if (use->Opcode() == Op_ShenandoahWriteBarrier && phase->C->get_alias_index(use->adr_type()) == alias) { - Node* m = use->find_out_with(Op_ShenandoahWBMemProj); - if (m != NULL) { - if (trace) { tty->print("XX Next mem"); m->dump(); } - // follow the memory edges - wq.push(m); - } - } - } - } - - if (controls.size() == 0) { - return false; - } - - for (uint i = 0; i < controls.size(); i++) { - Node *n = controls.at(i); - - if (trace) { tty->print("X checking"); n->dump(); } - - if (n->unique_ctrl_out() != NULL) { - continue; - } - - if (n->Opcode() == Op_NeverBranch) { - Node* taken = n->as_Multi()->proj_out(0); - if (!controls.member(taken)) { - if (trace) { tty->print("X not seen"); taken->dump(); } - return false; - } - continue; - } - - for (DUIterator_Fast jmax, j = n->fast_outs(jmax); j < jmax; j++) { - Node* u = n->fast_out(j); - - if (u->is_CFG()) { - if (!controls.member(u)) { - if (u->is_Proj() && u->as_Proj()->is_uncommon_trap_proj(Deoptimization::Reason_none)) { - if (trace) { tty->print("X not seen but unc"); u->dump(); } - } else { - Node* c = u; - do { - c = c->unique_ctrl_out(); - } while (c != NULL && c->is_Region()); - if (c != NULL && c->Opcode() == Op_Halt) { - if (trace) { tty->print("X not seen but halt"); c->dump(); } - } else { - if (trace) { tty->print("X not seen"); u->dump(); } - return false; - } - } - } else { - if (trace) { tty->print("X seen"); u->dump(); } - } - } - } - } - return true; -} -#endif - -Node* ShenandoahBarrierNode::dom_mem(Node* mem, Node*& mem_ctrl, Node* n, Node* rep_ctrl, int alias, PhaseIdealLoop* phase) { - ResourceMark rm; - VectorSet wq(Thread::current()->resource_area()); - wq.set(mem->_idx); - mem_ctrl = phase->get_ctrl(mem); - while (!is_dominator(mem_ctrl, rep_ctrl, mem, n, phase)) { - mem = next_mem(mem, alias); - if (wq.test_set(mem->_idx)) { - return NULL; // hit an unexpected loop - } - mem_ctrl = phase->ctrl_or_self(mem); - } - if (mem->is_MergeMem()) { - mem = mem->as_MergeMem()->memory_at(alias); - mem_ctrl = phase->ctrl_or_self(mem); - } - return mem; -} - -Node* ShenandoahBarrierNode::dom_mem(Node* mem, Node* ctrl, int alias, Node*& mem_ctrl, PhaseIdealLoop* phase) { +Node* ShenandoahBarrierC2Support::dom_mem(Node* mem, Node* ctrl, int alias, Node*& mem_ctrl, PhaseIdealLoop* phase) { ResourceMark rm; VectorSet wq(Thread::current()->resource_area()); wq.set(mem->_idx); @@ -1655,650 +857,7 @@ return mem; } -static void disconnect_barrier_mem(Node* wb, PhaseIterGVN& igvn) { - Node* mem_in = wb->in(ShenandoahBarrierNode::Memory); - Node* proj = wb->find_out_with(Op_ShenandoahWBMemProj); - - for (DUIterator_Last imin, i = proj->last_outs(imin); i >= imin; ) { - Node* u = proj->last_out(i); - igvn.rehash_node_delayed(u); - int nb = u->replace_edge(proj, mem_in); - assert(nb > 0, "no replacement?"); - i -= nb; - } -} - -Node* ShenandoahWriteBarrierNode::move_above_predicates(LoopNode* cl, Node* val_ctrl, PhaseIdealLoop* phase) { - Node* entry = cl->skip_strip_mined(-1)->in(LoopNode::EntryControl); - Node* above_pred = phase->skip_all_loop_predicates(entry); - Node* ctrl = entry; - while (ctrl != above_pred) { - Node* next = ctrl->in(0); - if (!phase->is_dominator(val_ctrl, next)) { - break; - } - ctrl = next; - } - return ctrl; -} - -static MemoryGraphFixer* find_fixer(GrowableArray& memory_graph_fixers, int alias) { - for (int i = 0; i < memory_graph_fixers.length(); i++) { - if (memory_graph_fixers.at(i)->alias() == alias) { - return memory_graph_fixers.at(i); - } - } - return NULL; -} - -static MemoryGraphFixer* create_fixer(GrowableArray& memory_graph_fixers, int alias, PhaseIdealLoop* phase, bool include_lsm) { - assert(find_fixer(memory_graph_fixers, alias) == NULL, "none should exist yet"); - MemoryGraphFixer* fixer = new MemoryGraphFixer(alias, include_lsm, phase); - memory_graph_fixers.push(fixer); - return fixer; -} - -void ShenandoahWriteBarrierNode::try_move_before_loop_helper(LoopNode* cl, Node* val_ctrl, GrowableArray& memory_graph_fixers, PhaseIdealLoop* phase, bool include_lsm, Unique_Node_List& uses) { - assert(cl->is_Loop(), "bad control"); - Node* ctrl = move_above_predicates(cl, val_ctrl, phase); - Node* mem_ctrl = NULL; - int alias = phase->C->get_alias_index(adr_type()); - - MemoryGraphFixer* fixer = find_fixer(memory_graph_fixers, alias); - if (fixer == NULL) { - fixer = create_fixer(memory_graph_fixers, alias, phase, include_lsm); - } - - Node* proj = find_out_with(Op_ShenandoahWBMemProj); - - fixer->remove(proj); - Node* mem = fixer->find_mem(ctrl, NULL); - - assert(!ShenandoahVerifyOptoBarriers || memory_dominates_all_paths(mem, ctrl, alias, phase), "can't fix the memory graph"); - - phase->set_ctrl_and_loop(this, ctrl); - phase->igvn().replace_input_of(this, Control, ctrl); - - disconnect_barrier_mem(this, phase->igvn()); - - phase->igvn().replace_input_of(this, Memory, mem); - phase->set_ctrl_and_loop(proj, ctrl); - - fixer->fix_mem(ctrl, ctrl, mem, mem, proj, uses); - assert(proj->outcnt() > 0, "disconnected write barrier"); -} - -LoopNode* ShenandoahWriteBarrierNode::try_move_before_pre_loop(Node* c, Node* val_ctrl, PhaseIdealLoop* phase) { - // A write barrier between a pre and main loop can get in the way of - // vectorization. Move it above the pre loop if possible - CountedLoopNode* cl = NULL; - if (c->is_IfFalse() && - c->in(0)->is_CountedLoopEnd()) { - cl = c->in(0)->as_CountedLoopEnd()->loopnode(); - } else if (c->is_IfProj() && - c->in(0)->is_If() && - c->in(0)->in(0)->is_IfFalse() && - c->in(0)->in(0)->in(0)->is_CountedLoopEnd()) { - cl = c->in(0)->in(0)->in(0)->as_CountedLoopEnd()->loopnode(); - } - if (cl != NULL && - cl->is_pre_loop() && - val_ctrl != cl && - phase->is_dominator(val_ctrl, cl)) { - return cl; - } - return NULL; -} - -void ShenandoahWriteBarrierNode::try_move_before_loop(GrowableArray& memory_graph_fixers, PhaseIdealLoop* phase, bool include_lsm, Unique_Node_List& uses) { - Node *n_ctrl = phase->get_ctrl(this); - IdealLoopTree *n_loop = phase->get_loop(n_ctrl); - Node* val = in(ValueIn); - Node* val_ctrl = phase->get_ctrl(val); - if (n_loop != phase->ltree_root() && !n_loop->_irreducible) { - IdealLoopTree *val_loop = phase->get_loop(val_ctrl); - Node* mem = in(Memory); - IdealLoopTree *mem_loop = phase->get_loop(phase->get_ctrl(mem)); - if (!n_loop->is_member(val_loop) && - n_loop->is_member(mem_loop)) { - Node* n_loop_head = n_loop->_head; - - if (n_loop_head->is_Loop()) { - LoopNode* loop = n_loop_head->as_Loop(); - if (n_loop_head->is_CountedLoop() && n_loop_head->as_CountedLoop()->is_main_loop()) { - LoopNode* res = try_move_before_pre_loop(n_loop_head->in(LoopNode::EntryControl), val_ctrl, phase); - if (res != NULL) { - loop = res; - } - } - - try_move_before_loop_helper(loop, val_ctrl, memory_graph_fixers, phase, include_lsm, uses); - } - } - } - LoopNode* ctrl = try_move_before_pre_loop(in(0), val_ctrl, phase); - if (ctrl != NULL) { - try_move_before_loop_helper(ctrl, val_ctrl, memory_graph_fixers, phase, include_lsm, uses); - } -} - -Node* ShenandoahWriteBarrierNode::would_subsume(ShenandoahBarrierNode* other, PhaseIdealLoop* phase) { - Node* val = in(ValueIn); - Node* val_ctrl = phase->get_ctrl(val); - Node* other_mem = other->in(Memory); - Node* other_ctrl = phase->get_ctrl(other); - Node* this_ctrl = phase->get_ctrl(this); - IdealLoopTree* this_loop = phase->get_loop(this_ctrl); - IdealLoopTree* other_loop = phase->get_loop(other_ctrl); - - Node* ctrl = phase->dom_lca(other_ctrl, this_ctrl); - - if (ctrl->is_Proj() && - ctrl->in(0)->is_Call() && - ctrl->unique_ctrl_out() != NULL && - ctrl->unique_ctrl_out()->Opcode() == Op_Catch && - !phase->is_dominator(val_ctrl, ctrl->in(0)->in(0))) { - return NULL; - } - - IdealLoopTree* loop = phase->get_loop(ctrl); - - // We don't want to move a write barrier in a loop - // If the LCA is in a inner loop, try a control out of loop if possible - while (!loop->is_member(this_loop) && (other->Opcode() != Op_ShenandoahWriteBarrier || !loop->is_member(other_loop))) { - ctrl = phase->idom(ctrl); - if (ctrl->is_MultiBranch()) { - ctrl = ctrl->in(0); - } - if (ctrl != val_ctrl && phase->is_dominator(ctrl, val_ctrl)) { - return NULL; - } - loop = phase->get_loop(ctrl); - } - - if (ShenandoahDontIncreaseWBFreq) { - Node* this_iffproj = no_branches(this_ctrl, ctrl, true, phase); - if (other->Opcode() == Op_ShenandoahWriteBarrier) { - Node* other_iffproj = no_branches(other_ctrl, ctrl, true, phase); - if (other_iffproj == NULL || this_iffproj == NULL) { - return ctrl; - } else if (other_iffproj != NodeSentinel && this_iffproj != NodeSentinel && - other_iffproj->in(0) == this_iffproj->in(0)) { - return ctrl; - } - } else if (this_iffproj == NULL) { - return ctrl; - } - return NULL; - } - - return ctrl; -} - -void ShenandoahWriteBarrierNode::optimize_before_expansion(PhaseIdealLoop* phase, GrowableArray memory_graph_fixers, bool include_lsm) { - bool progress = false; - Unique_Node_List uses; - do { - progress = false; - for (int i = 0; i < ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_barriers_count(); i++) { - ShenandoahWriteBarrierNode* wb = ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_barrier(i); - - wb->try_move_before_loop(memory_graph_fixers, phase, include_lsm, uses); - - Node* val = wb->in(ValueIn); - - for (DUIterator_Fast jmax, j = val->fast_outs(jmax); j < jmax; j++) { - Node* u = val->fast_out(j); - if (u != wb && u->is_ShenandoahBarrier()) { - Node* rep_ctrl = wb->would_subsume(u->as_ShenandoahBarrier(), phase); - - if (rep_ctrl != NULL) { - Node* other = u; - Node* val_ctrl = phase->get_ctrl(val); - if (rep_ctrl->is_Proj() && - rep_ctrl->in(0)->is_Call() && - rep_ctrl->unique_ctrl_out() != NULL && - rep_ctrl->unique_ctrl_out()->Opcode() == Op_Catch) { - rep_ctrl = rep_ctrl->in(0)->in(0); - - assert(phase->is_dominator(val_ctrl, rep_ctrl), "bad control"); - } else { - LoopNode* c = ShenandoahWriteBarrierNode::try_move_before_pre_loop(rep_ctrl, val_ctrl, phase); - if (c != NULL) { - rep_ctrl = ShenandoahWriteBarrierNode::move_above_predicates(c, val_ctrl, phase); - } else { - while (rep_ctrl->is_IfProj()) { - CallStaticJavaNode* unc = rep_ctrl->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none); - if (unc != NULL) { - int req = unc->uncommon_trap_request(); - Deoptimization::DeoptReason trap_reason = Deoptimization::trap_request_reason(req); - if ((trap_reason == Deoptimization::Reason_loop_limit_check || - trap_reason == Deoptimization::Reason_predicate || - trap_reason == Deoptimization::Reason_profile_predicate) && - phase->is_dominator(val_ctrl, rep_ctrl->in(0)->in(0))) { - rep_ctrl = rep_ctrl->in(0)->in(0); - continue; - } - } - break; - } - } - } - - Node* wb_ctrl = phase->get_ctrl(wb); - Node* other_ctrl = phase->get_ctrl(other); - int alias = phase->C->get_alias_index(wb->adr_type()); - MemoryGraphFixer* fixer = find_fixer(memory_graph_fixers, alias);; - if (!is_dominator(wb_ctrl, other_ctrl, wb, other, phase)) { - if (fixer == NULL) { - fixer = create_fixer(memory_graph_fixers, alias, phase, include_lsm); - } - Node* mem = fixer->find_mem(rep_ctrl, phase->get_ctrl(other) == rep_ctrl ? other : NULL); - - if (mem->has_out_with(Op_Lock) || mem->has_out_with(Op_Unlock)) { - continue; - } - - Node* wb_proj = wb->find_out_with(Op_ShenandoahWBMemProj); - fixer->remove(wb_proj); - Node* mem_for_ctrl = fixer->find_mem(rep_ctrl, NULL); - - if (wb->in(Memory) != mem) { - disconnect_barrier_mem(wb, phase->igvn()); - phase->igvn().replace_input_of(wb, Memory, mem); - } - if (rep_ctrl != wb_ctrl) { - phase->set_ctrl_and_loop(wb, rep_ctrl); - phase->igvn().replace_input_of(wb, Control, rep_ctrl); - phase->set_ctrl_and_loop(wb_proj, rep_ctrl); - progress = true; - } - - fixer->fix_mem(rep_ctrl, rep_ctrl, mem, mem_for_ctrl, wb_proj, uses); - - assert(!ShenandoahVerifyOptoBarriers || ShenandoahWriteBarrierNode::memory_dominates_all_paths(mem, rep_ctrl, alias, phase), "can't fix the memory graph"); - } - - if (other->Opcode() == Op_ShenandoahWriteBarrier) { - Node* other_proj = other->find_out_with(Op_ShenandoahWBMemProj); - if (fixer != NULL) { - fixer->remove(other_proj); - } - phase->igvn().replace_node(other_proj, other->in(Memory)); - } - phase->igvn().replace_node(other, wb); - --j; --jmax; - } - } - } - } - } while(progress); -} - -// Some code duplication with PhaseIdealLoop::split_if_with_blocks_pre() -Node* ShenandoahWriteBarrierNode::try_split_thru_phi(PhaseIdealLoop* phase) { - Node *ctrl = phase->get_ctrl(this); - if (ctrl == NULL) { - return this; - } - Node *blk = phase->has_local_phi_input(this); - if (blk == NULL) { - return this; - } - - if (in(0) != blk) { - return this; - } - - int policy = blk->req() >> 2; - - if (blk->is_CountedLoop()) { - IdealLoopTree *lp = phase->get_loop(blk); - if (lp && lp->_rce_candidate) { - return this; - } - } - - if (phase->C->live_nodes() > 35000) { - return this; - } - - uint unique = phase->C->unique(); - Node *phi = phase->split_thru_phi(this, blk, policy); - if (phi == NULL) { - return this; - } - - Node* mem_phi = new PhiNode(blk, Type::MEMORY, phase->C->alias_type(adr_type())->adr_type()); - for (uint i = 1; i < blk->req(); i++) { - Node* n = phi->in(i); - if (n->Opcode() == Op_ShenandoahWriteBarrier && - n->_idx >= unique) { - Node* proj = new ShenandoahWBMemProjNode(n); - phase->register_new_node(proj, phase->get_ctrl(n)); - mem_phi->init_req(i, proj); - } else { - Node* mem = in(ShenandoahBarrierNode::Memory); - if (mem->is_Phi() && mem->in(0) == blk) { - mem = mem->in(i); - } - mem_phi->init_req(i, mem); - } - } - phase->register_new_node(mem_phi, blk); - - - Node* proj = find_out_with(Op_ShenandoahWBMemProj); - phase->igvn().replace_node(proj, mem_phi); - phase->igvn().replace_node(this, phi); - - return phi; -} - -void ShenandoahReadBarrierNode::try_move(PhaseIdealLoop* phase) { - Node *n_ctrl = phase->get_ctrl(this); - if (n_ctrl == NULL) { - return; - } - Node* mem = in(MemNode::Memory); - int alias = phase->C->get_alias_index(adr_type()); - const bool trace = false; - -#ifdef ASSERT - if (trace) { tty->print("Trying to move mem of"); dump(); } -#endif - - Node* new_mem = mem; - - ResourceMark rm; - VectorSet seen(Thread::current()->resource_area()); - Node_List phis; - - for (;;) { -#ifdef ASSERT - if (trace) { tty->print("Looking for dominator from"); mem->dump(); } -#endif - if (mem->is_Proj() && mem->in(0)->is_Start()) { - if (new_mem != in(MemNode::Memory)) { -#ifdef ASSERT - if (trace) { tty->print("XXX Setting mem to"); new_mem->dump(); tty->print(" for "); dump(); } -#endif - phase->igvn().replace_input_of(this, MemNode::Memory, new_mem); - } - return; - } - - Node* candidate = mem; - do { - if (!is_independent(mem)) { - if (trace) { tty->print_cr("Not independent"); } - if (new_mem != in(MemNode::Memory)) { -#ifdef ASSERT - if (trace) { tty->print("XXX Setting mem to"); new_mem->dump(); tty->print(" for "); dump(); } -#endif - phase->igvn().replace_input_of(this, MemNode::Memory, new_mem); - } - return; - } - if (seen.test_set(mem->_idx)) { - if (trace) { tty->print_cr("Already seen"); } - ShouldNotReachHere(); - // Strange graph - if (new_mem != in(MemNode::Memory)) { -#ifdef ASSERT - if (trace) { tty->print("XXX Setting mem to"); new_mem->dump(); tty->print(" for "); dump(); } -#endif - phase->igvn().replace_input_of(this, MemNode::Memory, new_mem); - } - return; - } - if (mem->is_Phi()) { - phis.push(mem); - } - mem = next_mem(mem, alias); - if (mem->bottom_type() == Type::MEMORY) { - candidate = mem; - } - assert(is_dominator(phase->ctrl_or_self(mem), n_ctrl, mem, this, phase) == phase->is_dominator(phase->ctrl_or_self(mem), n_ctrl), "strange dominator"); -#ifdef ASSERT - if (trace) { tty->print("Next mem is"); mem->dump(); } -#endif - } while (mem->bottom_type() != Type::MEMORY || !phase->is_dominator(phase->ctrl_or_self(mem), n_ctrl)); - - assert(mem->bottom_type() == Type::MEMORY, "bad mem"); - - bool not_dom = false; - for (uint i = 0; i < phis.size() && !not_dom; i++) { - Node* nn = phis.at(i); - -#ifdef ASSERT - if (trace) { tty->print("Looking from phi"); nn->dump(); } -#endif - assert(nn->is_Phi(), "phis only"); - for (uint j = 2; j < nn->req() && !not_dom; j++) { - Node* m = nn->in(j); -#ifdef ASSERT - if (trace) { tty->print("Input %d is", j); m->dump(); } -#endif - while (m != mem && !seen.test_set(m->_idx)) { - if (is_dominator(phase->ctrl_or_self(m), phase->ctrl_or_self(mem), m, mem, phase)) { - not_dom = true; - // Scheduling anomaly -#ifdef ASSERT - if (trace) { tty->print("Giving up"); m->dump(); } -#endif - break; - } - if (!is_independent(m)) { - if (trace) { tty->print_cr("Not independent"); } - if (new_mem != in(MemNode::Memory)) { -#ifdef ASSERT - if (trace) { tty->print("XXX Setting mem to"); new_mem->dump(); tty->print(" for "); dump(); } -#endif - phase->igvn().replace_input_of(this, MemNode::Memory, new_mem); - } - return; - } - if (m->is_Phi()) { - phis.push(m); - } - m = next_mem(m, alias); -#ifdef ASSERT - if (trace) { tty->print("Next mem is"); m->dump(); } -#endif - } - } - } - if (!not_dom) { - new_mem = mem; - phis.clear(); - } else { - seen.Clear(); - } - } -} - -CallStaticJavaNode* ShenandoahWriteBarrierNode::pin_and_expand_null_check(PhaseIterGVN& igvn) { - Node* val = in(ValueIn); - - const Type* val_t = igvn.type(val); - - if (val_t->meet(TypePtr::NULL_PTR) != val_t && - val->Opcode() == Op_CastPP && - val->in(0) != NULL && - val->in(0)->Opcode() == Op_IfTrue && - val->in(0)->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none) && - val->in(0)->in(0)->is_If() && - val->in(0)->in(0)->in(1)->Opcode() == Op_Bool && - val->in(0)->in(0)->in(1)->as_Bool()->_test._test == BoolTest::ne && - val->in(0)->in(0)->in(1)->in(1)->Opcode() == Op_CmpP && - val->in(0)->in(0)->in(1)->in(1)->in(1) == val->in(1) && - val->in(0)->in(0)->in(1)->in(1)->in(2)->bottom_type() == TypePtr::NULL_PTR) { - assert(val->in(0)->in(0)->in(1)->in(1)->in(1) == val->in(1), ""); - CallStaticJavaNode* unc = val->in(0)->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none); - return unc; - } - return NULL; -} - -void ShenandoahWriteBarrierNode::pin_and_expand_move_barrier(PhaseIdealLoop* phase, GrowableArray& memory_graph_fixers, Unique_Node_List& uses) { - Node* unc = pin_and_expand_null_check(phase->igvn()); - Node* val = in(ValueIn); - - if (unc != NULL) { - Node* ctrl = phase->get_ctrl(this); - Node* unc_ctrl = val->in(0); - - // Don't move write barrier in a loop - IdealLoopTree* loop = phase->get_loop(ctrl); - IdealLoopTree* unc_loop = phase->get_loop(unc_ctrl); - - if (!unc_loop->is_member(loop)) { - return; - } - - Node* branch = no_branches(ctrl, unc_ctrl, false, phase); - assert(branch == NULL || branch == NodeSentinel, "was not looking for a branch"); - if (branch == NodeSentinel) { - return; - } - - RegionNode* r = new RegionNode(3); - IfNode* iff = unc_ctrl->in(0)->as_If(); - - Node* ctrl_use = unc_ctrl->unique_ctrl_out(); - Node* unc_ctrl_clone = unc_ctrl->clone(); - phase->register_control(unc_ctrl_clone, loop, iff); - Node* c = unc_ctrl_clone; - Node* new_cast = clone_null_check(c, val, unc_ctrl_clone, phase); - r->init_req(1, new_cast->in(0)->in(0)->as_If()->proj_out(0)); - - phase->igvn().replace_input_of(unc_ctrl, 0, c->in(0)); - phase->set_idom(unc_ctrl, c->in(0), phase->dom_depth(unc_ctrl)); - phase->lazy_replace(c, unc_ctrl); - c = NULL;; - phase->igvn().replace_input_of(val, 0, unc_ctrl_clone); - phase->set_ctrl(val, unc_ctrl_clone); - - IfNode* new_iff = new_cast->in(0)->in(0)->as_If(); - fix_null_check(unc, unc_ctrl_clone, r, uses, phase); - Node* iff_proj = iff->proj_out(0); - r->init_req(2, iff_proj); - phase->register_control(r, phase->ltree_root(), iff); - - Node* new_bol = new_iff->in(1)->clone(); - Node* new_cmp = new_bol->in(1)->clone(); - assert(new_cmp->Opcode() == Op_CmpP, "broken"); - assert(new_cmp->in(1) == val->in(1), "broken"); - new_bol->set_req(1, new_cmp); - new_cmp->set_req(1, this); - phase->register_new_node(new_bol, new_iff->in(0)); - phase->register_new_node(new_cmp, new_iff->in(0)); - phase->igvn().replace_input_of(new_iff, 1, new_bol); - phase->igvn().replace_input_of(new_cast, 1, this); - - for (DUIterator_Fast imax, i = this->fast_outs(imax); i < imax; i++) { - Node* u = this->fast_out(i); - if (u == new_cast || u->Opcode() == Op_ShenandoahWBMemProj || u == new_cmp) { - continue; - } - phase->igvn().rehash_node_delayed(u); - int nb = u->replace_edge(this, new_cast); - assert(nb > 0, "no update?"); - --i; imax -= nb; - } - - for (DUIterator_Fast imax, i = val->fast_outs(imax); i < imax; i++) { - Node* u = val->fast_out(i); - if (u == this) { - continue; - } - phase->igvn().rehash_node_delayed(u); - int nb = u->replace_edge(val, new_cast); - assert(nb > 0, "no update?"); - --i; imax -= nb; - } - - Node* new_ctrl = unc_ctrl_clone; - - int alias = phase->C->get_alias_index(adr_type()); - MemoryGraphFixer* fixer = find_fixer(memory_graph_fixers, alias); - if (fixer == NULL) { - fixer = create_fixer(memory_graph_fixers, alias, phase, true); - } - - Node* proj = find_out_with(Op_ShenandoahWBMemProj); - fixer->remove(proj); - Node* mem = fixer->find_mem(new_ctrl, NULL); - - if (in(Memory) != mem) { - disconnect_barrier_mem(this, phase->igvn()); - phase->igvn().replace_input_of(this, Memory, mem); - } - - phase->set_ctrl_and_loop(this, new_ctrl); - phase->igvn().replace_input_of(this, Control, new_ctrl); - phase->set_ctrl_and_loop(proj, new_ctrl); - - fixer->fix_mem(new_ctrl, new_ctrl, mem, mem, proj, uses); - } -} - -void ShenandoahWriteBarrierNode::pin_and_expand_helper(PhaseIdealLoop* phase) { - Node* val = in(ValueIn); - CallStaticJavaNode* unc = pin_and_expand_null_check(phase->igvn()); - Node* rep = this; - Node* ctrl = phase->get_ctrl(this); - if (unc != NULL && val->in(0) == ctrl) { - Node* unc_ctrl = val->in(0); - IfNode* other_iff = unc_ctrl->unique_ctrl_out()->as_If(); - ProjNode* other_unc_ctrl = other_iff->proj_out(1); - Node* cast = NULL; - for (DUIterator_Fast imax, i = other_unc_ctrl->fast_outs(imax); i < imax && cast == NULL; i++) { - Node* u = other_unc_ctrl->fast_out(i); - if (u->Opcode() == Op_CastPP && u->in(1) == this) { - cast = u; - } - } - assert(other_unc_ctrl->is_uncommon_trap_if_pattern(Deoptimization::Reason_none) == unc, "broken"); - rep = cast; - } - - // Replace all uses of barrier's input that are dominated by ctrl - // with the value returned by the barrier: no need to keep both - // live. - for (DUIterator_Fast imax, i = val->fast_outs(imax); i < imax; i++) { - Node* u = val->fast_out(i); - if (u != this) { - if (u->is_Phi()) { - int nb = 0; - for (uint j = 1; j < u->req(); j++) { - if (u->in(j) == val) { - Node* c = u->in(0)->in(j); - if (phase->is_dominator(ctrl, c)) { - phase->igvn().replace_input_of(u, j, rep); - nb++; - } - } - } - if (nb > 0) { - imax -= nb; - --i; - } - } else { - Node* c = phase->ctrl_or_self(u); - if (is_dominator(ctrl, c, this, u, phase)) { - phase->igvn().rehash_node_delayed(u); - int nb = u->replace_edge(val, rep); - assert(nb > 0, "no update?"); - --i, imax -= nb; - } - } - } - } -} - -Node* ShenandoahWriteBarrierNode::find_bottom_mem(Node* ctrl, PhaseIdealLoop* phase) { +Node* ShenandoahBarrierC2Support::find_bottom_mem(Node* ctrl, PhaseIdealLoop* phase) { Node* mem = NULL; Node* c = ctrl; do { @@ -2355,7 +914,7 @@ return mem; } -void ShenandoahWriteBarrierNode::follow_barrier_uses(Node* n, Node* ctrl, Unique_Node_List& uses, PhaseIdealLoop* phase) { +void ShenandoahBarrierC2Support::follow_barrier_uses(Node* n, Node* ctrl, Unique_Node_List& uses, PhaseIdealLoop* phase) { for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) { Node* u = n->fast_out(i); if (!u->is_CFG() && phase->get_ctrl(u) == ctrl && (!u->is_Phi() || !u->in(0)->is_Loop() || u->in(LoopNode::LoopBackControl) != n)) { @@ -2375,7 +934,7 @@ inner->clear_strip_mined(); } -void ShenandoahWriteBarrierNode::test_heap_stable(Node*& ctrl, Node* raw_mem, Node*& heap_stable_ctrl, +void ShenandoahBarrierC2Support::test_heap_stable(Node*& ctrl, Node* raw_mem, Node*& heap_stable_ctrl, PhaseIdealLoop* phase) { IdealLoopTree* loop = phase->get_loop(ctrl); Node* thread = new ThreadLocalNode(); @@ -2407,7 +966,7 @@ assert(is_heap_stable_test(heap_stable_iff), "Should match the shape"); } -void ShenandoahWriteBarrierNode::test_null(Node*& ctrl, Node* val, Node*& null_ctrl, PhaseIdealLoop* phase) { +void ShenandoahBarrierC2Support::test_null(Node*& ctrl, Node* val, Node*& null_ctrl, PhaseIdealLoop* phase) { const Type* val_t = phase->igvn().type(val); if (val_t->meet(TypePtr::NULL_PTR) == val_t) { IdealLoopTree* loop = phase->get_loop(ctrl); @@ -2424,7 +983,7 @@ } } -Node* ShenandoahWriteBarrierNode::clone_null_check(Node*& c, Node* val, Node* unc_ctrl, PhaseIdealLoop* phase) { +Node* ShenandoahBarrierC2Support::clone_null_check(Node*& c, Node* val, Node* unc_ctrl, PhaseIdealLoop* phase) { IdealLoopTree *loop = phase->get_loop(c); Node* iff = unc_ctrl->in(0); assert(iff->is_If(), "broken"); @@ -2445,7 +1004,7 @@ return val; } -void ShenandoahWriteBarrierNode::fix_null_check(Node* unc, Node* unc_ctrl, Node* new_unc_ctrl, +void ShenandoahBarrierC2Support::fix_null_check(Node* unc, Node* unc_ctrl, Node* new_unc_ctrl, Unique_Node_List& uses, PhaseIdealLoop* phase) { IfNode* iff = unc_ctrl->in(0)->as_If(); Node* proj = iff->proj_out(0); @@ -2494,7 +1053,7 @@ assert(nb == 1, "only use expected"); } -void ShenandoahWriteBarrierNode::in_cset_fast_test(Node*& ctrl, Node*& not_cset_ctrl, Node* val, Node* raw_mem, PhaseIdealLoop* phase) { +void ShenandoahBarrierC2Support::in_cset_fast_test(Node*& ctrl, Node*& not_cset_ctrl, Node* val, Node* raw_mem, PhaseIdealLoop* phase) { IdealLoopTree *loop = phase->get_loop(ctrl); Node* raw_rbtrue = new CastP2XNode(ctrl, val); phase->register_new_node(raw_rbtrue, ctrl); @@ -2523,23 +1082,18 @@ phase->register_control(ctrl, loop, in_cset_fast_test_iff); } -void ShenandoahWriteBarrierNode::call_wb_stub(Node*& ctrl, Node*& val, Node*& result_mem, - Node* raw_mem, Node* wb_mem, - int alias, - PhaseIdealLoop* phase) { +void ShenandoahBarrierC2Support::call_lrb_stub(Node*& ctrl, Node*& val, Node*& result_mem, Node* raw_mem, PhaseIdealLoop* phase) { IdealLoopTree*loop = phase->get_loop(ctrl); const TypePtr* obj_type = phase->igvn().type(val)->is_oopptr()->cast_to_nonconst(); // The slow path stub consumes and produces raw memory in addition // to the existing memory edges Node* base = find_bottom_mem(ctrl, phase); - MergeMemNode* mm = MergeMemNode::make(base); - mm->set_memory_at(alias, wb_mem); mm->set_memory_at(Compile::AliasIdxRaw, raw_mem); phase->register_new_node(mm, ctrl); - Node* call = new CallLeafNode(ShenandoahBarrierSetC2::shenandoah_write_barrier_Type(), CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_JRT), "shenandoah_write_barrier", TypeRawPtr::BOTTOM); + Node* call = new CallLeafNode(ShenandoahBarrierSetC2::shenandoah_write_barrier_Type(), CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_JRT), "shenandoah_write_barrier", TypeRawPtr::BOTTOM); call->init_req(TypeFunc::Control, ctrl); call->init_req(TypeFunc::I_O, phase->C->top()); call->init_req(TypeFunc::Memory, mm); @@ -2557,7 +1111,7 @@ phase->register_new_node(val, ctrl); } -void ShenandoahWriteBarrierNode::fix_ctrl(Node* barrier, Node* region, const MemoryGraphFixer& fixer, Unique_Node_List& uses, Unique_Node_List& uses_to_ignore, uint last, PhaseIdealLoop* phase) { +void ShenandoahBarrierC2Support::fix_ctrl(Node* barrier, Node* region, const MemoryGraphFixer& fixer, Unique_Node_List& uses, Unique_Node_List& uses_to_ignore, uint last, PhaseIdealLoop* phase) { Node* ctrl = phase->get_ctrl(barrier); Node* init_raw_mem = fixer.find_mem(ctrl, barrier); @@ -2610,26 +1164,32 @@ } } -void ShenandoahWriteBarrierNode::pin_and_expand(PhaseIdealLoop* phase) { - Node_List enqueue_barriers; - if (ShenandoahStoreValEnqueueBarrier) { - Unique_Node_List wq; - wq.push(phase->C->root()); - for (uint i = 0; i < wq.size(); i++) { - Node* n = wq.at(i); - if (n->Opcode() == Op_ShenandoahEnqueueBarrier) { - enqueue_barriers.push(n); - } - for (uint i = 0; i < n->req(); i++) { - Node* in = n->in(i); - if (in != NULL) { - wq.push(in); - } - } +static Node* create_phis_on_call_return(Node* ctrl, Node* c, Node* n, Node* n_clone, const CallProjections& projs, PhaseIdealLoop* phase) { + Node* region = NULL; + while (c != ctrl) { + if (c->is_Region()) { + region = c; + } + c = phase->idom(c); + } + assert(region != NULL, ""); + Node* phi = new PhiNode(region, n->bottom_type()); + for (uint j = 1; j < region->req(); j++) { + Node* in = region->in(j); + if (phase->is_dominator(projs.fallthrough_catchproj, in)) { + phi->init_req(j, n); + } else if (phase->is_dominator(projs.catchall_catchproj, in)) { + phi->init_req(j, n_clone); + } else { + phi->init_req(j, create_phis_on_call_return(ctrl, in, n, n_clone, projs, phase)); } } + phase->register_new_node(phi, region); + return phi; +} - const bool trace = false; +void ShenandoahBarrierC2Support::pin_and_expand(PhaseIdealLoop* phase) { + ShenandoahBarrierSetC2State* state = ShenandoahBarrierSetC2::bsc2()->state(); // Collect raw memory state at CFG points in the entire graph and // record it in memory_nodes. Optimize the raw memory graph in the @@ -2637,34 +1197,9 @@ // simpler. GrowableArray memory_graph_fixers; - // Let's try to common write barriers again - optimize_before_expansion(phase, memory_graph_fixers, true); - Unique_Node_List uses; - for (int i = 0; i < ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_barriers_count(); i++) { - ShenandoahWriteBarrierNode* wb = ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_barrier(i); - Node* ctrl = phase->get_ctrl(wb); - - Node* val = wb->in(ValueIn); - if (ctrl->is_Proj() && ctrl->in(0)->is_CallJava()) { - assert(is_dominator(phase->get_ctrl(val), ctrl->in(0)->in(0), val, ctrl->in(0), phase), "can't move"); - phase->set_ctrl(wb, ctrl->in(0)->in(0)); - } else if (ctrl->is_CallRuntime()) { - assert(is_dominator(phase->get_ctrl(val), ctrl->in(0), val, ctrl, phase), "can't move"); - phase->set_ctrl(wb, ctrl->in(0)); - } - - assert(wb->Opcode() == Op_ShenandoahWriteBarrier, "only for write barriers"); - // Look for a null check that dominates this barrier and move the - // barrier right after the null check to enable implicit null - // checks - wb->pin_and_expand_move_barrier(phase, memory_graph_fixers, uses); - - wb->pin_and_expand_helper(phase); - } - - for (uint i = 0; i < enqueue_barriers.size(); i++) { - Node* barrier = enqueue_barriers.at(i); + for (int i = 0; i < state->enqueue_barriers_count(); i++) { + Node* barrier = state->enqueue_barrier(i); Node* ctrl = phase->get_ctrl(barrier); IdealLoopTree* loop = phase->get_loop(ctrl); if (loop->_head->is_OuterStripMinedLoop()) { @@ -2676,24 +1211,386 @@ } } - for (int i = ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_barriers_count(); i > 0; i--) { - int cnt = ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_barriers_count(); - ShenandoahWriteBarrierNode* wb = ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_barrier(i-1); - Node* ctrl = phase->get_ctrl(wb); - IdealLoopTree* loop = phase->get_loop(ctrl); - if (loop->_head->is_OuterStripMinedLoop()) { - // Expanding a barrier here will break loop strip mining - // verification. Transform the loop so the loop nest doesn't - // appear as strip mined. - OuterStripMinedLoopNode* outer = loop->_head->as_OuterStripMinedLoop(); - hide_strip_mined_loop(outer, outer->unique_ctrl_out()->as_CountedLoop(), phase); + Node_Stack stack(0); + Node_List clones; + for (int i = state->load_reference_barriers_count() - 1; i >= 0; i--) { + ShenandoahLoadReferenceBarrierNode* lrb = state->load_reference_barrier(i); + if (lrb->get_barrier_strength() == ShenandoahLoadReferenceBarrierNode::NONE) { + continue; + } + + Node* ctrl = phase->get_ctrl(lrb); + Node* val = lrb->in(ShenandoahLoadReferenceBarrierNode::ValueIn); + + CallStaticJavaNode* unc = NULL; + Node* unc_ctrl = NULL; + Node* uncasted_val = val; + + for (DUIterator_Fast imax, i = lrb->fast_outs(imax); i < imax; i++) { + Node* u = lrb->fast_out(i); + if (u->Opcode() == Op_CastPP && + u->in(0) != NULL && + phase->is_dominator(u->in(0), ctrl)) { + const Type* u_t = phase->igvn().type(u); + + if (u_t->meet(TypePtr::NULL_PTR) != u_t && + u->in(0)->Opcode() == Op_IfTrue && + u->in(0)->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none) && + u->in(0)->in(0)->is_If() && + u->in(0)->in(0)->in(1)->Opcode() == Op_Bool && + u->in(0)->in(0)->in(1)->as_Bool()->_test._test == BoolTest::ne && + u->in(0)->in(0)->in(1)->in(1)->Opcode() == Op_CmpP && + u->in(0)->in(0)->in(1)->in(1)->in(1) == val && + u->in(0)->in(0)->in(1)->in(1)->in(2)->bottom_type() == TypePtr::NULL_PTR) { + IdealLoopTree* loop = phase->get_loop(ctrl); + IdealLoopTree* unc_loop = phase->get_loop(u->in(0)); + + if (!unc_loop->is_member(loop)) { + continue; + } + + Node* branch = no_branches(ctrl, u->in(0), false, phase); + assert(branch == NULL || branch == NodeSentinel, "was not looking for a branch"); + if (branch == NodeSentinel) { + continue; + } + + phase->igvn().replace_input_of(u, 1, val); + phase->igvn().replace_input_of(lrb, ShenandoahLoadReferenceBarrierNode::ValueIn, u); + phase->set_ctrl(u, u->in(0)); + phase->set_ctrl(lrb, u->in(0)); + unc = u->in(0)->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none); + unc_ctrl = u->in(0); + val = u; + + for (DUIterator_Fast jmax, j = val->fast_outs(jmax); j < jmax; j++) { + Node* u = val->fast_out(j); + if (u == lrb) continue; + phase->igvn().rehash_node_delayed(u); + int nb = u->replace_edge(val, lrb); + --j; jmax -= nb; + } + + RegionNode* r = new RegionNode(3); + IfNode* iff = unc_ctrl->in(0)->as_If(); + + Node* ctrl_use = unc_ctrl->unique_ctrl_out(); + Node* unc_ctrl_clone = unc_ctrl->clone(); + phase->register_control(unc_ctrl_clone, loop, iff); + Node* c = unc_ctrl_clone; + Node* new_cast = clone_null_check(c, val, unc_ctrl_clone, phase); + r->init_req(1, new_cast->in(0)->in(0)->as_If()->proj_out(0)); + + phase->igvn().replace_input_of(unc_ctrl, 0, c->in(0)); + phase->set_idom(unc_ctrl, c->in(0), phase->dom_depth(unc_ctrl)); + phase->lazy_replace(c, unc_ctrl); + c = NULL;; + phase->igvn().replace_input_of(val, 0, unc_ctrl_clone); + phase->set_ctrl(val, unc_ctrl_clone); + + IfNode* new_iff = new_cast->in(0)->in(0)->as_If(); + fix_null_check(unc, unc_ctrl_clone, r, uses, phase); + Node* iff_proj = iff->proj_out(0); + r->init_req(2, iff_proj); + phase->register_control(r, phase->ltree_root(), iff); + + Node* new_bol = new_iff->in(1)->clone(); + Node* new_cmp = new_bol->in(1)->clone(); + assert(new_cmp->Opcode() == Op_CmpP, "broken"); + assert(new_cmp->in(1) == val->in(1), "broken"); + new_bol->set_req(1, new_cmp); + new_cmp->set_req(1, lrb); + phase->register_new_node(new_bol, new_iff->in(0)); + phase->register_new_node(new_cmp, new_iff->in(0)); + phase->igvn().replace_input_of(new_iff, 1, new_bol); + phase->igvn().replace_input_of(new_cast, 1, lrb); + + for (DUIterator_Fast imax, i = lrb->fast_outs(imax); i < imax; i++) { + Node* u = lrb->fast_out(i); + if (u == new_cast || u == new_cmp) { + continue; + } + phase->igvn().rehash_node_delayed(u); + int nb = u->replace_edge(lrb, new_cast); + assert(nb > 0, "no update?"); + --i; imax -= nb; + } + + for (DUIterator_Fast imax, i = val->fast_outs(imax); i < imax; i++) { + Node* u = val->fast_out(i); + if (u == lrb) { + continue; + } + phase->igvn().rehash_node_delayed(u); + int nb = u->replace_edge(val, new_cast); + assert(nb > 0, "no update?"); + --i; imax -= nb; + } + + ctrl = unc_ctrl_clone; + phase->set_ctrl_and_loop(lrb, ctrl); + break; + } + } + } + if (ctrl->is_Proj() && ctrl->in(0)->is_CallJava()) { + CallNode* call = ctrl->in(0)->as_CallJava(); + CallProjections projs; + call->extract_projections(&projs, false, false); + + Node* lrb_clone = lrb->clone(); + phase->register_new_node(lrb_clone, projs.catchall_catchproj); + phase->set_ctrl(lrb, projs.fallthrough_catchproj); + + stack.push(lrb, 0); + clones.push(lrb_clone); + + do { + assert(stack.size() == clones.size(), ""); + Node* n = stack.node(); +#ifdef ASSERT + if (n->is_Load()) { + Node* mem = n->in(MemNode::Memory); + for (DUIterator_Fast jmax, j = mem->fast_outs(jmax); j < jmax; j++) { + Node* u = mem->fast_out(j); + assert(!u->is_Store() || !u->is_LoadStore() || phase->get_ctrl(u) != ctrl, "anti dependent store?"); + } + } +#endif + uint idx = stack.index(); + Node* n_clone = clones.at(clones.size()-1); + if (idx < n->outcnt()) { + Node* u = n->raw_out(idx); + Node* c = phase->ctrl_or_self(u); + if (c == ctrl) { + stack.set_index(idx+1); + assert(!u->is_CFG(), ""); + stack.push(u, 0); + Node* u_clone = u->clone(); + int nb = u_clone->replace_edge(n, n_clone); + assert(nb > 0, "should have replaced some uses"); + phase->register_new_node(u_clone, projs.catchall_catchproj); + clones.push(u_clone); + phase->set_ctrl(u, projs.fallthrough_catchproj); + } else { + bool replaced = false; + if (u->is_Phi()) { + for (uint k = 1; k < u->req(); k++) { + if (u->in(k) == n) { + if (phase->is_dominator(projs.catchall_catchproj, u->in(0)->in(k))) { + phase->igvn().replace_input_of(u, k, n_clone); + replaced = true; + } else if (!phase->is_dominator(projs.fallthrough_catchproj, u->in(0)->in(k))) { + phase->igvn().replace_input_of(u, k, create_phis_on_call_return(ctrl, u->in(0)->in(k), n, n_clone, projs, phase)); + replaced = true; + } + } + } + } else { + if (phase->is_dominator(projs.catchall_catchproj, c)) { + phase->igvn().rehash_node_delayed(u); + int nb = u->replace_edge(n, n_clone); + assert(nb > 0, "should have replaced some uses"); + replaced = true; + } else if (!phase->is_dominator(projs.fallthrough_catchproj, c)) { + phase->igvn().rehash_node_delayed(u); + int nb = u->replace_edge(n, create_phis_on_call_return(ctrl, c, n, n_clone, projs, phase)); + assert(nb > 0, "should have replaced some uses"); + replaced = true; + } + } + if (!replaced) { + stack.set_index(idx+1); + } + } + } else { + // assert(n_clone->outcnt() > 0, ""); + // assert(n->outcnt() > 0, ""); + stack.pop(); + clones.pop(); + } + } while (stack.size() > 0); + assert(stack.size() == 0 && clones.size() == 0, ""); + ctrl = projs.fallthrough_catchproj; } } + // Expand load-reference-barriers MemoryGraphFixer fixer(Compile::AliasIdxRaw, true, phase); Unique_Node_List uses_to_ignore; - for (uint i = 0; i < enqueue_barriers.size(); i++) { - Node* barrier = enqueue_barriers.at(i); + for (int i = state->load_reference_barriers_count() - 1; i >= 0; i--) { + ShenandoahLoadReferenceBarrierNode* lrb = state->load_reference_barrier(i); + if (lrb->get_barrier_strength() == ShenandoahLoadReferenceBarrierNode::NONE) { + phase->igvn().replace_node(lrb, lrb->in(ShenandoahLoadReferenceBarrierNode::ValueIn)); + continue; + } + uint last = phase->C->unique(); + Node* ctrl = phase->get_ctrl(lrb); + Node* val = lrb->in(ShenandoahLoadReferenceBarrierNode::ValueIn); + + + Node* orig_ctrl = ctrl; + + Node* raw_mem = fixer.find_mem(ctrl, lrb); + Node* init_raw_mem = raw_mem; + Node* raw_mem_for_ctrl = fixer.find_mem(ctrl, NULL); + // int alias = phase->C->get_alias_index(lrb->adr_type()); + + IdealLoopTree *loop = phase->get_loop(ctrl); + CallStaticJavaNode* unc = lrb->pin_and_expand_null_check(phase->igvn()); + Node* unc_ctrl = NULL; + if (unc != NULL) { + if (val->in(ShenandoahLoadReferenceBarrierNode::Control) != ctrl) { + unc = NULL; + } else { + unc_ctrl = val->in(ShenandoahLoadReferenceBarrierNode::Control); + } + } + + Node* uncasted_val = val; + if (unc != NULL) { + uncasted_val = val->in(1); + } + + Node* heap_stable_ctrl = NULL; + Node* null_ctrl = NULL; + + assert(val->bottom_type()->make_oopptr(), "need oop"); + assert(val->bottom_type()->make_oopptr()->const_oop() == NULL, "expect non-constant"); + + enum { _heap_stable = 1, _not_cset, _not_equal, _evac_path, _null_path, PATH_LIMIT }; + Node* region = new RegionNode(PATH_LIMIT); + Node* val_phi = new PhiNode(region, uncasted_val->bottom_type()->is_oopptr()); + Node* raw_mem_phi = PhiNode::make(region, raw_mem, Type::MEMORY, TypeRawPtr::BOTTOM); + + // Stable path. + test_heap_stable(ctrl, raw_mem, heap_stable_ctrl, phase); + IfNode* heap_stable_iff = heap_stable_ctrl->in(0)->as_If(); + + // Heap stable case + region->init_req(_heap_stable, heap_stable_ctrl); + val_phi->init_req(_heap_stable, uncasted_val); + raw_mem_phi->init_req(_heap_stable, raw_mem); + + Node* reg2_ctrl = NULL; + // Null case + test_null(ctrl, val, null_ctrl, phase); + if (null_ctrl != NULL) { + reg2_ctrl = null_ctrl->in(0); + region->init_req(_null_path, null_ctrl); + val_phi->init_req(_null_path, uncasted_val); + raw_mem_phi->init_req(_null_path, raw_mem); + } else { + region->del_req(_null_path); + val_phi->del_req(_null_path); + raw_mem_phi->del_req(_null_path); + } + + // Test for in-cset. + // Wires !in_cset(obj) to slot 2 of region and phis + Node* not_cset_ctrl = NULL; + in_cset_fast_test(ctrl, not_cset_ctrl, uncasted_val, raw_mem, phase); + if (not_cset_ctrl != NULL) { + if (reg2_ctrl == NULL) reg2_ctrl = not_cset_ctrl->in(0); + region->init_req(_not_cset, not_cset_ctrl); + val_phi->init_req(_not_cset, uncasted_val); + raw_mem_phi->init_req(_not_cset, raw_mem); + } + + // Resolve object when orig-value is in cset. + // Make the unconditional resolve for fwdptr. + Node* new_val = uncasted_val; + if (unc_ctrl != NULL) { + // Clone the null check in this branch to allow implicit null check + new_val = clone_null_check(ctrl, val, unc_ctrl, phase); + fix_null_check(unc, unc_ctrl, ctrl->in(0)->as_If()->proj_out(0), uses, phase); + + IfNode* iff = unc_ctrl->in(0)->as_If(); + phase->igvn().replace_input_of(iff, 1, phase->igvn().intcon(1)); + } + Node* addr = new AddPNode(new_val, uncasted_val, phase->igvn().MakeConX(ShenandoahBrooksPointer::byte_offset())); + phase->register_new_node(addr, ctrl); + assert(val->bottom_type()->isa_oopptr(), "what else?"); + const TypePtr* obj_type = val->bottom_type()->is_oopptr(); + const TypePtr* adr_type = TypeRawPtr::BOTTOM; + Node* fwd = new LoadPNode(ctrl, raw_mem, addr, adr_type, obj_type, MemNode::unordered); + phase->register_new_node(fwd, ctrl); + + // Only branch to LRB stub if object is not forwarded; otherwise reply with fwd ptr + Node* cmp = new CmpPNode(fwd, new_val); + phase->register_new_node(cmp, ctrl); + Node* bol = new BoolNode(cmp, BoolTest::eq); + phase->register_new_node(bol, ctrl); + + IfNode* iff = new IfNode(ctrl, bol, PROB_UNLIKELY(0.999), COUNT_UNKNOWN); + if (reg2_ctrl == NULL) reg2_ctrl = iff; + phase->register_control(iff, loop, ctrl); + Node* if_not_eq = new IfFalseNode(iff); + phase->register_control(if_not_eq, loop, iff); + Node* if_eq = new IfTrueNode(iff); + phase->register_control(if_eq, loop, iff); + + // Wire up not-equal-path in slots 3. + region->init_req(_not_equal, if_not_eq); + val_phi->init_req(_not_equal, fwd); + raw_mem_phi->init_req(_not_equal, raw_mem); + + // Call wb-stub and wire up that path in slots 4 + Node* result_mem = NULL; + ctrl = if_eq; + call_lrb_stub(ctrl, fwd, result_mem, raw_mem, phase); + region->init_req(_evac_path, ctrl); + val_phi->init_req(_evac_path, fwd); + raw_mem_phi->init_req(_evac_path, result_mem); + + phase->register_control(region, loop, heap_stable_iff); + Node* out_val = val_phi; + phase->register_new_node(val_phi, region); + phase->register_new_node(raw_mem_phi, region); + + fix_ctrl(lrb, region, fixer, uses, uses_to_ignore, last, phase); + + ctrl = orig_ctrl; + + if (unc != NULL) { + for (DUIterator_Fast imax, i = val->fast_outs(imax); i < imax; i++) { + Node* u = val->fast_out(i); + Node* c = phase->ctrl_or_self(u); + if (u != lrb && (c != ctrl || is_dominator_same_ctrl(c, lrb, u, phase))) { + phase->igvn().rehash_node_delayed(u); + int nb = u->replace_edge(val, out_val); + --i, imax -= nb; + } + } + if (val->outcnt() == 0) { + phase->igvn()._worklist.push(val); + } + } + phase->igvn().replace_node(lrb, out_val); + + follow_barrier_uses(out_val, ctrl, uses, phase); + + for(uint next = 0; next < uses.size(); next++ ) { + Node *n = uses.at(next); + assert(phase->get_ctrl(n) == ctrl, "bad control"); + assert(n != init_raw_mem, "should leave input raw mem above the barrier"); + phase->set_ctrl(n, region); + follow_barrier_uses(n, ctrl, uses, phase); + } + + // The slow path call produces memory: hook the raw memory phi + // from the expanded load reference barrier with the rest of the graph + // which may require adding memory phis at every post dominated + // region and at enclosing loop heads. Use the memory state + // collected in memory_nodes to fix the memory graph. Update that + // memory state as we go. + fixer.fix_mem(ctrl, region, init_raw_mem, raw_mem_for_ctrl, raw_mem_phi, uses); + } + // Done expanding load-reference-barriers. + assert(ShenandoahBarrierSetC2::bsc2()->state()->load_reference_barriers_count() == 0, "all load reference barrier nodes should have been replaced"); + + for (int i = state->enqueue_barriers_count() - 1; i >= 0; i--) { + Node* barrier = state->enqueue_barrier(i); Node* pre_val = barrier->in(1); if (phase->igvn().type(pre_val)->higher_equal(TypePtr::NULL_PTR)) { @@ -2840,212 +1737,11 @@ phase->igvn().replace_node(barrier, pre_val); } + assert(state->enqueue_barriers_count() == 0, "all enqueue barrier nodes should have been replaced"); - for (int i = ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_barriers_count(); i > 0; i--) { - int cnt = ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_barriers_count(); - ShenandoahWriteBarrierNode* wb = ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_barrier(i-1); - - uint last = phase->C->unique(); - Node* ctrl = phase->get_ctrl(wb); - Node* orig_ctrl = ctrl; - - Node* raw_mem = fixer.find_mem(ctrl, wb); - Node* init_raw_mem = raw_mem; - Node* raw_mem_for_ctrl = fixer.find_mem(ctrl, NULL); - int alias = phase->C->get_alias_index(wb->adr_type()); - Node* wb_mem = wb->in(Memory); - Node* init_wb_mem = wb_mem; - - Node* val = wb->in(ValueIn); - Node* wbproj = wb->find_out_with(Op_ShenandoahWBMemProj); - IdealLoopTree *loop = phase->get_loop(ctrl); - - assert(val->Opcode() != Op_ShenandoahWriteBarrier, "No chain of write barriers"); - - CallStaticJavaNode* unc = wb->pin_and_expand_null_check(phase->igvn()); - Node* unc_ctrl = NULL; - if (unc != NULL) { - if (val->in(0) != ctrl) { - unc = NULL; - } else { - unc_ctrl = val->in(0); - } - } - - Node* uncasted_val = val; - if (unc != NULL) { - uncasted_val = val->in(1); - } - - Node* heap_stable_ctrl = NULL; - Node* null_ctrl = NULL; - - assert(val->bottom_type()->make_oopptr(), "need oop"); - assert(val->bottom_type()->make_oopptr()->const_oop() == NULL, "expect non-constant"); - - enum { _heap_stable = 1, _heap_unstable, PATH_LIMIT }; - Node* region = new RegionNode(PATH_LIMIT); - Node* val_phi = new PhiNode(region, uncasted_val->bottom_type()->is_oopptr()); - Node* mem_phi = PhiNode::make(region, wb_mem, Type::MEMORY, phase->C->alias_type(wb->adr_type())->adr_type()); - Node* raw_mem_phi = PhiNode::make(region, raw_mem, Type::MEMORY, TypeRawPtr::BOTTOM); - - enum { _not_cset = 1, _not_equal, _evac_path, _null_path, PATH_LIMIT2 }; - Node* region2 = new RegionNode(PATH_LIMIT2); - Node* val_phi2 = new PhiNode(region2, uncasted_val->bottom_type()->is_oopptr()); - Node* mem_phi2 = PhiNode::make(region2, wb_mem, Type::MEMORY, phase->C->alias_type(wb->adr_type())->adr_type()); - Node* raw_mem_phi2 = PhiNode::make(region2, raw_mem, Type::MEMORY, TypeRawPtr::BOTTOM); - - // Stable path. - test_heap_stable(ctrl, raw_mem, heap_stable_ctrl, phase); - IfNode* heap_stable_iff = heap_stable_ctrl->in(0)->as_If(); - - // Heap stable case - region->init_req(_heap_stable, heap_stable_ctrl); - val_phi->init_req(_heap_stable, uncasted_val); - mem_phi->init_req(_heap_stable, wb_mem); - raw_mem_phi->init_req(_heap_stable, raw_mem); - - Node* reg2_ctrl = NULL; - // Null case - test_null(ctrl, val, null_ctrl, phase); - if (null_ctrl != NULL) { - reg2_ctrl = null_ctrl->in(0); - region2->init_req(_null_path, null_ctrl); - val_phi2->init_req(_null_path, uncasted_val); - mem_phi2->init_req(_null_path, wb_mem); - raw_mem_phi2->init_req(_null_path, raw_mem); - } else { - region2->del_req(_null_path); - val_phi2->del_req(_null_path); - mem_phi2->del_req(_null_path); - raw_mem_phi2->del_req(_null_path); - } - - // Test for in-cset. - // Wires !in_cset(obj) to slot 2 of region and phis - Node* not_cset_ctrl = NULL; - in_cset_fast_test(ctrl, not_cset_ctrl, uncasted_val, raw_mem, phase); - if (not_cset_ctrl != NULL) { - if (reg2_ctrl == NULL) reg2_ctrl = not_cset_ctrl->in(0); - region2->init_req(_not_cset, not_cset_ctrl); - val_phi2->init_req(_not_cset, uncasted_val); - mem_phi2->init_req(_not_cset, wb_mem); - raw_mem_phi2->init_req(_not_cset, raw_mem); - } - - // Resolve object when orig-value is in cset. - // Make the unconditional resolve for fwdptr, not the read barrier. - Node* new_val = uncasted_val; - if (unc_ctrl != NULL) { - // Clone the null check in this branch to allow implicit null check - new_val = clone_null_check(ctrl, val, unc_ctrl, phase); - fix_null_check(unc, unc_ctrl, ctrl->in(0)->as_If()->proj_out(0), uses, phase); - - IfNode* iff = unc_ctrl->in(0)->as_If(); - phase->igvn().replace_input_of(iff, 1, phase->igvn().intcon(1)); - } - Node* addr = new AddPNode(new_val, uncasted_val, phase->igvn().MakeConX(ShenandoahBrooksPointer::byte_offset())); - phase->register_new_node(addr, ctrl); - assert(val->bottom_type()->isa_oopptr(), "what else?"); - const TypePtr* obj_type = val->bottom_type()->is_oopptr(); - const TypePtr* adr_type = ShenandoahBarrierNode::brooks_pointer_type(obj_type); - Node* fwd = new LoadPNode(ctrl, wb_mem, addr, adr_type, obj_type, MemNode::unordered); - phase->register_new_node(fwd, ctrl); - - // Only branch to WB stub if object is not forwarded; otherwise reply with fwd ptr - Node* cmp = new CmpPNode(fwd, new_val); - phase->register_new_node(cmp, ctrl); - Node* bol = new BoolNode(cmp, BoolTest::eq); - phase->register_new_node(bol, ctrl); - - IfNode* iff = new IfNode(ctrl, bol, PROB_UNLIKELY(0.999), COUNT_UNKNOWN); - if (reg2_ctrl == NULL) reg2_ctrl = iff; - phase->register_control(iff, loop, ctrl); - Node* if_not_eq = new IfFalseNode(iff); - phase->register_control(if_not_eq, loop, iff); - Node* if_eq = new IfTrueNode(iff); - phase->register_control(if_eq, loop, iff); - - // Wire up not-equal-path in slots 3. - region2->init_req(_not_equal, if_not_eq); - val_phi2->init_req(_not_equal, fwd); - mem_phi2->init_req(_not_equal, wb_mem); - raw_mem_phi2->init_req(_not_equal, raw_mem); - - // Call wb-stub and wire up that path in slots 4 - Node* result_mem = NULL; - ctrl = if_eq; - call_wb_stub(ctrl, new_val, result_mem, - raw_mem, wb_mem, - alias, phase); - region2->init_req(_evac_path, ctrl); - val_phi2->init_req(_evac_path, new_val); - mem_phi2->init_req(_evac_path, result_mem); - raw_mem_phi2->init_req(_evac_path, result_mem); - - phase->register_control(region2, loop, reg2_ctrl); - phase->register_new_node(val_phi2, region2); - phase->register_new_node(mem_phi2, region2); - phase->register_new_node(raw_mem_phi2, region2); - - region->init_req(_heap_unstable, region2); - val_phi->init_req(_heap_unstable, val_phi2); - mem_phi->init_req(_heap_unstable, mem_phi2); - raw_mem_phi->init_req(_heap_unstable, raw_mem_phi2); - - phase->register_control(region, loop, heap_stable_iff); - Node* out_val = val_phi; - phase->register_new_node(val_phi, region); - phase->register_new_node(mem_phi, region); - phase->register_new_node(raw_mem_phi, region); - - fix_ctrl(wb, region, fixer, uses, uses_to_ignore, last, phase); - - ctrl = orig_ctrl; - - phase->igvn().replace_input_of(wbproj, ShenandoahWBMemProjNode::WriteBarrier, phase->C->top()); - phase->igvn().replace_node(wbproj, mem_phi); - if (unc != NULL) { - for (DUIterator_Fast imax, i = val->fast_outs(imax); i < imax; i++) { - Node* u = val->fast_out(i); - Node* c = phase->ctrl_or_self(u); - if (u != wb && (c != ctrl || is_dominator_same_ctrl(c, wb, u, phase))) { - phase->igvn().rehash_node_delayed(u); - int nb = u->replace_edge(val, out_val); - --i, imax -= nb; - } - } - if (val->outcnt() == 0) { - phase->igvn()._worklist.push(val); - } - } - phase->igvn().replace_node(wb, out_val); - - follow_barrier_uses(mem_phi, ctrl, uses, phase); - follow_barrier_uses(out_val, ctrl, uses, phase); - - for(uint next = 0; next < uses.size(); next++ ) { - Node *n = uses.at(next); - assert(phase->get_ctrl(n) == ctrl, "bad control"); - assert(n != init_raw_mem, "should leave input raw mem above the barrier"); - phase->set_ctrl(n, region); - follow_barrier_uses(n, ctrl, uses, phase); - } - - // The slow path call produces memory: hook the raw memory phi - // from the expanded write barrier with the rest of the graph - // which may require adding memory phis at every post dominated - // region and at enclosing loop heads. Use the memory state - // collected in memory_nodes to fix the memory graph. Update that - // memory state as we go. - fixer.fix_mem(ctrl, region, init_raw_mem, raw_mem_for_ctrl, raw_mem_phi, uses); - assert(ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_barriers_count() == cnt - 1, "not replaced"); - } - - assert(ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_barriers_count() == 0, "all write barrier nodes should have been replaced"); } -void ShenandoahWriteBarrierNode::move_heap_stable_test_out_of_loop(IfNode* iff, PhaseIdealLoop* phase) { +void ShenandoahBarrierC2Support::move_heap_stable_test_out_of_loop(IfNode* iff, PhaseIdealLoop* phase) { IdealLoopTree *loop = phase->get_loop(iff); Node* loop_head = loop->_head; Node* entry_c = loop_head->in(LoopNode::EntryControl); @@ -3078,7 +1774,7 @@ } } -bool ShenandoahWriteBarrierNode::identical_backtoback_ifs(Node *n, PhaseIdealLoop* phase) { +bool ShenandoahBarrierC2Support::identical_backtoback_ifs(Node* n, PhaseIdealLoop* phase) { if (!n->is_If() || n->is_CountedLoopEnd()) { return false; } @@ -3113,7 +1809,7 @@ return true; } -void ShenandoahWriteBarrierNode::merge_back_to_back_tests(Node* n, PhaseIdealLoop* phase) { +void ShenandoahBarrierC2Support::merge_back_to_back_tests(Node* n, PhaseIdealLoop* phase) { assert(is_heap_stable_test(n), "no other tests"); if (identical_backtoback_ifs(n, phase)) { Node* n_ctrl = n->in(0); @@ -3149,7 +1845,7 @@ } } -IfNode* ShenandoahWriteBarrierNode::find_unswitching_candidate(const IdealLoopTree *loop, PhaseIdealLoop* phase) { +IfNode* ShenandoahBarrierC2Support::find_unswitching_candidate(const IdealLoopTree* loop, PhaseIdealLoop* phase) { // Find first invariant test that doesn't exit the loop LoopNode *head = loop->_head->as_Loop(); IfNode* unswitch_iff = NULL; @@ -3194,10 +1890,9 @@ } -void ShenandoahWriteBarrierNode::optimize_after_expansion(VectorSet &visited, Node_Stack &stack, Node_List &old_new, PhaseIdealLoop* phase) { +void ShenandoahBarrierC2Support::optimize_after_expansion(VectorSet &visited, Node_Stack &stack, Node_List &old_new, PhaseIdealLoop* phase) { Node_List heap_stable_tests; Node_List gc_state_loads; - stack.push(phase->C->start(), 0); do { Node* n = stack.node(); @@ -3274,7 +1969,7 @@ } #ifdef ASSERT -void ShenandoahBarrierNode::verify_raw_mem(RootNode* root) { +void ShenandoahBarrierC2Support::verify_raw_mem(RootNode* root) { const bool trace = false; ResourceMark rm; Unique_Node_List nodes; @@ -3372,6 +2067,10 @@ } #endif +ShenandoahEnqueueBarrierNode::ShenandoahEnqueueBarrierNode(Node* val) : Node(NULL, val) { + ShenandoahBarrierSetC2::bsc2()->state()->add_enqueue_barrier(this); +} + const Type* ShenandoahEnqueueBarrierNode::bottom_type() const { if (in(1) == NULL || in(1)->is_top()) { return Type::TOP; @@ -3531,6 +2230,26 @@ Node* call = in->in(0)->in(0); assert(call->is_Call(), ""); mem = call->in(TypeFunc::Memory); + } else if (in->Opcode() == Op_NeverBranch) { + ResourceMark rm; + Unique_Node_List wq; + wq.push(in); + wq.push(in->as_Multi()->proj_out(0)); + for (uint j = 1; j < wq.size(); j++) { + Node* c = wq.at(j); + assert(!c->is_Root(), "shouldn't leave loop"); + if (c->is_SafePoint()) { + assert(mem == NULL, "only one safepoint"); + mem = c->in(TypeFunc::Memory); + } + for (DUIterator_Fast kmax, k = c->fast_outs(kmax); k < kmax; k++) { + Node* u = c->fast_out(k); + if (u->is_CFG()) { + wq.push(u); + } + } + } + assert(mem != NULL, "should have found safepoint"); } } } else { @@ -3569,12 +2288,6 @@ assert(_alias == Compile::AliasIdxRaw, ""); stack.push(mem, mem->req()); mem = mem->in(MemNode::Memory); - } else if (mem->Opcode() == Op_ShenandoahWriteBarrier) { - assert(_alias != Compile::AliasIdxRaw, ""); - mem = mem->in(ShenandoahBarrierNode::Memory); - } else if (mem->Opcode() == Op_ShenandoahWBMemProj) { - stack.push(mem, mem->req()); - mem = mem->in(ShenandoahWBMemProjNode::WriteBarrier); } else { #ifdef ASSERT mem->dump(); @@ -3628,7 +2341,7 @@ while (progress) { progress = false; iteration++; - assert(iteration <= 2+max_depth || _phase->C->has_irreducible_loop(), ""); + assert(iteration <= 2+max_depth || _phase->C->has_irreducible_loop() || has_never_branch(_phase->C->root()), ""); if (trace) { tty->print_cr("XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"); } IdealLoopTree* last_updated_ilt = NULL; for (int i = rpo_list.size() - 1; i >= 0; i--) { @@ -3796,7 +2509,7 @@ mem = _memory_nodes[c->_idx]; } if (n != NULL && mem_is_valid(mem, c)) { - while (!ShenandoahWriteBarrierNode::is_dominator_same_ctrl(c, mem, n, _phase) && _phase->ctrl_or_self(mem) == ctrl) { + while (!ShenandoahBarrierC2Support::is_dominator_same_ctrl(c, mem, n, _phase) && _phase->ctrl_or_self(mem) == ctrl) { mem = next_mem(mem, _alias); } if (mem->is_MergeMem()) { @@ -3842,12 +2555,6 @@ } else if (old->Opcode() == Op_SCMemProj) { assert(_alias == Compile::AliasIdxRaw, ""); old = old->in(0); - } else if (old->Opcode() == Op_ShenandoahWBMemProj) { - assert(_alias != Compile::AliasIdxRaw, ""); - old = old->in(ShenandoahWBMemProjNode::WriteBarrier); - } else if (old->Opcode() == Op_ShenandoahWriteBarrier) { - assert(_alias != Compile::AliasIdxRaw, ""); - old = old->in(ShenandoahBarrierNode::Memory); } else { ShouldNotReachHere(); } @@ -3857,7 +2564,7 @@ _memory_nodes.map(ctrl->_idx, mem); _memory_nodes.map(new_ctrl->_idx, mem_for_ctrl); } - uint input = prev->Opcode() == Op_ShenandoahWriteBarrier ? (uint)ShenandoahBarrierNode::Memory : (uint)MemNode::Memory; + uint input = (uint)MemNode::Memory; _phase->igvn().replace_input_of(prev, input, new_mem); } else { uses.clear(); @@ -3925,19 +2632,14 @@ } else { DEBUG_ONLY(if (trace) { tty->print("ZZZ NOT setting mem"); m->dump(); }); for (;;) { - assert(m->is_Mem() || m->is_LoadStore() || m->is_Proj() || m->Opcode() == Op_ShenandoahWriteBarrier || m->Opcode() == Op_ShenandoahWBMemProj, ""); + assert(m->is_Mem() || m->is_LoadStore() || m->is_Proj(), ""); Node* next = NULL; if (m->is_Proj()) { next = m->in(0); - } else if (m->Opcode() == Op_ShenandoahWBMemProj) { - next = m->in(ShenandoahWBMemProjNode::WriteBarrier); - } else if (m->is_Mem() || m->is_LoadStore()) { + } else { + assert(m->is_Mem() || m->is_LoadStore(), ""); assert(_alias == Compile::AliasIdxRaw, ""); next = m->in(MemNode::Memory); - } else { - assert(_alias != Compile::AliasIdxRaw, ""); - assert (m->Opcode() == Op_ShenandoahWriteBarrier, ""); - next = m->in(ShenandoahBarrierNode::Memory); } if (_phase->get_ctrl(next) != u) { break; @@ -3954,8 +2656,8 @@ } DEBUG_ONLY(if (trace) { tty->print("ZZZ setting to phi"); m->dump(); }); - assert(m->is_Mem() || m->is_LoadStore() || m->Opcode() == Op_ShenandoahWriteBarrier, ""); - uint input = (m->is_Mem() || m->is_LoadStore()) ? (uint)MemNode::Memory : (uint)ShenandoahBarrierNode::Memory; + assert(m->is_Mem() || m->is_LoadStore(), ""); + uint input = (uint)MemNode::Memory; _phase->igvn().replace_input_of(m, input, phi); push = false; } @@ -4181,20 +2883,7 @@ for (DUIterator i = mem->outs(); mem->has_out(i); i++) { Node* u = mem->out(i); if (u != replacement && u->_idx < last) { - if (u->is_ShenandoahBarrier() && _alias != Compile::AliasIdxRaw) { - if (_phase->C->get_alias_index(u->adr_type()) == _alias && ShenandoahWriteBarrierNode::is_dominator(rep_ctrl, _phase->ctrl_or_self(u), replacement, u, _phase)) { - _phase->igvn().replace_input_of(u, u->find_edge(mem), rep_proj); - assert(u->find_edge(mem) == -1, "only one edge"); - --i; - } - } else if (u->is_Mem()) { - if (_phase->C->get_alias_index(u->adr_type()) == _alias && ShenandoahWriteBarrierNode::is_dominator(rep_ctrl, _phase->ctrl_or_self(u), replacement, u, _phase)) { - assert(_alias == Compile::AliasIdxRaw , "only raw memory can lead to a memory operation"); - _phase->igvn().replace_input_of(u, u->find_edge(mem), rep_proj); - assert(u->find_edge(mem) == -1, "only one edge"); - --i; - } - } else if (u->is_MergeMem()) { + if (u->is_MergeMem()) { MergeMemNode* u_mm = u->as_MergeMem(); if (u_mm->memory_at(_alias) == mem) { MergeMemNode* newmm = NULL; @@ -4222,7 +2911,7 @@ } } } else { - if (rep_ctrl != uu && ShenandoahWriteBarrierNode::is_dominator(rep_ctrl, _phase->ctrl_or_self(uu), replacement, uu, _phase)) { + if (rep_ctrl != uu && ShenandoahBarrierC2Support::is_dominator(rep_ctrl, _phase->ctrl_or_self(uu), replacement, uu, _phase)) { if (newmm == NULL) { newmm = clone_merge_mem(u, mem, rep_proj, rep_ctrl, i); } @@ -4263,10 +2952,11 @@ u->Opcode() == Op_Rethrow || u->Opcode() == Op_Return || u->Opcode() == Op_SafePoint || + u->Opcode() == Op_StoreLConditional || (u->is_CallStaticJava() && u->as_CallStaticJava()->uncommon_trap_request() != 0) || (u->is_CallStaticJava() && u->as_CallStaticJava()->_entry_point == OptoRuntime::rethrow_stub()) || u->Opcode() == Op_CallLeaf, ""); - if (ShenandoahWriteBarrierNode::is_dominator(rep_ctrl, _phase->ctrl_or_self(u), replacement, u, _phase)) { + if (ShenandoahBarrierC2Support::is_dominator(rep_ctrl, _phase->ctrl_or_self(u), replacement, u, _phase)) { if (mm == NULL) { mm = allocate_merge_mem(mem, rep_proj, rep_ctrl); } @@ -4274,7 +2964,7 @@ --i; } } else if (_phase->C->get_alias_index(u->adr_type()) == _alias) { - if (ShenandoahWriteBarrierNode::is_dominator(rep_ctrl, _phase->ctrl_or_self(u), replacement, u, _phase)) { + if (ShenandoahBarrierC2Support::is_dominator(rep_ctrl, _phase->ctrl_or_self(u), replacement, u, _phase)) { _phase->igvn().replace_input_of(u, u->find_edge(mem), rep_proj); --i; } @@ -4283,11 +2973,322 @@ } } -void MemoryGraphFixer::remove(Node* n) { - assert(n->Opcode() == Op_ShenandoahWBMemProj, ""); - Node* c = _phase->get_ctrl(n); - Node* mem = find_mem(c, NULL); - if (mem == n) { - _memory_nodes.map(c->_idx, mem->in(ShenandoahWBMemProjNode::WriteBarrier)->in(ShenandoahBarrierNode::Memory)); +ShenandoahLoadReferenceBarrierNode::ShenandoahLoadReferenceBarrierNode(Node* ctrl, Node* obj) +: Node(ctrl, obj) { + ShenandoahBarrierSetC2::bsc2()->state()->add_load_reference_barrier(this); +} + +const Type* ShenandoahLoadReferenceBarrierNode::bottom_type() const { + if (in(ValueIn) == NULL || in(ValueIn)->is_top()) { + return Type::TOP; } + const Type* t = in(ValueIn)->bottom_type(); + if (t == TypePtr::NULL_PTR) { + return t; + } + return t->is_oopptr(); } + +const Type* ShenandoahLoadReferenceBarrierNode::Value(PhaseGVN* phase) const { + // Either input is TOP ==> the result is TOP + const Type *t2 = phase->type(in(ValueIn)); + if( t2 == Type::TOP ) return Type::TOP; + + if (t2 == TypePtr::NULL_PTR) { + return t2; + } + + const Type* type = t2->is_oopptr()/*->cast_to_nonconst()*/; + return type; +} + +Node* ShenandoahLoadReferenceBarrierNode::Identity(PhaseGVN* phase) { + Node* value = in(ValueIn); + if (!needs_barrier(phase, value)) { + return value; + } + return this; +} + +bool ShenandoahLoadReferenceBarrierNode::needs_barrier(PhaseGVN* phase, Node* n) { + Unique_Node_List visited; + return needs_barrier_impl(phase, n, visited); +} + +bool ShenandoahLoadReferenceBarrierNode::needs_barrier_impl(PhaseGVN* phase, Node* n, Unique_Node_List &visited) { + if (n == NULL) return false; + if (visited.member(n)) { + return false; // Been there. + } + visited.push(n); + + if (n->is_Allocate()) { + // tty->print_cr("optimize barrier on alloc"); + return false; + } + if (n->is_Call()) { + // tty->print_cr("optimize barrier on call"); + return false; + } + + const Type* type = phase->type(n); + if (type == Type::TOP) { + return false; + } + if (type->make_ptr()->higher_equal(TypePtr::NULL_PTR)) { + // tty->print_cr("optimize barrier on null"); + return false; + } + if (type->make_oopptr() && type->make_oopptr()->const_oop() != NULL) { + // tty->print_cr("optimize barrier on constant"); + return false; + } + + switch (n->Opcode()) { + case Op_AddP: + return true; // TODO: Can refine? + case Op_LoadP: + case Op_ShenandoahCompareAndExchangeN: + case Op_ShenandoahCompareAndExchangeP: + case Op_CompareAndExchangeN: + case Op_CompareAndExchangeP: + case Op_GetAndSetN: + case Op_GetAndSetP: + return true; + case Op_Phi: { + for (uint i = 1; i < n->req(); i++) { + if (needs_barrier_impl(phase, n->in(i), visited)) return true; + } + return false; + } + case Op_CheckCastPP: + case Op_CastPP: + return needs_barrier_impl(phase, n->in(1), visited); + case Op_Proj: + return needs_barrier_impl(phase, n->in(0), visited); + case Op_ShenandoahLoadReferenceBarrier: + // tty->print_cr("optimize barrier on barrier"); + return false; + case Op_Parm: + // tty->print_cr("optimize barrier on input arg"); + return false; + case Op_DecodeN: + case Op_EncodeP: + return needs_barrier_impl(phase, n->in(1), visited); + case Op_LoadN: + return true; + case Op_CMoveP: + return needs_barrier_impl(phase, n->in(2), visited) || + needs_barrier_impl(phase, n->in(3), visited); + case Op_ShenandoahEnqueueBarrier: + return needs_barrier_impl(phase, n->in(1), visited); + default: + break; + } +#ifdef ASSERT + tty->print("need barrier on?: "); + tty->print_cr("ins:"); + n->dump(2); + tty->print_cr("outs:"); + n->dump(-2); + ShouldNotReachHere(); +#endif + return true; +} + +ShenandoahLoadReferenceBarrierNode::Strength ShenandoahLoadReferenceBarrierNode::get_barrier_strength() { + Unique_Node_List visited; + Node_Stack stack(0); + stack.push(this, 0); + Strength strength = NONE; + while (strength != STRONG && stack.size() > 0) { + Node* n = stack.node(); + if (visited.member(n)) { + stack.pop(); + continue; + } + visited.push(n); + bool visit_users = false; + switch (n->Opcode()) { + case Op_StoreN: + case Op_StoreP: { + strength = STRONG; + break; + } + case Op_CmpP: { + if (!n->in(1)->bottom_type()->higher_equal(TypePtr::NULL_PTR) && + !n->in(2)->bottom_type()->higher_equal(TypePtr::NULL_PTR)) { + strength = STRONG; + } + break; + } + case Op_CallStaticJava: { + strength = STRONG; + break; + } + case Op_CallDynamicJava: + case Op_CallLeaf: + case Op_CallLeafNoFP: + case Op_CompareAndSwapL: + case Op_CompareAndSwapI: + case Op_CompareAndSwapB: + case Op_CompareAndSwapS: + case Op_ShenandoahCompareAndSwapN: + case Op_ShenandoahCompareAndSwapP: + case Op_ShenandoahWeakCompareAndSwapN: + case Op_ShenandoahWeakCompareAndSwapP: + case Op_ShenandoahCompareAndExchangeN: + case Op_ShenandoahCompareAndExchangeP: + case Op_CompareAndExchangeL: + case Op_CompareAndExchangeI: + case Op_CompareAndExchangeB: + case Op_CompareAndExchangeS: + case Op_WeakCompareAndSwapL: + case Op_WeakCompareAndSwapI: + case Op_WeakCompareAndSwapB: + case Op_WeakCompareAndSwapS: + case Op_GetAndSetL: + case Op_GetAndSetI: + case Op_GetAndSetB: + case Op_GetAndSetS: + case Op_GetAndSetP: + case Op_GetAndSetN: + case Op_GetAndAddL: + case Op_GetAndAddI: + case Op_GetAndAddB: + case Op_GetAndAddS: + case Op_ShenandoahEnqueueBarrier: + case Op_FastLock: + case Op_FastUnlock: + case Op_Rethrow: + case Op_Return: + case Op_StoreB: + case Op_StoreC: + case Op_StoreD: + case Op_StoreF: + case Op_StoreL: + case Op_StoreLConditional: + case Op_StoreI: + case Op_StoreVector: + case Op_StrInflatedCopy: + case Op_StrCompressedCopy: + case Op_EncodeP: + case Op_CastP2X: + case Op_SafePoint: + case Op_EncodeISOArray: + strength = STRONG; + break; + case Op_LoadB: + case Op_LoadUB: + case Op_LoadUS: + case Op_LoadD: + case Op_LoadF: + case Op_LoadL: + case Op_LoadI: + case Op_LoadS: + case Op_LoadN: + case Op_LoadP: + case Op_LoadVector: { + const TypePtr* adr_type = n->adr_type(); + int alias_idx = Compile::current()->get_alias_index(adr_type); + Compile::AliasType* alias_type = Compile::current()->alias_type(alias_idx); + ciField* field = alias_type->field(); + bool is_static = field != NULL && field->is_static(); + bool is_final = field != NULL && field->is_final(); + bool is_stable = field != NULL && field->is_stable(); + if (ShenandoahOptimizeStaticFinals && is_static && is_final) { + // Leave strength as is. + } else if (ShenandoahOptimizeInstanceFinals && !is_static && is_final) { + // Leave strength as is. + } else if (ShenandoahOptimizeStableFinals && (is_stable || (adr_type->isa_aryptr() && adr_type->isa_aryptr()->is_stable()))) { + // Leave strength as is. + } else { + strength = WEAK; + } + break; + } + case Op_AryEq: { + Node* n1 = n->in(2); + Node* n2 = n->in(3); + if (!ShenandoahOptimizeStableFinals || + !n1->bottom_type()->isa_aryptr() || !n1->bottom_type()->isa_aryptr()->is_stable() || + !n2->bottom_type()->isa_aryptr() || !n2->bottom_type()->isa_aryptr()->is_stable()) { + strength = WEAK; + } + break; + } + case Op_StrEquals: + case Op_StrComp: + case Op_StrIndexOf: + case Op_StrIndexOfChar: + if (!ShenandoahOptimizeStableFinals) { + strength = WEAK; + } + break; + case Op_Conv2B: + case Op_LoadRange: + case Op_LoadKlass: + case Op_LoadNKlass: + // NONE, i.e. leave current strength as is + break; + case Op_AddP: + case Op_CheckCastPP: + case Op_CastPP: + case Op_CMoveP: + case Op_Phi: + case Op_ShenandoahLoadReferenceBarrier: + visit_users = true; + break; + default: { +#ifdef ASSERT + tty->print_cr("Unknown node in get_barrier_strength:"); + n->dump(1); + ShouldNotReachHere(); +#else + strength = STRONG; +#endif + } + } +#ifdef ASSERT +/* + if (strength == STRONG) { + tty->print("strengthening node: "); + n->dump(); + } + */ +#endif + stack.pop(); + if (visit_users) { + for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) { + Node* user = n->fast_out(i); + if (user != NULL) { + stack.push(user, 0); + } + } + } + } + return strength; +} + +CallStaticJavaNode* ShenandoahLoadReferenceBarrierNode::pin_and_expand_null_check(PhaseIterGVN& igvn) { + Node* val = in(ValueIn); + + const Type* val_t = igvn.type(val); + + if (val_t->meet(TypePtr::NULL_PTR) != val_t && + val->Opcode() == Op_CastPP && + val->in(0) != NULL && + val->in(0)->Opcode() == Op_IfTrue && + val->in(0)->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none) && + val->in(0)->in(0)->is_If() && + val->in(0)->in(0)->in(1)->Opcode() == Op_Bool && + val->in(0)->in(0)->in(1)->as_Bool()->_test._test == BoolTest::ne && + val->in(0)->in(0)->in(1)->in(1)->Opcode() == Op_CmpP && + val->in(0)->in(0)->in(1)->in(1)->in(1) == val->in(1) && + val->in(0)->in(0)->in(1)->in(1)->in(2)->bottom_type() == TypePtr::NULL_PTR) { + assert(val->in(0)->in(0)->in(1)->in(1)->in(1) == val->in(1), ""); + CallStaticJavaNode* unc = val->in(0)->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none); + return unc; + } + return NULL; +} diff --git a/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.hpp b/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.hpp --- a/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.hpp +++ b/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.hpp @@ -36,10 +36,8 @@ class PhaseGVN; class MemoryGraphFixer; -class ShenandoahBarrierNode : public TypeNode { +class ShenandoahBarrierC2Support : public AllStatic { private: - bool _allow_fromspace; - #ifdef ASSERT enum verify_type { ShenandoahLoad, @@ -50,204 +48,49 @@ }; static bool verify_helper(Node* in, Node_Stack& phis, VectorSet& visited, verify_type t, bool trace, Unique_Node_List& barriers_used); -#endif - -public: - enum { Control, - Memory, - ValueIn - }; - - ShenandoahBarrierNode(Node* ctrl, Node* mem, Node* obj, bool allow_fromspace) - : TypeNode(obj->bottom_type()->isa_oopptr() ? obj->bottom_type()->is_oopptr()->cast_to_nonconst() : obj->bottom_type(), 3), - _allow_fromspace(allow_fromspace) { - - init_req(Control, ctrl); - init_req(Memory, mem); - init_req(ValueIn, obj); - - init_class_id(Class_ShenandoahBarrier); - } - - static Node* skip_through_barrier(Node* n); - - static const TypeOopPtr* brooks_pointer_type(const Type* t) { - return t->is_oopptr()->cast_to_nonconst()->add_offset(ShenandoahBrooksPointer::byte_offset())->is_oopptr(); - } - - virtual const TypePtr* adr_type() const { - if (bottom_type() == Type::TOP) { - return NULL; - } - //const TypePtr* adr_type = in(MemNode::Address)->bottom_type()->is_ptr(); - const TypePtr* adr_type = brooks_pointer_type(bottom_type()); - assert(adr_type->offset() == ShenandoahBrooksPointer::byte_offset(), "sane offset"); - assert(Compile::current()->alias_type(adr_type)->is_rewritable(), "brooks ptr must be rewritable"); - return adr_type; - } - - virtual uint ideal_reg() const { return Op_RegP; } - virtual uint match_edge(uint idx) const { - return idx >= ValueIn; - } - - Node* Identity_impl(PhaseGVN* phase); - - virtual const Type* Value(PhaseGVN* phase) const; - virtual bool depends_only_on_test() const { - return true; - }; - - static bool needs_barrier(PhaseGVN* phase, ShenandoahBarrierNode* orig, Node* n, Node* rb_mem, bool allow_fromspace); - -#ifdef ASSERT static void report_verify_failure(const char* msg, Node* n1 = NULL, Node* n2 = NULL); - static void verify(RootNode* root); static void verify_raw_mem(RootNode* root); #endif -#ifndef PRODUCT - virtual void dump_spec(outputStream *st) const; -#endif - - // protected: - static Node* dom_mem(Node* mem, Node*& mem_ctrl, Node* n, Node* rep_ctrl, int alias, PhaseIdealLoop* phase); static Node* dom_mem(Node* mem, Node* ctrl, int alias, Node*& mem_ctrl, PhaseIdealLoop* phase); - static bool is_dominator(Node *d_c, Node *n_c, Node* d, Node* n, PhaseIdealLoop* phase); - static bool is_dominator_same_ctrl(Node* c, Node* d, Node* n, PhaseIdealLoop* phase); static Node* no_branches(Node* c, Node* dom, bool allow_one_proj, PhaseIdealLoop* phase); - static bool build_loop_late_post(PhaseIdealLoop* phase, Node* n); - bool sink_node(PhaseIdealLoop* phase, Node* ctrl, Node* n_ctrl); - -protected: - uint hash() const; - bool cmp(const Node& n) const; - uint size_of() const; - -private: - static bool needs_barrier_impl(PhaseGVN* phase, ShenandoahBarrierNode* orig, Node* n, Node* rb_mem, bool allow_fromspace, Unique_Node_List &visited); - - static bool dominates_memory(PhaseGVN* phase, Node* b1, Node* b2, bool linear); - static bool dominates_memory_impl(PhaseGVN* phase, Node* b1, Node* b2, Node* current, bool linear); -}; - -class ShenandoahReadBarrierNode : public ShenandoahBarrierNode { -public: - ShenandoahReadBarrierNode(Node* ctrl, Node* mem, Node* obj) - : ShenandoahBarrierNode(ctrl, mem, obj, true) { - assert(UseShenandoahGC && (ShenandoahReadBarrier || ShenandoahStoreValReadBarrier || - ShenandoahWriteBarrier || ShenandoahAcmpBarrier), - "should be enabled"); - } - ShenandoahReadBarrierNode(Node* ctrl, Node* mem, Node* obj, bool allow_fromspace) - : ShenandoahBarrierNode(ctrl, mem, obj, allow_fromspace) { - assert(UseShenandoahGC && (ShenandoahReadBarrier || ShenandoahStoreValReadBarrier || - ShenandoahWriteBarrier || ShenandoahAcmpBarrier), - "should be enabled"); - } - - virtual Node *Ideal(PhaseGVN *phase, bool can_reshape); - virtual Node* Identity(PhaseGVN* phase); - virtual int Opcode() const; - - bool is_independent(Node* mem); - - void try_move(PhaseIdealLoop* phase); - -private: - static bool is_independent(const Type* in_type, const Type* this_type); - static bool dominates_memory_rb(PhaseGVN* phase, Node* b1, Node* b2, bool linear); - static bool dominates_memory_rb_impl(PhaseGVN* phase, Node* b1, Node* b2, Node* current, bool linear); -}; - -class ShenandoahWriteBarrierNode : public ShenandoahBarrierNode { -public: - ShenandoahWriteBarrierNode(Compile* C, Node* ctrl, Node* mem, Node* obj); - - virtual int Opcode() const; - virtual Node *Ideal(PhaseGVN *phase, bool can_reshape); - virtual Node* Identity(PhaseGVN* phase); - virtual bool depends_only_on_test() const { return false; } - - static bool expand(Compile* C, PhaseIterGVN& igvn); - static bool is_gc_state_load(Node *n); static bool is_heap_state_test(Node* iff, int mask); - static bool is_heap_stable_test(Node* iff); static bool try_common_gc_state_load(Node *n, PhaseIdealLoop *phase); static bool has_safepoint_between(Node* start, Node* stop, PhaseIdealLoop *phase); - - static LoopNode* try_move_before_pre_loop(Node* c, Node* val_ctrl, PhaseIdealLoop* phase); - static Node* move_above_predicates(LoopNode* cl, Node* val_ctrl, PhaseIdealLoop* phase); -#ifdef ASSERT - static bool memory_dominates_all_paths(Node* mem, Node* rep_ctrl, int alias, PhaseIdealLoop* phase); - static void memory_dominates_all_paths_helper(Node* c, Node* rep_ctrl, Unique_Node_List& controls, PhaseIdealLoop* phase); -#endif - void try_move_before_loop(GrowableArray& memory_graph_fixers, PhaseIdealLoop* phase, bool include_lsm, Unique_Node_List& uses); - void try_move_before_loop_helper(LoopNode* cl, Node* val_ctrl, GrowableArray& memory_graph_fixers, PhaseIdealLoop* phase, bool include_lsm, Unique_Node_List& uses); - static void pin_and_expand(PhaseIdealLoop* phase); - CallStaticJavaNode* pin_and_expand_null_check(PhaseIterGVN& igvn); - void pin_and_expand_move_barrier(PhaseIdealLoop* phase, GrowableArray& memory_graph_fixers, Unique_Node_List& uses); - void pin_and_expand_helper(PhaseIdealLoop* phase); static Node* find_bottom_mem(Node* ctrl, PhaseIdealLoop* phase); static void follow_barrier_uses(Node* n, Node* ctrl, Unique_Node_List& uses, PhaseIdealLoop* phase); static void test_null(Node*& ctrl, Node* val, Node*& null_ctrl, PhaseIdealLoop* phase); - static void test_heap_stable(Node*& ctrl, Node* raw_mem, Node*& heap_stable_ctrl, PhaseIdealLoop* phase); - static void call_wb_stub(Node*& ctrl, Node*& val, Node*& result_mem, - Node* raw_mem, Node* wb_mem, int alias, - PhaseIdealLoop* phase); + static void call_lrb_stub(Node*& ctrl, Node*& val, Node*& result_mem, Node* raw_mem, PhaseIdealLoop* phase); static Node* clone_null_check(Node*& c, Node* val, Node* unc_ctrl, PhaseIdealLoop* phase); static void fix_null_check(Node* unc, Node* unc_ctrl, Node* new_unc_ctrl, Unique_Node_List& uses, PhaseIdealLoop* phase); static void in_cset_fast_test(Node*& ctrl, Node*& not_cset_ctrl, Node* val, Node* raw_mem, PhaseIdealLoop* phase); static void move_heap_stable_test_out_of_loop(IfNode* iff, PhaseIdealLoop* phase); - - static void optimize_after_expansion(VectorSet &visited, Node_Stack &nstack, Node_List &old_new, PhaseIdealLoop* phase); static void merge_back_to_back_tests(Node* n, PhaseIdealLoop* phase); static bool identical_backtoback_ifs(Node *n, PhaseIdealLoop* phase); static void fix_ctrl(Node* barrier, Node* region, const MemoryGraphFixer& fixer, Unique_Node_List& uses, Unique_Node_List& uses_to_ignore, uint last, PhaseIdealLoop* phase); - - static void optimize_before_expansion(PhaseIdealLoop* phase, GrowableArray memory_graph_fixers, bool include_lsm); - Node* would_subsume(ShenandoahBarrierNode* other, PhaseIdealLoop* phase); static IfNode* find_unswitching_candidate(const IdealLoopTree *loop, PhaseIdealLoop* phase); - Node* try_split_thru_phi(PhaseIdealLoop* phase); -}; +public: + static bool is_dominator(Node* d_c, Node* n_c, Node* d, Node* n, PhaseIdealLoop* phase); + static bool is_dominator_same_ctrl(Node* c, Node* d, Node* n, PhaseIdealLoop* phase); -class ShenandoahWBMemProjNode : public Node { -public: - enum { Control, - WriteBarrier }; + static bool is_gc_state_load(Node* n); + static bool is_heap_stable_test(Node* iff); - ShenandoahWBMemProjNode(Node *src) : Node(NULL, src) { - assert(UseShenandoahGC && ShenandoahWriteBarrier, "should be enabled"); - assert(src->Opcode() == Op_ShenandoahWriteBarrier || src->is_Mach(), "epxect wb"); - } - virtual Node* Identity(PhaseGVN* phase); + static bool expand(Compile* C, PhaseIterGVN& igvn); + static void pin_and_expand(PhaseIdealLoop* phase); + static void optimize_after_expansion(VectorSet& visited, Node_Stack& nstack, Node_List& old_new, PhaseIdealLoop* phase); - virtual int Opcode() const; - virtual bool is_CFG() const { return false; } - virtual const Type *bottom_type() const {return Type::MEMORY;} - virtual const TypePtr *adr_type() const { - Node* wb = in(WriteBarrier); - if (wb == NULL || wb->is_top()) return NULL; // node is dead - assert(wb->Opcode() == Op_ShenandoahWriteBarrier || (wb->is_Mach() && wb->as_Mach()->ideal_Opcode() == Op_ShenandoahWriteBarrier) || wb->is_Phi(), "expect wb"); - return ShenandoahBarrierNode::brooks_pointer_type(wb->bottom_type()); - } - - virtual uint ideal_reg() const { return 0;} // memory projections don't have a register - virtual const Type *Value(PhaseGVN* phase ) const { - return bottom_type(); - } -#ifndef PRODUCT - virtual void dump_spec(outputStream *st) const {}; +#ifdef ASSERT + static void verify(RootNode* root); #endif }; class ShenandoahEnqueueBarrierNode : public Node { public: - ShenandoahEnqueueBarrierNode(Node* val) : Node(NULL, val) { - } + ShenandoahEnqueueBarrierNode(Node* val); const Type *bottom_type() const; const Type* Value(PhaseGVN* phase) const; @@ -289,7 +132,6 @@ Node* find_mem(Node* ctrl, Node* n) const; void fix_mem(Node* ctrl, Node* region, Node* mem, Node* mem_for_ctrl, Node* mem_phi, Unique_Node_List& uses); int alias() const { return _alias; } - void remove(Node* n); }; class ShenandoahCompareAndSwapPNode : public CompareAndSwapPNode { @@ -382,4 +224,41 @@ virtual int Opcode() const; }; +class ShenandoahLoadReferenceBarrierNode : public Node { +public: + enum { + Control, + ValueIn + }; + + enum Strength { + NONE, WEAK, STRONG, NA + }; + + ShenandoahLoadReferenceBarrierNode(Node* ctrl, Node* val); + + virtual int Opcode() const; + virtual const Type* bottom_type() const; + virtual const Type* Value(PhaseGVN* phase) const; + virtual const class TypePtr *adr_type() const { return TypeOopPtr::BOTTOM; } + virtual uint match_edge(uint idx) const { + return idx >= ValueIn; + } + virtual uint ideal_reg() const { return Op_RegP; } + + virtual Node* Identity(PhaseGVN* phase); + + uint size_of() const { + return sizeof(*this); + } + + Strength get_barrier_strength(); + CallStaticJavaNode* pin_and_expand_null_check(PhaseIterGVN& igvn); + +private: + bool needs_barrier(PhaseGVN* phase, Node* n); + bool needs_barrier_impl(PhaseGVN* phase, Node* n, Unique_Node_List &visited); +}; + + #endif // SHARE_GC_SHENANDOAH_C2_SHENANDOAHSUPPORT_HPP diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp @@ -41,13 +41,10 @@ SHENANDOAH_ERGO_ENABLE_FLAG(ShenandoahImplicitGCInvokesConcurrent); // Final configuration checks + SHENANDOAH_CHECK_FLAG_SET(ShenandoahLoadRefBarrier); SHENANDOAH_CHECK_FLAG_SET(ShenandoahSATBBarrier); - SHENANDOAH_CHECK_FLAG_SET(ShenandoahReadBarrier); - SHENANDOAH_CHECK_FLAG_SET(ShenandoahWriteBarrier); - SHENANDOAH_CHECK_FLAG_SET(ShenandoahStoreValReadBarrier); SHENANDOAH_CHECK_FLAG_SET(ShenandoahKeepAliveBarrier); SHENANDOAH_CHECK_FLAG_SET(ShenandoahCASBarrier); - SHENANDOAH_CHECK_FLAG_SET(ShenandoahAcmpBarrier); SHENANDOAH_CHECK_FLAG_SET(ShenandoahCloneBarrier); } diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAggressiveHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAggressiveHeuristics.cpp --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAggressiveHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAggressiveHeuristics.cpp @@ -47,13 +47,10 @@ } // Final configuration checks + SHENANDOAH_CHECK_FLAG_SET(ShenandoahLoadRefBarrier); SHENANDOAH_CHECK_FLAG_SET(ShenandoahSATBBarrier); - SHENANDOAH_CHECK_FLAG_SET(ShenandoahReadBarrier); - SHENANDOAH_CHECK_FLAG_SET(ShenandoahWriteBarrier); - SHENANDOAH_CHECK_FLAG_SET(ShenandoahStoreValReadBarrier); SHENANDOAH_CHECK_FLAG_SET(ShenandoahKeepAliveBarrier); SHENANDOAH_CHECK_FLAG_SET(ShenandoahCASBarrier); - SHENANDOAH_CHECK_FLAG_SET(ShenandoahAcmpBarrier); SHENANDOAH_CHECK_FLAG_SET(ShenandoahCloneBarrier); } diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahCompactHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahCompactHeuristics.cpp --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahCompactHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahCompactHeuristics.cpp @@ -42,13 +42,10 @@ SHENANDOAH_ERGO_OVERRIDE_DEFAULT(ShenandoahGarbageThreshold, 10); // Final configuration checks + SHENANDOAH_CHECK_FLAG_SET(ShenandoahLoadRefBarrier); SHENANDOAH_CHECK_FLAG_SET(ShenandoahSATBBarrier); - SHENANDOAH_CHECK_FLAG_SET(ShenandoahReadBarrier); - SHENANDOAH_CHECK_FLAG_SET(ShenandoahWriteBarrier); - SHENANDOAH_CHECK_FLAG_SET(ShenandoahStoreValReadBarrier); SHENANDOAH_CHECK_FLAG_SET(ShenandoahKeepAliveBarrier); SHENANDOAH_CHECK_FLAG_SET(ShenandoahCASBarrier); - SHENANDOAH_CHECK_FLAG_SET(ShenandoahAcmpBarrier); SHENANDOAH_CHECK_FLAG_SET(ShenandoahCloneBarrier); } diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahPassiveHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahPassiveHeuristics.cpp --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahPassiveHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahPassiveHeuristics.cpp @@ -43,14 +43,11 @@ } // Disable known barriers by default. + SHENANDOAH_ERGO_DISABLE_FLAG(ShenandoahLoadRefBarrier); SHENANDOAH_ERGO_DISABLE_FLAG(ShenandoahSATBBarrier); SHENANDOAH_ERGO_DISABLE_FLAG(ShenandoahKeepAliveBarrier); - SHENANDOAH_ERGO_DISABLE_FLAG(ShenandoahWriteBarrier); - SHENANDOAH_ERGO_DISABLE_FLAG(ShenandoahReadBarrier); SHENANDOAH_ERGO_DISABLE_FLAG(ShenandoahStoreValEnqueueBarrier); - SHENANDOAH_ERGO_DISABLE_FLAG(ShenandoahStoreValReadBarrier); SHENANDOAH_ERGO_DISABLE_FLAG(ShenandoahCASBarrier); - SHENANDOAH_ERGO_DISABLE_FLAG(ShenandoahAcmpBarrier); SHENANDOAH_ERGO_DISABLE_FLAG(ShenandoahCloneBarrier); // Final configuration checks diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahStaticHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahStaticHeuristics.cpp --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahStaticHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahStaticHeuristics.cpp @@ -40,13 +40,10 @@ SHENANDOAH_ERGO_ENABLE_FLAG(ShenandoahImplicitGCInvokesConcurrent); // Final configuration checks + SHENANDOAH_CHECK_FLAG_SET(ShenandoahLoadRefBarrier); SHENANDOAH_CHECK_FLAG_SET(ShenandoahSATBBarrier); - SHENANDOAH_CHECK_FLAG_SET(ShenandoahReadBarrier); - SHENANDOAH_CHECK_FLAG_SET(ShenandoahWriteBarrier); - SHENANDOAH_CHECK_FLAG_SET(ShenandoahStoreValReadBarrier); SHENANDOAH_CHECK_FLAG_SET(ShenandoahKeepAliveBarrier); SHENANDOAH_CHECK_FLAG_SET(ShenandoahCASBarrier); - SHENANDOAH_CHECK_FLAG_SET(ShenandoahAcmpBarrier); SHENANDOAH_CHECK_FLAG_SET(ShenandoahCloneBarrier); } diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahTraversalHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahTraversalHeuristics.cpp --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahTraversalHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahTraversalHeuristics.cpp @@ -37,7 +37,6 @@ _last_cset_select(0) { FLAG_SET_DEFAULT(ShenandoahSATBBarrier, false); - FLAG_SET_DEFAULT(ShenandoahStoreValReadBarrier, false); FLAG_SET_DEFAULT(ShenandoahStoreValEnqueueBarrier, true); FLAG_SET_DEFAULT(ShenandoahKeepAliveBarrier, false); FLAG_SET_DEFAULT(ShenandoahAllowMixedAllocs, false); @@ -53,11 +52,9 @@ SHENANDOAH_ERGO_ENABLE_FLAG(ShenandoahImplicitGCInvokesConcurrent); // Final configuration checks - SHENANDOAH_CHECK_FLAG_SET(ShenandoahReadBarrier); - SHENANDOAH_CHECK_FLAG_SET(ShenandoahWriteBarrier); + SHENANDOAH_CHECK_FLAG_SET(ShenandoahLoadRefBarrier); SHENANDOAH_CHECK_FLAG_SET(ShenandoahStoreValEnqueueBarrier); SHENANDOAH_CHECK_FLAG_SET(ShenandoahCASBarrier); - SHENANDOAH_CHECK_FLAG_SET(ShenandoahAcmpBarrier); SHENANDOAH_CHECK_FLAG_SET(ShenandoahCloneBarrier); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahArguments.cpp b/src/hotspot/share/gc/shenandoah/shenandoahArguments.cpp --- a/src/hotspot/share/gc/shenandoah/shenandoahArguments.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahArguments.cpp @@ -46,12 +46,8 @@ FLAG_SET_DEFAULT(ShenandoahSATBBarrier, false); FLAG_SET_DEFAULT(ShenandoahKeepAliveBarrier, false); - FLAG_SET_DEFAULT(ShenandoahWriteBarrier, false); - FLAG_SET_DEFAULT(ShenandoahReadBarrier, false); FLAG_SET_DEFAULT(ShenandoahStoreValEnqueueBarrier, false); - FLAG_SET_DEFAULT(ShenandoahStoreValReadBarrier, false); FLAG_SET_DEFAULT(ShenandoahCASBarrier, false); - FLAG_SET_DEFAULT(ShenandoahAcmpBarrier, false); FLAG_SET_DEFAULT(ShenandoahCloneBarrier, false); #endif @@ -111,12 +107,8 @@ if (ShenandoahVerifyOptoBarriers && (!FLAG_IS_DEFAULT(ShenandoahSATBBarrier) || !FLAG_IS_DEFAULT(ShenandoahKeepAliveBarrier) || - !FLAG_IS_DEFAULT(ShenandoahWriteBarrier) || - !FLAG_IS_DEFAULT(ShenandoahReadBarrier) || !FLAG_IS_DEFAULT(ShenandoahStoreValEnqueueBarrier) || - !FLAG_IS_DEFAULT(ShenandoahStoreValReadBarrier) || !FLAG_IS_DEFAULT(ShenandoahCASBarrier) || - !FLAG_IS_DEFAULT(ShenandoahAcmpBarrier) || !FLAG_IS_DEFAULT(ShenandoahCloneBarrier) )) { warning("Unusual barrier configuration, disabling C2 barrier verification"); @@ -164,13 +156,6 @@ FLAG_SET_DEFAULT(UseAOT, false); } - // JNI fast get field stuff is not currently supported by Shenandoah. - // It would introduce another heap memory access for reading the forwarding - // pointer, which would have to be guarded by the signal handler machinery. - // See: - // http://mail.openjdk.java.net/pipermail/hotspot-dev/2018-June/032763.html - FLAG_SET_DEFAULT(UseFastJNIAccessors, false); - // TLAB sizing policy makes resizing decisions before each GC cycle. It averages // historical data, assigning more recent data the weight according to TLABAllocationWeight. // Current default is good for generational collectors that run frequent young GCs. diff --git a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp --- a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp @@ -218,31 +218,25 @@ } } -oop ShenandoahBarrierSet::read_barrier(oop src) { - // Check for forwarded objects, because on Full GC path we might deal with - // non-trivial fwdptrs that contain Full GC specific metadata. We could check - // for is_full_gc_in_progress(), but this also covers the case of stable heap, - // which provides a bit of performance improvement. - if (ShenandoahReadBarrier && _heap->has_forwarded_objects()) { - return ShenandoahBarrierSet::resolve_forwarded(src); +oop ShenandoahBarrierSet::load_reference_barrier_not_null(oop obj) { + if (ShenandoahLoadRefBarrier && _heap->has_forwarded_objects()) { + return load_reference_barrier_impl(obj); } else { - return src; + return obj; } } -bool ShenandoahBarrierSet::obj_equals(oop obj1, oop obj2) { - bool eq = oopDesc::equals_raw(obj1, obj2); - if (! eq && ShenandoahAcmpBarrier) { - OrderAccess::loadload(); - obj1 = resolve_forwarded(obj1); - obj2 = resolve_forwarded(obj2); - eq = oopDesc::equals_raw(obj1, obj2); +oop ShenandoahBarrierSet::load_reference_barrier(oop obj) { + if (obj != NULL) { + return load_reference_barrier_not_null(obj); + } else { + return obj; } - return eq; } -oop ShenandoahBarrierSet::write_barrier_mutator(oop obj) { - assert(UseShenandoahGC && ShenandoahWriteBarrier, "should be enabled"); + +oop ShenandoahBarrierSet::load_reference_barrier_mutator(oop obj) { + assert(ShenandoahLoadRefBarrier, "should be enabled"); assert(_heap->is_gc_in_progress_mask(ShenandoahHeap::EVACUATION | ShenandoahHeap::TRAVERSAL), "evac should be in progress"); shenandoah_assert_in_cset(NULL, obj); @@ -288,8 +282,8 @@ return fwd; } -oop ShenandoahBarrierSet::write_barrier_impl(oop obj) { - assert(UseShenandoahGC && ShenandoahWriteBarrier, "should be enabled"); +oop ShenandoahBarrierSet::load_reference_barrier_impl(oop obj) { + assert(ShenandoahLoadRefBarrier, "should be enabled"); if (!CompressedOops::is_null(obj)) { bool evac_in_progress = _heap->is_gc_in_progress_mask(ShenandoahHeap::EVACUATION | ShenandoahHeap::TRAVERSAL); oop fwd = resolve_forwarded_not_null(obj); @@ -311,23 +305,10 @@ } } -oop ShenandoahBarrierSet::write_barrier(oop obj) { - if (ShenandoahWriteBarrier && _heap->has_forwarded_objects()) { - return write_barrier_impl(obj); - } else { - return obj; - } -} - -oop ShenandoahBarrierSet::storeval_barrier(oop obj) { +void ShenandoahBarrierSet::storeval_barrier(oop obj) { if (ShenandoahStoreValEnqueueBarrier && !CompressedOops::is_null(obj) && _heap->is_concurrent_traversal_in_progress()) { - obj = write_barrier(obj); enqueue(obj); } - if (ShenandoahStoreValReadBarrier) { - obj = resolve_forwarded(obj); - } - return obj; } void ShenandoahBarrierSet::keep_alive_barrier(oop obj) { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.hpp b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.hpp --- a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.hpp @@ -87,24 +87,15 @@ virtual void on_thread_attach(Thread* thread); virtual void on_thread_detach(Thread* thread); - virtual oop read_barrier(oop src); - static inline oop resolve_forwarded_not_null(oop p); static inline oop resolve_forwarded(oop p); - virtual oop write_barrier(oop obj); + void storeval_barrier(oop obj); + void keep_alive_barrier(oop obj); - oop write_barrier_mutator(oop obj); - - virtual oop storeval_barrier(oop obj); - - virtual void keep_alive_barrier(oop obj); - - bool obj_equals(oop obj1, oop obj2); - -#ifdef CHECK_UNHANDLED_OOPS - bool oop_equals_operator_allowed() { return !ShenandoahVerifyObjectEquals; } -#endif + oop load_reference_barrier(oop obj); + oop load_reference_barrier_mutator(oop obj); + oop load_reference_barrier_not_null(oop obj); void enqueue(oop obj); @@ -114,7 +105,7 @@ template void write_ref_array_loop(HeapWord* start, size_t count); - oop write_barrier_impl(oop obj); + oop load_reference_barrier_impl(oop obj); static void keep_alive_if_weak(DecoratorSet decorators, oop value) { assert((decorators & ON_UNKNOWN_OOP_REF) == 0, "Reference strength must be known"); @@ -149,114 +140,31 @@ class AccessBarrier: public BarrierSet::AccessBarrier { typedef BarrierSet::AccessBarrier Raw; - public: - // Primitive heap accesses. These accessors get resolved when - // IN_HEAP is set (e.g. when using the HeapAccess API), it is - // not an oop_* overload, and the barrier strength is AS_NORMAL. template - static T load_in_heap(T* addr) { - ShouldNotReachHere(); - return Raw::template load(addr); - } + static oop oop_atomic_cmpxchg_in_heap_impl(oop new_value, T* addr, oop compare_value); template - static T load_in_heap_at(oop base, ptrdiff_t offset) { - base = ShenandoahBarrierSet::resolve_forwarded(base); - return Raw::template load_at(base, offset); - } + static oop oop_atomic_xchg_in_heap_impl(oop new_value, T* addr); - template - static void store_in_heap(T* addr, T value) { - ShouldNotReachHere(); - Raw::store(addr, value); - } - - template - static void store_in_heap_at(oop base, ptrdiff_t offset, T value) { - base = ShenandoahBarrierSet::barrier_set()->write_barrier(base); - Raw::store_at(base, offset, value); - } - - template - static T atomic_cmpxchg_in_heap(T new_value, T* addr, T compare_value) { - ShouldNotReachHere(); - return Raw::atomic_cmpxchg(new_value, addr, compare_value); - } - - template - static T atomic_cmpxchg_in_heap_at(T new_value, oop base, ptrdiff_t offset, T compare_value) { - base = ShenandoahBarrierSet::barrier_set()->write_barrier(base); - return Raw::atomic_cmpxchg_at(new_value, base, offset, compare_value); - } - - template - static T atomic_xchg_in_heap(T new_value, T* addr) { - ShouldNotReachHere(); - return Raw::atomic_xchg(new_value, addr); - } - - template - static T atomic_xchg_in_heap_at(T new_value, oop base, ptrdiff_t offset) { - base = ShenandoahBarrierSet::barrier_set()->write_barrier(base); - return Raw::atomic_xchg_at(new_value, base, offset); - } - - template - static void arraycopy_in_heap(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw, - arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw, - size_t length); - + public: // Heap oop accesses. These accessors get resolved when // IN_HEAP is set (e.g. when using the HeapAccess API), it is // an oop_* overload, and the barrier strength is AS_NORMAL. template - static oop oop_load_in_heap(T* addr) { - // ShouldNotReachHere(); - oop value = Raw::template oop_load(addr); - keep_alive_if_weak(decorators, value); - return value; - } - - static oop oop_load_in_heap_at(oop base, ptrdiff_t offset) { - base = ShenandoahBarrierSet::resolve_forwarded(base); - oop value = Raw::template oop_load_at(base, offset); - keep_alive_if_weak(AccessBarrierSupport::resolve_possibly_unknown_oop_ref_strength(base, offset), value); - return value; - } + static oop oop_load_in_heap(T* addr); + static oop oop_load_in_heap_at(oop base, ptrdiff_t offset); template - static void oop_store_in_heap(T* addr, oop value) { - const bool keep_alive = (decorators & AS_NO_KEEPALIVE) == 0; - if (keep_alive) { - ShenandoahBarrierSet::barrier_set()->write_ref_field_pre_work(addr, value); - } - Raw::oop_store(addr, value); - } - - static void oop_store_in_heap_at(oop base, ptrdiff_t offset, oop value) { - base = ShenandoahBarrierSet::barrier_set()->write_barrier(base); - value = ShenandoahBarrierSet::barrier_set()->storeval_barrier(value); - - oop_store_in_heap(AccessInternal::oop_field_addr(base, offset), value); - } + static void oop_store_in_heap(T* addr, oop value); + static void oop_store_in_heap_at(oop base, ptrdiff_t offset, oop value); template static oop oop_atomic_cmpxchg_in_heap(oop new_value, T* addr, oop compare_value); - - static oop oop_atomic_cmpxchg_in_heap_at(oop new_value, oop base, ptrdiff_t offset, oop compare_value) { - base = ShenandoahBarrierSet::barrier_set()->write_barrier(base); - new_value = ShenandoahBarrierSet::barrier_set()->storeval_barrier(new_value); - return oop_atomic_cmpxchg_in_heap(new_value, AccessInternal::oop_field_addr(base, offset), compare_value); - } + static oop oop_atomic_cmpxchg_in_heap_at(oop new_value, oop base, ptrdiff_t offset, oop compare_value); template static oop oop_atomic_xchg_in_heap(oop new_value, T* addr); - - static oop oop_atomic_xchg_in_heap_at(oop new_value, oop base, ptrdiff_t offset) { - base = ShenandoahBarrierSet::barrier_set()->write_barrier(base); - new_value = ShenandoahBarrierSet::barrier_set()->storeval_barrier(new_value); - return oop_atomic_xchg_in_heap(new_value, AccessInternal::oop_field_addr(base, offset)); - } + static oop oop_atomic_xchg_in_heap_at(oop new_value, oop base, ptrdiff_t offset); template static bool oop_arraycopy_in_heap(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw, @@ -268,19 +176,13 @@ // Needed for loads on non-heap weak references template - static oop oop_load_not_in_heap(T* addr) { - oop value = Raw::oop_load_not_in_heap(addr); - keep_alive_if_weak(decorators, value); - return value; - } + static oop oop_load_not_in_heap(T* addr); - static oop resolve(oop obj) { - return ShenandoahBarrierSet::barrier_set()->write_barrier(obj); - } + template + static oop oop_atomic_cmpxchg_not_in_heap(oop new_value, T* addr, oop compare_value); - static bool equals(oop o1, oop o2) { - return ShenandoahBarrierSet::barrier_set()->obj_equals(o1, o2); - } + template + static oop oop_atomic_xchg_not_in_heap(oop new_value, T* addr); }; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp --- a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp @@ -52,7 +52,49 @@ template template -inline oop ShenandoahBarrierSet::AccessBarrier::oop_atomic_cmpxchg_in_heap(oop new_value, T* addr, oop compare_value) { +inline oop ShenandoahBarrierSet::AccessBarrier::oop_load_in_heap(T* addr) { + oop value = Raw::oop_load_in_heap(addr); + value = ShenandoahBarrierSet::barrier_set()->load_reference_barrier(value); + keep_alive_if_weak(decorators, value); + return value; +} + +template +inline oop ShenandoahBarrierSet::AccessBarrier::oop_load_in_heap_at(oop base, ptrdiff_t offset) { + oop value = Raw::oop_load_in_heap_at(base, offset); + value = ShenandoahBarrierSet::barrier_set()->load_reference_barrier(value); + keep_alive_if_weak(AccessBarrierSupport::resolve_possibly_unknown_oop_ref_strength(base, offset), value); + return value; +} + +template +template +inline oop ShenandoahBarrierSet::AccessBarrier::oop_load_not_in_heap(T* addr) { + oop value = Raw::oop_load_not_in_heap(addr); + value = ShenandoahBarrierSet::barrier_set()->load_reference_barrier(value); + keep_alive_if_weak(decorators, value); + return value; +} + +template +template +inline void ShenandoahBarrierSet::AccessBarrier::oop_store_in_heap(T* addr, oop value) { + ShenandoahBarrierSet::barrier_set()->storeval_barrier(value); + const bool keep_alive = (decorators & AS_NO_KEEPALIVE) == 0; + if (keep_alive) { + ShenandoahBarrierSet::barrier_set()->write_ref_field_pre_work(addr, value); + } + Raw::oop_store_in_heap(addr, value); +} + +template +inline void ShenandoahBarrierSet::AccessBarrier::oop_store_in_heap_at(oop base, ptrdiff_t offset, oop value) { + oop_store_in_heap(AccessInternal::oop_field_addr(base, offset), value); +} + +template +template +inline oop ShenandoahBarrierSet::AccessBarrier::oop_atomic_cmpxchg_not_in_heap(oop new_value, T* addr, oop compare_value) { oop res; oop expected = compare_value; do { @@ -60,42 +102,79 @@ res = Raw::oop_atomic_cmpxchg(new_value, addr, compare_value); expected = res; } while ((! oopDesc::equals_raw(compare_value, expected)) && oopDesc::equals_raw(resolve_forwarded(compare_value), resolve_forwarded(expected))); - if (oopDesc::equals_raw(expected, compare_value)) { - const bool keep_alive = (decorators & AS_NO_KEEPALIVE) == 0; - if (keep_alive && ShenandoahSATBBarrier && !CompressedOops::is_null(compare_value) && - ShenandoahHeap::heap()->is_concurrent_mark_in_progress()) { - ShenandoahBarrierSet::barrier_set()->enqueue(compare_value); - } + if (res != NULL) { + return ShenandoahBarrierSet::barrier_set()->load_reference_barrier_not_null(res); + } else { + return res; } - return res; +} + +template +template +inline oop ShenandoahBarrierSet::AccessBarrier::oop_atomic_cmpxchg_in_heap_impl(oop new_value, T* addr, oop compare_value) { + ShenandoahBarrierSet::barrier_set()->storeval_barrier(new_value); + oop result = oop_atomic_cmpxchg_not_in_heap(new_value, addr, compare_value); + const bool keep_alive = (decorators & AS_NO_KEEPALIVE) == 0; + if (keep_alive && ShenandoahSATBBarrier && !CompressedOops::is_null(result) && + oopDesc::equals_raw(result, compare_value) && + ShenandoahHeap::heap()->is_concurrent_mark_in_progress()) { + ShenandoahBarrierSet::barrier_set()->enqueue(result); + } + return result; +} + +template +template +inline oop ShenandoahBarrierSet::AccessBarrier::oop_atomic_cmpxchg_in_heap(oop new_value, T* addr, oop compare_value) { + oop result = oop_atomic_cmpxchg_in_heap_impl(new_value, addr, compare_value); + keep_alive_if_weak(decorators, result); + return result; +} + +template +inline oop ShenandoahBarrierSet::AccessBarrier::oop_atomic_cmpxchg_in_heap_at(oop new_value, oop base, ptrdiff_t offset, oop compare_value) { + oop result = oop_atomic_cmpxchg_in_heap_impl(new_value, AccessInternal::oop_field_addr(base, offset), compare_value); + keep_alive_if_weak(AccessBarrierSupport::resolve_possibly_unknown_oop_ref_strength(base, offset), result); + return result; +} + +template +template +inline oop ShenandoahBarrierSet::AccessBarrier::oop_atomic_xchg_not_in_heap(oop new_value, T* addr) { + oop previous = Raw::oop_atomic_xchg(new_value, addr); + if (previous != NULL) { + return ShenandoahBarrierSet::barrier_set()->load_reference_barrier_not_null(previous); + } else { + return previous; + } +} + +template +template +inline oop ShenandoahBarrierSet::AccessBarrier::oop_atomic_xchg_in_heap_impl(oop new_value, T* addr) { + ShenandoahBarrierSet::barrier_set()->storeval_barrier(new_value); + oop result = oop_atomic_xchg_not_in_heap(new_value, addr); + const bool keep_alive = (decorators & AS_NO_KEEPALIVE) == 0; + if (keep_alive && ShenandoahSATBBarrier && !CompressedOops::is_null(result) && + ShenandoahHeap::heap()->is_concurrent_mark_in_progress()) { + ShenandoahBarrierSet::barrier_set()->enqueue(result); + } + return result; } template template inline oop ShenandoahBarrierSet::AccessBarrier::oop_atomic_xchg_in_heap(oop new_value, T* addr) { - oop previous = Raw::oop_atomic_xchg(new_value, addr); - if (ShenandoahSATBBarrier) { - const bool keep_alive = (decorators & AS_NO_KEEPALIVE) == 0; - if (keep_alive && !CompressedOops::is_null(previous) && - ShenandoahHeap::heap()->is_concurrent_mark_in_progress()) { - ShenandoahBarrierSet::barrier_set()->enqueue(previous); - } - } - return previous; + oop result = oop_atomic_xchg_in_heap_impl(new_value, addr); + keep_alive_if_weak(addr, result); + return result; } template -template -void ShenandoahBarrierSet::AccessBarrier::arraycopy_in_heap(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw, - arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw, - size_t length) { - if (!CompressedOops::is_null(src_obj)) { - src_obj = arrayOop(ShenandoahBarrierSet::barrier_set()->read_barrier(src_obj)); - } - if (!CompressedOops::is_null(dst_obj)) { - dst_obj = arrayOop(ShenandoahBarrierSet::barrier_set()->write_barrier(dst_obj)); - } - Raw::arraycopy(src_obj, src_offset_in_bytes, src_raw, dst_obj, dst_offset_in_bytes, dst_raw, length); +inline oop ShenandoahBarrierSet::AccessBarrier::oop_atomic_xchg_in_heap_at(oop new_value, oop base, ptrdiff_t offset) { + oop result = oop_atomic_xchg_in_heap_impl(new_value, AccessInternal::oop_field_addr(base, offset)); + keep_alive_if_weak(AccessBarrierSupport::resolve_possibly_unknown_oop_ref_strength(base, offset), result); + return result; } template @@ -248,8 +327,6 @@ // Clone barrier support template void ShenandoahBarrierSet::AccessBarrier::clone_in_heap(oop src, oop dst, size_t size) { - src = arrayOop(ShenandoahBarrierSet::barrier_set()->read_barrier(src)); - dst = arrayOop(ShenandoahBarrierSet::barrier_set()->write_barrier(dst)); Raw::clone(src, dst, size); ShenandoahBarrierSet::barrier_set()->write_region(MemRegion((HeapWord*) dst, size)); } @@ -260,13 +337,6 @@ arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw, size_t length) { ShenandoahHeap* heap = ShenandoahHeap::heap(); - if (!CompressedOops::is_null(src_obj)) { - src_obj = arrayOop(ShenandoahBarrierSet::barrier_set()->read_barrier(src_obj)); - } - if (!CompressedOops::is_null(dst_obj)) { - dst_obj = arrayOop(ShenandoahBarrierSet::barrier_set()->write_barrier(dst_obj)); - } - bool satb = ShenandoahSATBBarrier && heap->is_concurrent_mark_in_progress(); bool checkcast = HasDecorator::value; bool disjoint = HasDecorator::value; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahCodeRoots.cpp b/src/hotspot/share/gc/shenandoah/shenandoahCodeRoots.cpp --- a/src/hotspot/share/gc/shenandoah/shenandoahCodeRoots.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahCodeRoots.cpp @@ -119,39 +119,6 @@ } }; -class ShenandoahNMethodOopInitializer : public OopClosure { -private: - ShenandoahHeap* const _heap; - -public: - ShenandoahNMethodOopInitializer() : _heap(ShenandoahHeap::heap()) {}; - -private: - template - inline void do_oop_work(T* p) { - T o = RawAccess<>::oop_load(p); - if (! CompressedOops::is_null(o)) { - oop obj1 = CompressedOops::decode_not_null(o); - oop obj2 = ShenandoahBarrierSet::barrier_set()->write_barrier(obj1); - if (! oopDesc::equals_raw(obj1, obj2)) { - shenandoah_assert_not_in_cset(NULL, obj2); - RawAccess::oop_store(p, obj2); - if (_heap->is_concurrent_traversal_in_progress()) { - ShenandoahBarrierSet::barrier_set()->enqueue(obj2); - } - } - } - } - -public: - void do_oop(oop* o) { - do_oop_work(o); - } - void do_oop(narrowOop* o) { - do_oop_work(o); - } -}; - ShenandoahCodeRoots::PaddedLock ShenandoahCodeRoots::_recorded_nms_lock; GrowableArray* ShenandoahCodeRoots::_recorded_nms; @@ -163,21 +130,13 @@ void ShenandoahCodeRoots::add_nmethod(nmethod* nm) { switch (ShenandoahCodeRootsStyle) { case 0: - case 1: { - ShenandoahNMethodOopInitializer init; - nm->oops_do(&init); - nm->fix_oop_relocations(); + case 1: break; - } case 2: { ShenandoahNMethodOopDetector detector; nm->oops_do(&detector); if (detector.has_oops()) { - ShenandoahNMethodOopInitializer init; - nm->oops_do(&init); - nm->fix_oop_relocations(); - ShenandoahNMethod* nmr = new ShenandoahNMethod(nm, detector.oops()); nmr->assert_alive_and_correct(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahEvacOOMHandler.hpp b/src/hotspot/share/gc/shenandoah/shenandoahEvacOOMHandler.hpp --- a/src/hotspot/share/gc/shenandoah/shenandoahEvacOOMHandler.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahEvacOOMHandler.hpp @@ -31,8 +31,8 @@ * Provides safe handling of out-of-memory situations during evacuation. * * When a Java thread encounters out-of-memory while evacuating an object in a - * write-barrier (i.e. it cannot copy the object to to-space), it does not necessarily - * follow we can return immediately from the WB (and store to from-space). + * load-reference-barrier (i.e. it cannot copy the object to to-space), it does not + * necessarily follow we can return immediately from the LRB (and store to from-space). * * In very basic case, on such failure we may wait until the the evacuation is over, * and then resolve the forwarded copy, and to the store there. This is possible @@ -64,17 +64,17 @@ * - failure: * - if offending value is a valid counter, then try again * - if offending value is OOM-during-evac special value: loop until - * counter drops to 0, then exit with read-barrier + * counter drops to 0, then exit with resolving the ptr * * Upon exit, exiting thread will decrease the counter using atomic dec. * * Upon OOM-during-evac, any thread will attempt to CAS OOM-during-evac * special value into the counter. Depending on result: - * - success: busy-loop until counter drops to zero, then exit with RB + * - success: busy-loop until counter drops to zero, then exit with resolve * - failure: * - offender is valid counter update: try again * - offender is OOM-during-evac: busy loop until counter drops to - * zero, then exit with RB + * zero, then exit with resolve */ class ShenandoahEvacOOMHandler { private: @@ -94,7 +94,7 @@ * * When this returns true, it is safe to continue with normal evacuation. * When this method returns false, evacuation must not be entered, and caller - * may safely continue with a read-barrier (if Java thread). + * may safely continue with a simple resolve (if Java thread). */ void enter_evacuation(); @@ -106,7 +106,7 @@ /** * Signal out-of-memory during evacuation. It will prevent any other threads * from entering the evacuation path, then wait until all threads have left the - * evacuation path, and then return. It is then safe to continue with a read-barrier. + * evacuation path, and then return. It is then safe to continue with a simple resolve. */ void handle_out_of_memory_during_evacuation(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -389,10 +389,6 @@ err_msg("Heuristics \"%s\" is experimental, and must be enabled via -XX:+UnlockExperimentalVMOptions.", _heuristics->name())); } - - if (ShenandoahStoreValEnqueueBarrier && ShenandoahStoreValReadBarrier) { - vm_exit_during_initialization("Cannot use both ShenandoahStoreValEnqueueBarrier and ShenandoahStoreValReadBarrier"); - } log_info(gc, init)("Shenandoah heuristics: %s", _heuristics->name()); } else { @@ -790,7 +786,7 @@ assert(req.is_gc_alloc(), "Can only accept GC allocs here"); result = allocate_memory_under_lock(req, in_new_region); // Do not call handle_alloc_failure() here, because we cannot block. - // The allocation failure would be handled by the WB slowpath with handle_alloc_failure_evac(). + // The allocation failure would be handled by the LRB slowpath with handle_alloc_failure_evac(). } if (in_new_region) { @@ -1104,7 +1100,6 @@ ShenandoahParallelWorkerSession worker_session(worker_id); ShenandoahEvacOOMScope oom_evac_scope; ShenandoahEvacuateUpdateRootsClosure cl; - MarkingCodeBlobClosure blobsCl(&cl, CodeBlobToOopClosure::FixRelocations); _rp->process_evacuate_roots(&cl, &blobsCl, worker_id); } @@ -2061,14 +2056,12 @@ } oop ShenandoahHeap::pin_object(JavaThread* thr, oop o) { - o = ShenandoahBarrierSet::barrier_set()->write_barrier(o); ShenandoahHeapLocker locker(lock()); heap_region_containing(o)->make_pinned(); return o; } void ShenandoahHeap::unpin_object(JavaThread* thr, oop o) { - o = ShenandoahBarrierSet::barrier_set()->read_barrier(o); ShenandoahHeapLocker locker(lock()); heap_region_containing(o)->make_unpinned(); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp @@ -270,16 +270,16 @@ // public: enum GCStateBitPos { - // Heap has forwarded objects: need RB, ACMP, CAS barriers. + // Heap has forwarded objects: needs LRB barriers. HAS_FORWARDED_BITPOS = 0, // Heap is under marking: needs SATB barriers. MARKING_BITPOS = 1, - // Heap is under evacuation: needs WB barriers. (Set together with UNSTABLE) + // Heap is under evacuation: needs LRB barriers. (Set together with HAS_FORWARDED) EVACUATION_BITPOS = 2, - // Heap is under updating: needs SVRB/SVWB barriers. + // Heap is under updating: needs no additional barriers. UPDATEREFS_BITPOS = 3, // Heap is under traversal collection diff --git a/src/hotspot/share/gc/shenandoah/shenandoahMarkCompact.cpp b/src/hotspot/share/gc/shenandoah/shenandoahMarkCompact.cpp --- a/src/hotspot/share/gc/shenandoah/shenandoahMarkCompact.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahMarkCompact.cpp @@ -129,7 +129,7 @@ // Once marking is done, which may have fixed up forwarded objects, we can drop it. // Coming out of Full GC, we would not have any forwarded objects. - // This also prevents read barrier from kicking in while adjusting pointers in phase3. + // This also prevents resolves with fwdptr from kicking in while adjusting pointers in phase3. heap->set_has_forwarded_objects(false); heap->set_full_gc_move_in_progress(true); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahOopClosures.hpp b/src/hotspot/share/gc/shenandoah/shenandoahOopClosures.hpp --- a/src/hotspot/share/gc/shenandoah/shenandoahOopClosures.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahOopClosures.hpp @@ -34,7 +34,7 @@ enum UpdateRefsMode { NONE, // No reference updating - RESOLVE, // Only a read-barrier (no reference updating) + RESOLVE, // Only a resolve (no reference updating) SIMPLE, // Reference updating using simple store CONCURRENT // Reference updating using CAS }; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahRootProcessor.cpp b/src/hotspot/share/gc/shenandoah/shenandoahRootProcessor.cpp --- a/src/hotspot/share/gc/shenandoah/shenandoahRootProcessor.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahRootProcessor.cpp @@ -242,13 +242,21 @@ _evacuation_tasks(new SubTasksDone(SHENANDOAH_EVAC_NumElements)), _srs(n_workers), _phase(phase), - _coderoots_cset_iterator(ShenandoahCodeRoots::cset_iterator()) + _coderoots_cset_iterator(ShenandoahCodeRoots::cset_iterator()), + _par_state_string(StringTable::weak_storage()) + { heap->phase_timings()->record_workers_start(_phase); + if (ShenandoahStringDedup::is_enabled()) { + StringDedup::gc_prologue(false); + } } ShenandoahRootEvacuator::~ShenandoahRootEvacuator() { delete _evacuation_tasks; + if (ShenandoahStringDedup::is_enabled()) { + StringDedup::gc_epilogue(); + } ShenandoahHeap::heap()->phase_timings()->record_workers_end(_phase); } @@ -270,11 +278,38 @@ _coderoots_cset_iterator.possibly_parallel_blobs_do(blobs); } + if (ShenandoahStringDedup::is_enabled()) { + ShenandoahForwardedIsAliveClosure is_alive; + ShenandoahStringDedup::parallel_oops_do(&is_alive, oops, worker_id); + } + + if (_evacuation_tasks->try_claim_task(SHENANDOAH_EVAC_Universe_oops_do)) { + ShenandoahWorkerTimingsTracker timer(worker_times, ShenandoahPhaseTimings::UniverseRoots, worker_id); + Universe::oops_do(oops); + } + + if (_evacuation_tasks->try_claim_task(SHENANDOAH_EVAC_Management_oops_do)) { + ShenandoahWorkerTimingsTracker timer(worker_times, ShenandoahPhaseTimings::ManagementRoots, worker_id); + Management::oops_do(oops); + } + if (_evacuation_tasks->try_claim_task(SHENANDOAH_EVAC_jvmti_oops_do)) { + ShenandoahWorkerTimingsTracker timer(worker_times, ShenandoahPhaseTimings::JVMTIRoots, worker_id); + JvmtiExport::oops_do(oops); ShenandoahForwardedIsAliveClosure is_alive; - ShenandoahWorkerTimingsTracker timer(worker_times, ShenandoahPhaseTimings::JVMTIRoots, worker_id); JvmtiExport::weak_oops_do(&is_alive, oops); } + + if (_evacuation_tasks->try_claim_task(SHENANDOAH_EVAC_SystemDictionary_oops_do)) { + ShenandoahWorkerTimingsTracker timer(worker_times, ShenandoahPhaseTimings::SystemDictionaryRoots, worker_id); + SystemDictionary::oops_do(oops); + } + + if (_evacuation_tasks->try_claim_task(SHENANDOAH_EVAC_ObjectSynchronizer_oops_do)) { + ShenandoahWorkerTimingsTracker timer(worker_times, ShenandoahPhaseTimings::ObjectSynchronizerRoots, worker_id); + ObjectSynchronizer::oops_do(oops); + } + } uint ShenandoahRootEvacuator::n_workers() const { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahRootProcessor.hpp b/src/hotspot/share/gc/shenandoah/shenandoahRootProcessor.hpp --- a/src/hotspot/share/gc/shenandoah/shenandoahRootProcessor.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahRootProcessor.hpp @@ -58,7 +58,7 @@ StrongRootsScope _srs; OopStorage::ParState _par_state_string; ShenandoahPhaseTimings::Phase _phase; - ParallelCLDRootIterator _cld_iterator; + ParallelCLDRootIterator _cld_iterator; ShenandoahAllCodeRootsIterator _coderoots_all_iterator; CodeBlobClosure* _threads_nmethods_cl; WeakProcessorPhaseTimes _weak_processor_timings; @@ -120,11 +120,16 @@ StrongRootsScope _srs; ShenandoahPhaseTimings::Phase _phase; ShenandoahCsetCodeRootsIterator _coderoots_cset_iterator; + OopStorage::ParState _par_state_string; enum Shenandoah_evacuate_roots_tasks { - SHENANDOAH_EVAC_jvmti_oops_do, - // Leave this one last. - SHENANDOAH_EVAC_NumElements + SHENANDOAH_EVAC_Universe_oops_do, + SHENANDOAH_EVAC_ObjectSynchronizer_oops_do, + SHENANDOAH_EVAC_Management_oops_do, + SHENANDOAH_EVAC_SystemDictionary_oops_do, + SHENANDOAH_EVAC_jvmti_oops_do, + // Leave this one last. + SHENANDOAH_EVAC_NumElements }; public: ShenandoahRootEvacuator(ShenandoahHeap* heap, uint n_workers, diff --git a/src/hotspot/share/gc/shenandoah/shenandoahRuntime.cpp b/src/hotspot/share/gc/shenandoah/shenandoahRuntime.cpp --- a/src/hotspot/share/gc/shenandoah/shenandoahRuntime.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahRuntime.cpp @@ -55,8 +55,8 @@ ShenandoahThreadLocalData::satb_mark_queue(thread).enqueue_known_active(orig); JRT_END -JRT_LEAF(oopDesc*, ShenandoahRuntime::write_barrier_JRT(oopDesc* src)) - oop result = ShenandoahBarrierSet::barrier_set()->write_barrier_mutator(src); +JRT_LEAF(oopDesc*, ShenandoahRuntime::load_reference_barrier_JRT(oopDesc* src)) + oop result = ShenandoahBarrierSet::barrier_set()->load_reference_barrier_mutator(src); return (oopDesc*) result; JRT_END diff --git a/src/hotspot/share/gc/shenandoah/shenandoahRuntime.hpp b/src/hotspot/share/gc/shenandoah/shenandoahRuntime.hpp --- a/src/hotspot/share/gc/shenandoah/shenandoahRuntime.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahRuntime.hpp @@ -37,7 +37,7 @@ static void write_ref_array_post_entry(HeapWord* dst, size_t length); static void write_ref_field_pre_entry(oopDesc* orig, JavaThread* thread); - static oopDesc* write_barrier_JRT(oopDesc* src); + static oopDesc* load_reference_barrier_JRT(oopDesc* src); static void shenandoah_clone_barrier(oopDesc* obj); }; diff --git a/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp b/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp --- a/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp @@ -244,7 +244,7 @@ "Time is in microseconds.") \ \ experimental(uintx, ShenandoahEvacAssist, 10, \ - "How many objects to evacuate on WB assist path. " \ + "How many objects to evacuate on LRB assist path. " \ "Use zero to disable.") \ \ experimental(bool, ShenandoahPacing, true, \ @@ -352,27 +352,18 @@ diagnostic(bool, ShenandoahKeepAliveBarrier, true, \ "Turn on/off keep alive barriers in Shenandoah") \ \ - diagnostic(bool, ShenandoahWriteBarrier, true, \ - "Turn on/off write barriers in Shenandoah") \ - \ - diagnostic(bool, ShenandoahReadBarrier, true, \ - "Turn on/off read barriers in Shenandoah") \ - \ diagnostic(bool, ShenandoahStoreValEnqueueBarrier, false, \ "Turn on/off enqueuing of oops for storeval barriers") \ \ - diagnostic(bool, ShenandoahStoreValReadBarrier, true, \ - "Turn on/off store val read barriers in Shenandoah") \ - \ diagnostic(bool, ShenandoahCASBarrier, true, \ "Turn on/off CAS barriers in Shenandoah") \ \ - diagnostic(bool, ShenandoahAcmpBarrier, true, \ - "Turn on/off acmp barriers in Shenandoah") \ - \ diagnostic(bool, ShenandoahCloneBarrier, true, \ "Turn on/off clone barriers in Shenandoah") \ \ + diagnostic(bool, ShenandoahLoadRefBarrier, true, \ + "Turn on/off load-reference barriers in Shenandoah") \ + \ diagnostic(bool, ShenandoahStoreCheck, false, \ "Emit additional code that checks objects are written to only" \ " in to-space") \ @@ -401,20 +392,13 @@ "Turn it off for maximum compatibility with reflection or JNI " \ "code that manipulates final fields.") \ \ - diagnostic(bool, ShenandoahDecreaseRegisterPressure, false, \ - "Try to reuse after-barrier values to reduce register pressure") \ - \ experimental(bool, ShenandoahCommonGCStateLoads, false, \ "Enable commonming for GC state loads in generated code.") \ \ develop(bool, ShenandoahVerifyOptoBarriers, false, \ "Verify no missing barriers in C2") \ \ - experimental(bool, ShenandoahDontIncreaseWBFreq, true, \ - "Common 2 WriteBarriers or WriteBarrier and a ReadBarrier only " \ - "if the resulting WriteBarrier isn't executed more frequently") \ - \ experimental(bool, ShenandoahLoopOptsAfterExpansion, true, \ - "Attempt more loop opts after write barrier expansion") \ + "Attempt more loop opts after barrier expansion") \ #endif // SHARE_GC_SHENANDOAH_SHENANDOAH_GLOBALS_HPP diff --git a/src/hotspot/share/opto/classes.hpp b/src/hotspot/share/opto/classes.hpp --- a/src/hotspot/share/opto/classes.hpp +++ b/src/hotspot/share/opto/classes.hpp @@ -281,9 +281,7 @@ shmacro(ShenandoahWeakCompareAndSwapN) shmacro(ShenandoahWeakCompareAndSwapP) shmacro(ShenandoahEnqueueBarrier) -shmacro(ShenandoahReadBarrier) -shmacro(ShenandoahWriteBarrier) -shmacro(ShenandoahWBMemProj) +shmacro(ShenandoahLoadReferenceBarrier) macro(SCMemProj) macro(SqrtD) macro(SqrtF) diff --git a/src/hotspot/share/opto/compile.cpp b/src/hotspot/share/opto/compile.cpp --- a/src/hotspot/share/opto/compile.cpp +++ b/src/hotspot/share/opto/compile.cpp @@ -3070,7 +3070,7 @@ Node *m = wq.at(next); for (DUIterator_Fast imax, i = m->fast_outs(imax); i < imax; i++) { Node* use = m->fast_out(i); - if (use->is_Mem() || use->is_EncodeNarrowPtr() || use->is_ShenandoahBarrier()) { + if (use->is_Mem() || use->is_EncodeNarrowPtr()) { use->ensure_control_or_add_prec(n->in(0)); } else { switch(use->Opcode()) { diff --git a/src/hotspot/share/opto/lcm.cpp b/src/hotspot/share/opto/lcm.cpp --- a/src/hotspot/share/opto/lcm.cpp +++ b/src/hotspot/share/opto/lcm.cpp @@ -178,7 +178,6 @@ case Op_LoadRange: case Op_LoadD_unaligned: case Op_LoadL_unaligned: - case Op_ShenandoahReadBarrier: assert(mach->in(2) == val, "should be address"); break; case Op_StoreB: diff --git a/src/hotspot/share/opto/library_call.cpp b/src/hotspot/share/opto/library_call.cpp --- a/src/hotspot/share/opto/library_call.cpp +++ b/src/hotspot/share/opto/library_call.cpp @@ -4485,7 +4485,7 @@ for (MergeMemStream mms(merged_memory(), mem->as_MergeMem()); mms.next_non_empty2(); ) { Node* n = mms.memory(); if (n != mms.memory2() && !(n->is_Proj() && n->in(0) == alloc->initialization())) { - assert(n->is_Store() || n->Opcode() == Op_ShenandoahWBMemProj, "what else?"); + assert(n->is_Store(), "what else?"); no_interfering_store = false; break; } @@ -4494,7 +4494,7 @@ for (MergeMemStream mms(merged_memory()); mms.next_non_empty(); ) { Node* n = mms.memory(); if (n != mem && !(n->is_Proj() && n->in(0) == alloc->initialization())) { - assert(n->is_Store() || n->Opcode() == Op_ShenandoahWBMemProj, "what else?"); + assert(n->is_Store(), "what else?"); no_interfering_store = false; break; } diff --git a/src/hotspot/share/opto/loopPredicate.cpp b/src/hotspot/share/opto/loopPredicate.cpp --- a/src/hotspot/share/opto/loopPredicate.cpp +++ b/src/hotspot/share/opto/loopPredicate.cpp @@ -536,9 +536,6 @@ if (_lpt->is_invariant(n)) { // known invariant _invariant.set(n->_idx); } else if (!n->is_CFG()) { - if (n->Opcode() == Op_ShenandoahWriteBarrier) { - return; - } Node *n_ctrl = _phase->ctrl_or_self(n); Node *u_ctrl = _phase->ctrl_or_self(use); // self if use is a CFG if (_phase->is_dominator(n_ctrl, u_ctrl)) { diff --git a/src/hotspot/share/opto/loopnode.cpp b/src/hotspot/share/opto/loopnode.cpp --- a/src/hotspot/share/opto/loopnode.cpp +++ b/src/hotspot/share/opto/loopnode.cpp @@ -3971,8 +3971,8 @@ } while(worklist.size() != 0 && LCA != early) { Node* s = worklist.pop(); - if (s->is_Load() || s->is_ShenandoahBarrier() || s->Opcode() == Op_SafePoint || - (s->is_CallStaticJava() && s->as_CallStaticJava()->uncommon_trap_request() != 0)) { + if (s->is_Load() || s->Opcode() == Op_SafePoint || + (s->is_CallStaticJava() && s->as_CallStaticJava()->uncommon_trap_request() != 0) || s->is_Phi()) { continue; } else if (s->is_MergeMem()) { for (DUIterator_Fast imax, i = s->fast_outs(imax); i < imax; i++) { diff --git a/src/hotspot/share/opto/loopnode.hpp b/src/hotspot/share/opto/loopnode.hpp --- a/src/hotspot/share/opto/loopnode.hpp +++ b/src/hotspot/share/opto/loopnode.hpp @@ -38,8 +38,6 @@ class LoopNode; class Node; class OuterStripMinedLoopEndNode; -class ShenandoahBarrierNode; -class ShenandoahWriteBarrierNode; class PathFrequency; class PhaseIdealLoop; class CountedLoopReserveKit; @@ -638,8 +636,7 @@ friend class IdealLoopTree; friend class SuperWord; friend class CountedLoopReserveKit; - friend class ShenandoahBarrierNode; - friend class ShenandoahWriteBarrierNode; + friend class ShenandoahBarrierC2Support; // Pre-computed def-use info PhaseIterGVN &_igvn; diff --git a/src/hotspot/share/opto/loopopts.cpp b/src/hotspot/share/opto/loopopts.cpp --- a/src/hotspot/share/opto/loopopts.cpp +++ b/src/hotspot/share/opto/loopopts.cpp @@ -1082,11 +1082,6 @@ Node* m = n->fast_out(j); if (m->is_FastLock()) return false; -#if INCLUDE_SHENANDOAHGC - if (m->is_ShenandoahBarrier() && m->has_out_with(Op_FastLock)) { - return false; - } -#endif #ifdef _LP64 if (m->Opcode() == Op_ConvI2L) return false; @@ -3210,7 +3205,7 @@ // if not pinned and not a load (which maybe anti-dependent on a store) // and not a CMove (Matcher expects only bool->cmove). - if (n->in(0) == NULL && !n->is_Load() && !n->is_CMove() && n->Opcode() != Op_ShenandoahWBMemProj) { + if (n->in(0) == NULL && !n->is_Load() && !n->is_CMove()) { cloned_for_outside_use += clone_for_use_outside_loop( loop, n, worklist ); sink_list.push(n); peel >>= n->_idx; // delete n from peel set. diff --git a/src/hotspot/share/opto/node.hpp b/src/hotspot/share/opto/node.hpp --- a/src/hotspot/share/opto/node.hpp +++ b/src/hotspot/share/opto/node.hpp @@ -142,7 +142,6 @@ class RootNode; class SafePointNode; class SafePointScalarObjectNode; -class ShenandoahBarrierNode; class StartNode; class State; class StoreNode; @@ -676,7 +675,6 @@ DEFINE_CLASS_ID(EncodeNarrowPtr, Type, 6) DEFINE_CLASS_ID(EncodeP, EncodeNarrowPtr, 0) DEFINE_CLASS_ID(EncodePKlass, EncodeNarrowPtr, 1) - DEFINE_CLASS_ID(ShenandoahBarrier, Type, 7) DEFINE_CLASS_ID(Proj, Node, 3) DEFINE_CLASS_ID(CatchProj, Proj, 0) @@ -875,7 +873,6 @@ DEFINE_CLASS_QUERY(Root) DEFINE_CLASS_QUERY(SafePoint) DEFINE_CLASS_QUERY(SafePointScalarObject) - DEFINE_CLASS_QUERY(ShenandoahBarrier) DEFINE_CLASS_QUERY(Start) DEFINE_CLASS_QUERY(Store) DEFINE_CLASS_QUERY(Sub) diff --git a/test/hotspot/jtreg/gc/shenandoah/options/TestSelectiveBarrierFlags.java b/test/hotspot/jtreg/gc/shenandoah/options/TestSelectiveBarrierFlags.java --- a/test/hotspot/jtreg/gc/shenandoah/options/TestSelectiveBarrierFlags.java +++ b/test/hotspot/jtreg/gc/shenandoah/options/TestSelectiveBarrierFlags.java @@ -43,13 +43,9 @@ public static void main(String[] args) throws Exception { String[][] opts = { new String[] { "ShenandoahKeepAliveBarrier" }, - new String[] { "ShenandoahWriteBarrier" }, - new String[] { "ShenandoahReadBarrier" }, - // StoreValRead+SATB are actually compatible, but we need to protect against - // StorveValEnqueue+SATB. TODO: Make it better. - new String[] { "ShenandoahSATBBarrier", "ShenandoahStoreValReadBarrier", "ShenandoahStoreValEnqueueBarrier" }, + new String[] { "ShenandoahLoadRefBarrier" }, + new String[] { "ShenandoahSATBBarrier", "ShenandoahStoreValEnqueueBarrier" }, new String[] { "ShenandoahCASBarrier" }, - new String[] { "ShenandoahAcmpBarrier" }, new String[] { "ShenandoahCloneBarrier" }, }; diff --git a/test/hotspot/jtreg/gc/shenandoah/options/TestWrongBarrierDisable.java b/test/hotspot/jtreg/gc/shenandoah/options/TestWrongBarrierDisable.java --- a/test/hotspot/jtreg/gc/shenandoah/options/TestWrongBarrierDisable.java +++ b/test/hotspot/jtreg/gc/shenandoah/options/TestWrongBarrierDisable.java @@ -38,21 +38,16 @@ public static void main(String[] args) throws Exception { String[] concurrent = { - "ShenandoahReadBarrier", - "ShenandoahWriteBarrier", + "ShenandoahLoadRefBarrier", "ShenandoahCASBarrier", - "ShenandoahAcmpBarrier", "ShenandoahCloneBarrier", "ShenandoahSATBBarrier", "ShenandoahKeepAliveBarrier", - "ShenandoahStoreValReadBarrier", }; String[] traversal = { - "ShenandoahReadBarrier", - "ShenandoahWriteBarrier", + "ShenandoahLoadRefBarrier", "ShenandoahCASBarrier", - "ShenandoahAcmpBarrier", "ShenandoahCloneBarrier", };