# HG changeset patch # User rkennke # Date 1547501769 -3600 # Mon Jan 14 22:36:09 2019 +0100 # Node ID 8622eaf192dc49b4a81375077771fc9923b7078b # Parent 5328dd5d2be89a4a4d7587a7b2a1ce5053efc71a 8217016: Shenandoah: Streamline generation of CAS barriers diff --git a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp --- a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp +++ b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp @@ -592,64 +592,19 @@ #ifndef _LP64 void ShenandoahBarrierSetAssembler::cmpxchg_oop(MacroAssembler* masm, Register res, Address addr, Register oldval, Register newval, - bool exchange, bool encode, Register tmp1, Register tmp2) { + bool exchange, Register tmp1, Register tmp2) { // Shenandoah has no 32-bit version for this. Unimplemented(); } #else void ShenandoahBarrierSetAssembler::cmpxchg_oop(MacroAssembler* masm, Register res, Address addr, Register oldval, Register newval, - bool exchange, bool encode, Register tmp1, Register tmp2) { - if (!ShenandoahCASBarrier) { -#ifdef _LP64 - if (UseCompressedOops) { - if (encode) { - __ encode_heap_oop(oldval); - __ mov(rscratch1, newval); - __ encode_heap_oop(rscratch1); - newval = rscratch1; - } - if (os::is_MP()) { - __ lock(); - } - // oldval (rax) is implicitly used by this instruction - __ cmpxchgl(newval, addr); - } else -#endif - { - if (os::is_MP()) { - __ lock(); - } - __ cmpxchgptr(newval, addr); - } - - if (!exchange) { - assert(res != NULL, "need result register"); - __ setb(Assembler::equal, res); - __ movzbl(res, res); - } - return; - } - + bool exchange, Register tmp1, Register tmp2) { assert(ShenandoahCASBarrier, "Should only be used when CAS barrier is enabled"); assert(oldval == rax, "must be in rax for implicit use in cmpxchg"); Label retry, done; - // Apply storeval barrier to newval. - if (encode) { - storeval_barrier(masm, newval, tmp1); - } - - if (UseCompressedOops) { - if (encode) { - __ encode_heap_oop(oldval); - __ mov(rscratch1, newval); - __ encode_heap_oop(rscratch1); - newval = rscratch1; - } - } - // Remember oldval for retry logic below if (UseCompressedOops) { __ movl(tmp1, oldval); diff --git a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.hpp b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.hpp --- a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.hpp +++ b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.hpp @@ -64,7 +64,6 @@ void write_barrier(MacroAssembler* masm, Register dst); void write_barrier_impl(MacroAssembler* masm, Register dst); - void storeval_barrier(MacroAssembler* masm, Register dst, Register tmp); void storeval_barrier_impl(MacroAssembler* masm, Register dst, Register tmp); address generate_shenandoah_wb(StubCodeGenerator* cgen); @@ -75,6 +74,7 @@ public: static address shenandoah_wb(); + void storeval_barrier(MacroAssembler* masm, Register dst, Register tmp); #ifdef COMPILER1 void gen_pre_barrier_stub(LIR_Assembler* ce, ShenandoahPreBarrierStub* stub); void gen_write_barrier_stub(LIR_Assembler* ce, ShenandoahWriteBarrierStub* stub); @@ -83,7 +83,7 @@ void cmpxchg_oop(MacroAssembler* masm, Register res, Address addr, Register oldval, Register newval, - bool exchange, bool encode, Register tmp1, Register tmp2); + bool exchange, Register tmp1, Register tmp2); virtual void arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Register src, Register dst, Register count); virtual void arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, BasicType type, diff --git a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetC1_x86.cpp b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetC1_x86.cpp --- a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetC1_x86.cpp +++ b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetC1_x86.cpp @@ -28,20 +28,36 @@ #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp" #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp" +#define __ masm->masm()-> + void LIR_OpShenandoahCompareAndSwap::emit_code(LIR_Assembler* masm) { Register addr = _addr->as_register_lo(); Register newval = _new_value->as_register(); Register cmpval = _cmp_value->as_register(); Register tmp1 = _tmp1->as_register(); Register tmp2 = _tmp2->as_register(); + Register result = result_opr()->as_register(); assert(cmpval == rax, "wrong register"); assert(newval != NULL, "new val must be register"); assert(cmpval != newval, "cmp and new values must be in different registers"); assert(cmpval != addr, "cmp and addr must be in different registers"); assert(newval != addr, "new value and addr must be in different registers"); - ShenandoahBarrierSet::assembler()->cmpxchg_oop(masm->masm(), NULL, Address(addr, 0), cmpval, newval, true, true, tmp1, tmp2); + + // Apply storeval barrier to newval. + ShenandoahBarrierSet::assembler()->storeval_barrier(masm->masm(), newval, tmp1); + + if (UseCompressedOops) { + __ encode_heap_oop(cmpval); + __ mov(rscratch1, newval); + __ encode_heap_oop(rscratch1); + newval = rscratch1; + } + + ShenandoahBarrierSet::assembler()->cmpxchg_oop(masm->masm(), result, Address(addr, 0), cmpval, newval, false, tmp1, tmp2); } +#undef __ + #ifdef ASSERT #define __ gen->lir(__FILE__, __LINE__)-> #else @@ -63,12 +79,9 @@ LIR_Opr t1 = gen->new_register(T_OBJECT); LIR_Opr t2 = gen->new_register(T_OBJECT); LIR_Opr addr = access.resolved_addr()->as_address_ptr()->base(); - - __ append(new LIR_OpShenandoahCompareAndSwap(addr, cmp_value.result(), new_value.result(), t1, t2, LIR_OprFact::illegalOpr)); + LIR_Opr result = gen->new_register(T_INT); - LIR_Opr result = gen->new_register(T_INT); - __ cmove(lir_cond_equal, LIR_OprFact::intConst(1), LIR_OprFact::intConst(0), - result, T_INT); + __ append(new LIR_OpShenandoahCompareAndSwap(addr, cmp_value.result(), new_value.result(), t1, t2, result)); return result; } } diff --git a/src/hotspot/cpu/x86/gc/shenandoah/shenandoah_x86_64.ad b/src/hotspot/cpu/x86/gc/shenandoah/shenandoah_x86_64.ad --- a/src/hotspot/cpu/x86/gc/shenandoah/shenandoah_x86_64.ad +++ b/src/hotspot/cpu/x86/gc/shenandoah/shenandoah_x86_64.ad @@ -83,7 +83,7 @@ ShenandoahBarrierSet::assembler()->cmpxchg_oop(&_masm, $res$$Register, $mem_ptr$$Address, $oldval$$Register, $newval$$Register, false, // swap - false, $tmp1$$Register, $tmp2$$Register + $tmp1$$Register, $tmp2$$Register ); %} ins_pipe( pipe_cmpxchg ); @@ -104,7 +104,7 @@ ShenandoahBarrierSet::assembler()->cmpxchg_oop(&_masm, $res$$Register, $mem_ptr$$Address, $oldval$$Register, $newval$$Register, false, // swap - false, $tmp1$$Register, $tmp2$$Register + $tmp1$$Register, $tmp2$$Register ); %} ins_pipe( pipe_cmpxchg ); @@ -123,7 +123,7 @@ ShenandoahBarrierSet::assembler()->cmpxchg_oop(&_masm, NULL, $mem_ptr$$Address, $oldval$$Register, $newval$$Register, true, // exchange - false, $tmp1$$Register, $tmp2$$Register + $tmp1$$Register, $tmp2$$Register ); %} ins_pipe( pipe_cmpxchg ); @@ -145,7 +145,7 @@ ShenandoahBarrierSet::assembler()->cmpxchg_oop(&_masm, NULL, $mem_ptr$$Address, $oldval$$Register, $newval$$Register, true, // exchange - false, $tmp1$$Register, $tmp2$$Register + $tmp1$$Register, $tmp2$$Register ); %} ins_pipe( pipe_cmpxchg ); diff --git a/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp b/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp --- a/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp +++ b/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp @@ -775,11 +775,19 @@ if (adr->bottom_type()->is_ptr_to_narrowoop()) { Node *newval_enc = kit->gvn().transform(new EncodePNode(new_val, new_val->bottom_type()->make_narrowoop())); Node *oldval_enc = kit->gvn().transform(new EncodePNode(expected_val, expected_val->bottom_type()->make_narrowoop())); - load_store = kit->gvn().transform(new ShenandoahCompareAndExchangeNNode(kit->control(), mem, adr, newval_enc, oldval_enc, adr_type, value_type->make_narrowoop(), mo)); + if (ShenandoahCASBarrier) { + load_store = kit->gvn().transform(new ShenandoahCompareAndExchangeNNode(kit->control(), mem, adr, newval_enc, oldval_enc, adr_type, value_type->make_narrowoop(), mo)); + } else { + load_store = kit->gvn().transform(new CompareAndExchangeNNode(kit->control(), mem, adr, newval_enc, oldval_enc, adr_type, value_type->make_narrowoop(), mo)); + } } else #endif { - load_store = kit->gvn().transform(new ShenandoahCompareAndExchangePNode(kit->control(), mem, adr, new_val, expected_val, adr_type, value_type->is_oopptr(), mo)); + if (ShenandoahCASBarrier) { + load_store = kit->gvn().transform(new ShenandoahCompareAndExchangePNode(kit->control(), mem, adr, new_val, expected_val, adr_type, value_type->is_oopptr(), mo)); + } else { + load_store = kit->gvn().transform(new CompareAndExchangePNode(kit->control(), mem, adr, new_val, expected_val, adr_type, value_type->is_oopptr(), mo)); + } } access.set_raw_access(load_store); @@ -813,18 +821,34 @@ if (adr->bottom_type()->is_ptr_to_narrowoop()) { Node *newval_enc = kit->gvn().transform(new EncodePNode(new_val, new_val->bottom_type()->make_narrowoop())); Node *oldval_enc = kit->gvn().transform(new EncodePNode(expected_val, expected_val->bottom_type()->make_narrowoop())); - if (is_weak_cas) { - load_store = kit->gvn().transform(new ShenandoahWeakCompareAndSwapNNode(kit->control(), mem, adr, newval_enc, oldval_enc, mo)); + if (ShenandoahCASBarrier) { + if (is_weak_cas) { + load_store = kit->gvn().transform(new ShenandoahWeakCompareAndSwapNNode(kit->control(), mem, adr, newval_enc, oldval_enc, mo)); + } else { + load_store = kit->gvn().transform(new ShenandoahCompareAndSwapNNode(kit->control(), mem, adr, newval_enc, oldval_enc, mo)); + } } else { - load_store = kit->gvn().transform(new ShenandoahCompareAndSwapNNode(kit->control(), mem, adr, newval_enc, oldval_enc, mo)); + if (is_weak_cas) { + load_store = kit->gvn().transform(new WeakCompareAndSwapNNode(kit->control(), mem, adr, newval_enc, oldval_enc, mo)); + } else { + load_store = kit->gvn().transform(new CompareAndSwapNNode(kit->control(), mem, adr, newval_enc, oldval_enc, mo)); + } } } else #endif { - if (is_weak_cas) { - load_store = kit->gvn().transform(new ShenandoahWeakCompareAndSwapPNode(kit->control(), mem, adr, new_val, expected_val, mo)); + if (ShenandoahCASBarrier) { + if (is_weak_cas) { + load_store = kit->gvn().transform(new ShenandoahWeakCompareAndSwapPNode(kit->control(), mem, adr, new_val, expected_val, mo)); + } else { + load_store = kit->gvn().transform(new ShenandoahCompareAndSwapPNode(kit->control(), mem, adr, new_val, expected_val, mo)); + } } else { - load_store = kit->gvn().transform(new ShenandoahCompareAndSwapPNode(kit->control(), mem, adr, new_val, expected_val, mo)); + if (is_weak_cas) { + load_store = kit->gvn().transform(new WeakCompareAndSwapPNode(kit->control(), mem, adr, new_val, expected_val, mo)); + } else { + load_store = kit->gvn().transform(new CompareAndSwapPNode(kit->control(), mem, adr, new_val, expected_val, mo)); + } } } access.set_raw_access(load_store);