# HG changeset patch # Parent 04dcc65c9d58ceffc481d224949a45beb9b75c3b 8217016: Shenandoah: Streamline generation of CAS barriers diff -r 04dcc65c9d58 src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp --- a/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp Fri Jan 25 18:13:25 2019 +0000 +++ b/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp Fri Jan 25 19:20:15 2019 -0500 @@ -423,45 +423,14 @@ } void ShenandoahBarrierSetAssembler::cmpxchg_oop(MacroAssembler* masm, Register addr, Register expected, Register new_val, - bool acquire, bool release, bool weak, bool encode, - Register tmp1, Register tmp2, Register tmp3, + bool acquire, bool release, bool weak, bool is_cae, Register result) { - if (!ShenandoahCASBarrier) { - if (UseCompressedOops) { - if (encode) { - __ encode_heap_oop(tmp1, expected); - expected = tmp1; - __ encode_heap_oop(tmp3, new_val); - new_val = tmp3; - } - __ cmpxchg(addr, expected, new_val, Assembler::word, /* acquire*/ true, /* release*/ true, /* weak*/ false, rscratch1); - __ membar(__ AnyAny); - } else { - __ cmpxchg(addr, expected, new_val, Assembler::xword, /* acquire*/ true, /* release*/ true, /* weak*/ false, rscratch1); - __ membar(__ AnyAny); - } - return; - } - - if (encode) { - storeval_barrier(masm, new_val, tmp3); - } - - if (UseCompressedOops) { - if (encode) { - __ encode_heap_oop(tmp1, expected); - expected = tmp1; - __ encode_heap_oop(tmp2, new_val); - new_val = tmp2; - } - } - bool is_cae = (result != noreg); + Register tmp = rscratch2; bool is_narrow = UseCompressedOops; Assembler::operand_size size = is_narrow ? Assembler::word : Assembler::xword; - if (! is_cae) result = rscratch1; - assert_different_registers(addr, expected, new_val, result, tmp3); + assert_different_registers(addr, expected, new_val, result, tmp); Label retry, done, fail; @@ -474,35 +443,38 @@ __ cmp(result, expected); } __ br(Assembler::NE, fail); - __ store_exclusive(tmp3, new_val, addr, size, release); + __ store_exclusive(tmp, new_val, addr, size, release); if (weak) { - __ cmpw(tmp3, 0u); // If the store fails, return NE to our caller + __ cmpw(tmp, 0u); // If the store fails, return NE to our caller } else { - __ cbnzw(tmp3, retry); + __ cbnzw(tmp, retry); } __ b(done); __ bind(fail); // Check if rb(expected)==rb(result) // Shuffle registers so that we have memory value ready for next expected. - __ mov(tmp3, expected); + __ mov(tmp, expected); __ mov(expected, result); if (is_narrow) { __ decode_heap_oop(result, result); - __ decode_heap_oop(tmp3, tmp3); + __ decode_heap_oop(tmp, tmp); } read_barrier_impl(masm, result); - read_barrier_impl(masm, tmp3); - __ cmp(result, tmp3); + read_barrier_impl(masm, tmp); + __ cmp(result, tmp); // Retry with expected now being the value we just loaded from addr. __ br(Assembler::EQ, retry); - if (is_narrow && is_cae) { + if (is_cae && is_narrow) { // For cmp-and-exchange and narrow oops, we need to restore // the compressed old-value. We moved it to 'expected' a few lines up. __ mov(result, expected); } __ bind(done); + if (!is_cae) { + __ cset(result, Assembler::EQ); + } } #ifdef COMPILER1 diff -r 04dcc65c9d58 src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.hpp --- a/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.hpp Fri Jan 25 18:13:25 2019 +0000 +++ b/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.hpp Fri Jan 25 19:20:15 2019 -0500 @@ -60,7 +60,6 @@ void read_barrier_not_null_impl(MacroAssembler* masm, Register dst); void write_barrier(MacroAssembler* masm, Register dst); void write_barrier_impl(MacroAssembler* masm, Register dst); - void storeval_barrier(MacroAssembler* masm, Register dst, Register tmp); void asm_acmp_barrier(MacroAssembler* masm, Register op1, Register op2); address generate_shenandoah_wb(StubCodeGenerator* cgen); @@ -68,6 +67,8 @@ public: static address shenandoah_wb(); + void storeval_barrier(MacroAssembler* masm, Register dst, Register tmp); + #ifdef COMPILER1 void gen_pre_barrier_stub(LIR_Assembler* ce, ShenandoahPreBarrierStub* stub); void gen_write_barrier_stub(LIR_Assembler* ce, ShenandoahWriteBarrierStub* stub); @@ -92,9 +93,7 @@ Label& slow_case); void cmpxchg_oop(MacroAssembler* masm, Register addr, Register expected, Register new_val, - bool acquire, bool release, bool weak, bool encode, - Register tmp1, Register tmp2, Register tmp3 = rscratch2, - Register result = noreg); + bool acquire, bool release, bool weak, bool is_cae, Register result); virtual void barrier_stubs_init(); }; diff -r 04dcc65c9d58 src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetC1_aarch64.cpp --- a/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetC1_aarch64.cpp Fri Jan 25 18:13:25 2019 +0000 +++ b/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetC1_aarch64.cpp Fri Jan 25 19:20:15 2019 -0500 @@ -28,15 +28,30 @@ #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp" #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp" +#define __ masm->masm()-> + void LIR_OpShenandoahCompareAndSwap::emit_code(LIR_Assembler* masm) { Register addr = _addr->as_register_lo(); Register newval = _new_value->as_register(); Register cmpval = _cmp_value->as_register(); Register tmp1 = _tmp1->as_register(); Register tmp2 = _tmp2->as_register(); - ShenandoahBarrierSet::assembler()->cmpxchg_oop(masm->masm(), addr, cmpval, newval, /*acquire*/ false, /*release*/ true, /*weak*/ false, true, tmp1, tmp2); + Register result = result_opr()->as_register(); + + ShenandoahBarrierSet::assembler()->storeval_barrier(masm->masm(), newval, rscratch2); + + if (UseCompressedOops) { + __ encode_heap_oop(tmp1, cmpval); + cmpval = tmp1; + __ encode_heap_oop(tmp2, newval); + newval = tmp2; + } + + ShenandoahBarrierSet::assembler()->cmpxchg_oop(masm->masm(), addr, cmpval, newval, /*acquire*/ false, /*release*/ true, /*weak*/ false, /*is_cae*/ false, result); } +#undef __ + #ifdef ASSERT #define __ gen->lir(__FILE__, __LINE__)-> #else @@ -58,13 +73,9 @@ LIR_Opr t1 = gen->new_register(T_OBJECT); LIR_Opr t2 = gen->new_register(T_OBJECT); LIR_Opr addr = access.resolved_addr()->as_address_ptr()->base(); + LIR_Opr result = gen->new_register(T_INT); - __ append(new LIR_OpShenandoahCompareAndSwap(addr, cmp_value.result(), new_value.result(), t1, t2, - LIR_OprFact::illegalOpr)); - - LIR_Opr result = gen->new_register(T_INT); - __ cmove(lir_cond_equal, LIR_OprFact::intConst(1), LIR_OprFact::intConst(0), - result, T_INT); + __ append(new LIR_OpShenandoahCompareAndSwap(addr, cmp_value.result(), new_value.result(), t1, t2, result)); return result; } } diff -r 04dcc65c9d58 src/hotspot/cpu/aarch64/gc/shenandoah/shenandoah_aarch64.ad --- a/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoah_aarch64.ad Fri Jan 25 18:13:25 2019 +0000 +++ b/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoah_aarch64.ad Fri Jan 25 19:20:15 2019 -0500 @@ -26,22 +26,22 @@ %} encode %{ - enc_class aarch64_enc_cmpxchg_oop_shenandoah(memory mem, iRegP oldval, iRegP newval, iRegPNoSp tmp) %{ + enc_class aarch64_enc_cmpxchg_oop_shenandoah(memory mem, iRegP oldval, iRegP newval, iRegPNoSp tmp, iRegINoSp res) %{ MacroAssembler _masm(&cbuf); guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding"); Register tmp = $tmp$$Register; __ mov(tmp, $oldval$$Register); // Must not clobber oldval. ShenandoahBarrierSet::assembler()->cmpxchg_oop(&_masm, $mem$$Register, tmp, $newval$$Register, - /*acquire*/ false, /*release*/ true, /*weak*/ false, /*encode*/ false, noreg, noreg); + /*acquire*/ false, /*release*/ true, /*weak*/ false, /*is_cae*/ false, $res$$Register); %} - enc_class aarch64_enc_cmpxchg_acq_oop_shenandoah(memory mem, iRegP oldval, iRegP newval, iRegPNoSp tmp) %{ + enc_class aarch64_enc_cmpxchg_acq_oop_shenandoah(memory mem, iRegP oldval, iRegP newval, iRegPNoSp tmp, iRegINoSp res) %{ MacroAssembler _masm(&cbuf); guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding"); Register tmp = $tmp$$Register; __ mov(tmp, $oldval$$Register); // Must not clobber oldval. ShenandoahBarrierSet::assembler()->cmpxchg_oop(&_masm, $mem$$Register, tmp, $newval$$Register, - /*acquire*/ true, /*release*/ true, /*weak*/ false, /*encode*/ false, noreg, noreg); + /*acquire*/ true, /*release*/ true, /*weak*/ false, /*is_cae*/ false, $res$$Register); %} %} @@ -66,11 +66,9 @@ format %{ "cmpxchg_shenandoah $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval with temp $tmp" - "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)" %} - ins_encode(aarch64_enc_cmpxchg_oop_shenandoah(mem, oldval, newval, tmp), - aarch64_enc_cset_eq(res)); + ins_encode(aarch64_enc_cmpxchg_oop_shenandoah(mem, oldval, newval, tmp, res)); ins_pipe(pipe_slow); %} @@ -84,14 +82,12 @@ format %{ "cmpxchgw_shenandoah $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval with temp $tmp" - "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)" %} ins_encode %{ Register tmp = $tmp$$Register; __ mov(tmp, $oldval$$Register); // Must not clobber oldval. - ShenandoahBarrierSet::assembler()->cmpxchg_oop(&_masm, $mem$$Register, tmp, $newval$$Register, /*acquire*/ false, /*release*/ true, /*weak*/ false, /*encode*/ false, noreg, noreg); - __ cset($res$$Register, Assembler::EQ); + ShenandoahBarrierSet::assembler()->cmpxchg_oop(&_masm, $mem$$Register, tmp, $newval$$Register, /*acquire*/ false, /*release*/ true, /*weak*/ false, /*is_cae*/ false, $res$$Register); %} ins_pipe(pipe_slow); @@ -107,11 +103,9 @@ format %{ "cmpxchg_acq_shenandoah_oop $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval with temp $tmp" - "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)" %} - ins_encode(aarch64_enc_cmpxchg_acq_oop_shenandoah(mem, oldval, newval, tmp), - aarch64_enc_cset_eq(res)); + ins_encode(aarch64_enc_cmpxchg_acq_oop_shenandoah(mem, oldval, newval, tmp, res)); ins_pipe(pipe_slow); %} @@ -126,14 +120,12 @@ format %{ "cmpxchgw_acq_shenandoah_narrow_oop $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval with temp $tmp" - "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)" %} ins_encode %{ Register tmp = $tmp$$Register; __ mov(tmp, $oldval$$Register); // Must not clobber oldval. - ShenandoahBarrierSet::assembler()->cmpxchg_oop(&_masm, $mem$$Register, tmp, $newval$$Register, /*acquire*/ true, /*release*/ true, /*weak*/ false, /*encode*/ false, noreg, noreg); - __ cset($res$$Register, Assembler::EQ); + ShenandoahBarrierSet::assembler()->cmpxchg_oop(&_masm, $mem$$Register, tmp, $newval$$Register, /*acquire*/ true, /*release*/ true, /*weak*/ false, /*is_cae*/ false, $res$$Register); %} ins_pipe(pipe_slow); @@ -150,7 +142,7 @@ Register tmp = $tmp$$Register; __ mov(tmp, $oldval$$Register); // Must not clobber oldval. ShenandoahBarrierSet::assembler()->cmpxchg_oop(&_masm, $mem$$Register, tmp, $newval$$Register, - /*acquire*/ false, /*release*/ true, /*weak*/ false, /* encode*/ false, noreg, noreg, rscratch2, $res$$Register); + /*acquire*/ false, /*release*/ true, /*weak*/ false, /*is_cae*/ true, $res$$Register); %} ins_pipe(pipe_slow); %} @@ -166,7 +158,7 @@ Register tmp = $tmp$$Register; __ mov(tmp, $oldval$$Register); // Must not clobber oldval. ShenandoahBarrierSet::assembler()->cmpxchg_oop(&_masm, $mem$$Register, tmp, $newval$$Register, - /*acquire*/ false, /*release*/ true, /*weak*/ false, /*encode*/ false, noreg, noreg, rscratch2, $res$$Register); + /*acquire*/ false, /*release*/ true, /*weak*/ false, /*is_cae*/ true, $res$$Register); %} ins_pipe(pipe_slow); %} @@ -183,7 +175,7 @@ Register tmp = $tmp$$Register; __ mov(tmp, $oldval$$Register); // Must not clobber oldval. ShenandoahBarrierSet::assembler()->cmpxchg_oop(&_masm, $mem$$Register, tmp, $newval$$Register, - /*acquire*/ true, /*release*/ true, /*weak*/ false, /* encode*/ false, noreg, noreg, rscratch2, $res$$Register); + /*acquire*/ true, /*release*/ true, /*weak*/ false, /*is_cae*/ true, $res$$Register); %} ins_pipe(pipe_slow); %} @@ -200,7 +192,7 @@ Register tmp = $tmp$$Register; __ mov(tmp, $oldval$$Register); // Must not clobber oldval. ShenandoahBarrierSet::assembler()->cmpxchg_oop(&_masm, $mem$$Register, tmp, $newval$$Register, - /*acquire*/ true, /*release*/ true, /*weak*/ false, /*encode*/ false, noreg, noreg, rscratch2, $res$$Register); + /*acquire*/ true, /*release*/ true, /*weak*/ false, /*is_cae*/ true, $res$$Register); %} ins_pipe(pipe_slow); %} @@ -217,8 +209,7 @@ Register tmp = $tmp$$Register; __ mov(tmp, $oldval$$Register); // Must not clobber oldval. ShenandoahBarrierSet::assembler()->cmpxchg_oop(&_masm, $mem$$Register, tmp, $newval$$Register, - /*acquire*/ false, /*release*/ true, /*weak*/ true, /*encode*/ false, noreg, noreg); - __ csetw($res$$Register, Assembler::EQ); + /*acquire*/ false, /*release*/ true, /*weak*/ true, /*is_cae*/ false, $res$$Register); %} ins_pipe(pipe_slow); %} @@ -229,14 +220,12 @@ effect(TEMP tmp, KILL cr); format %{ "cmpxchg_shenandoah $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval" - "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)" %} ins_encode %{ Register tmp = $tmp$$Register; __ mov(tmp, $oldval$$Register); // Must not clobber oldval. ShenandoahBarrierSet::assembler()->cmpxchg_oop(&_masm, $mem$$Register, tmp, $newval$$Register, - /*acquire*/ false, /*release*/ true, /*weak*/ true, /*encode*/ false, noreg, noreg); - __ csetw($res$$Register, Assembler::EQ); + /*acquire*/ false, /*release*/ true, /*weak*/ true, /*is_cae*/ false, $res$$Register); %} ins_pipe(pipe_slow); %} @@ -254,8 +243,7 @@ Register tmp = $tmp$$Register; __ mov(tmp, $oldval$$Register); // Must not clobber oldval. ShenandoahBarrierSet::assembler()->cmpxchg_oop(&_masm, $mem$$Register, tmp, $newval$$Register, - /*acquire*/ true, /*release*/ true, /*weak*/ true, /*encode*/ false, noreg, noreg); - __ csetw($res$$Register, Assembler::EQ); + /*acquire*/ true, /*release*/ true, /*weak*/ true, /*is_cae*/ false, $res$$Register); %} ins_pipe(pipe_slow); %} @@ -273,9 +261,7 @@ Register tmp = $tmp$$Register; __ mov(tmp, $oldval$$Register); // Must not clobber oldval. ShenandoahBarrierSet::assembler()->cmpxchg_oop(&_masm, $mem$$Register, tmp, $newval$$Register, - /*acquire*/ true, /*release*/ true, /*weak*/ true, /*encode*/ false, noreg, noreg); - __ csetw($res$$Register, Assembler::EQ); + /*acquire*/ true, /*release*/ true, /*weak*/ true, /*is_cae*/ false, $res$$Register); %} ins_pipe(pipe_slow); %} - diff -r 04dcc65c9d58 src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp --- a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp Fri Jan 25 18:13:25 2019 +0000 +++ b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp Fri Jan 25 19:20:15 2019 -0500 @@ -592,64 +592,19 @@ #ifndef _LP64 void ShenandoahBarrierSetAssembler::cmpxchg_oop(MacroAssembler* masm, Register res, Address addr, Register oldval, Register newval, - bool exchange, bool encode, Register tmp1, Register tmp2) { + bool exchange, Register tmp1, Register tmp2) { // Shenandoah has no 32-bit version for this. Unimplemented(); } #else void ShenandoahBarrierSetAssembler::cmpxchg_oop(MacroAssembler* masm, Register res, Address addr, Register oldval, Register newval, - bool exchange, bool encode, Register tmp1, Register tmp2) { - if (!ShenandoahCASBarrier) { -#ifdef _LP64 - if (UseCompressedOops) { - if (encode) { - __ encode_heap_oop(oldval); - __ mov(rscratch1, newval); - __ encode_heap_oop(rscratch1); - newval = rscratch1; - } - if (os::is_MP()) { - __ lock(); - } - // oldval (rax) is implicitly used by this instruction - __ cmpxchgl(newval, addr); - } else -#endif - { - if (os::is_MP()) { - __ lock(); - } - __ cmpxchgptr(newval, addr); - } - - if (!exchange) { - assert(res != NULL, "need result register"); - __ setb(Assembler::equal, res); - __ movzbl(res, res); - } - return; - } - + bool exchange, Register tmp1, Register tmp2) { assert(ShenandoahCASBarrier, "Should only be used when CAS barrier is enabled"); assert(oldval == rax, "must be in rax for implicit use in cmpxchg"); Label retry, done; - // Apply storeval barrier to newval. - if (encode) { - storeval_barrier(masm, newval, tmp1); - } - - if (UseCompressedOops) { - if (encode) { - __ encode_heap_oop(oldval); - __ mov(rscratch1, newval); - __ encode_heap_oop(rscratch1); - newval = rscratch1; - } - } - // Remember oldval for retry logic below if (UseCompressedOops) { __ movl(tmp1, oldval); diff -r 04dcc65c9d58 src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.hpp --- a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.hpp Fri Jan 25 18:13:25 2019 +0000 +++ b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.hpp Fri Jan 25 19:20:15 2019 -0500 @@ -64,7 +64,6 @@ void write_barrier(MacroAssembler* masm, Register dst); void write_barrier_impl(MacroAssembler* masm, Register dst); - void storeval_barrier(MacroAssembler* masm, Register dst, Register tmp); void storeval_barrier_impl(MacroAssembler* masm, Register dst, Register tmp); address generate_shenandoah_wb(StubCodeGenerator* cgen); @@ -75,6 +74,7 @@ public: static address shenandoah_wb(); + void storeval_barrier(MacroAssembler* masm, Register dst, Register tmp); #ifdef COMPILER1 void gen_pre_barrier_stub(LIR_Assembler* ce, ShenandoahPreBarrierStub* stub); void gen_write_barrier_stub(LIR_Assembler* ce, ShenandoahWriteBarrierStub* stub); @@ -83,7 +83,7 @@ void cmpxchg_oop(MacroAssembler* masm, Register res, Address addr, Register oldval, Register newval, - bool exchange, bool encode, Register tmp1, Register tmp2); + bool exchange, Register tmp1, Register tmp2); virtual void arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Register src, Register dst, Register count); virtual void arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, BasicType type, diff -r 04dcc65c9d58 src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetC1_x86.cpp --- a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetC1_x86.cpp Fri Jan 25 18:13:25 2019 +0000 +++ b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetC1_x86.cpp Fri Jan 25 19:20:15 2019 -0500 @@ -28,20 +28,36 @@ #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp" #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp" +#define __ masm->masm()-> + void LIR_OpShenandoahCompareAndSwap::emit_code(LIR_Assembler* masm) { Register addr = _addr->as_register_lo(); Register newval = _new_value->as_register(); Register cmpval = _cmp_value->as_register(); Register tmp1 = _tmp1->as_register(); Register tmp2 = _tmp2->as_register(); + Register result = result_opr()->as_register(); assert(cmpval == rax, "wrong register"); assert(newval != NULL, "new val must be register"); assert(cmpval != newval, "cmp and new values must be in different registers"); assert(cmpval != addr, "cmp and addr must be in different registers"); assert(newval != addr, "new value and addr must be in different registers"); - ShenandoahBarrierSet::assembler()->cmpxchg_oop(masm->masm(), NULL, Address(addr, 0), cmpval, newval, true, true, tmp1, tmp2); + + // Apply storeval barrier to newval. + ShenandoahBarrierSet::assembler()->storeval_barrier(masm->masm(), newval, tmp1); + + if (UseCompressedOops) { + __ encode_heap_oop(cmpval); + __ mov(rscratch1, newval); + __ encode_heap_oop(rscratch1); + newval = rscratch1; + } + + ShenandoahBarrierSet::assembler()->cmpxchg_oop(masm->masm(), result, Address(addr, 0), cmpval, newval, false, tmp1, tmp2); } +#undef __ + #ifdef ASSERT #define __ gen->lir(__FILE__, __LINE__)-> #else @@ -63,12 +79,9 @@ LIR_Opr t1 = gen->new_register(T_OBJECT); LIR_Opr t2 = gen->new_register(T_OBJECT); LIR_Opr addr = access.resolved_addr()->as_address_ptr()->base(); + LIR_Opr result = gen->new_register(T_INT); - __ append(new LIR_OpShenandoahCompareAndSwap(addr, cmp_value.result(), new_value.result(), t1, t2, LIR_OprFact::illegalOpr)); - - LIR_Opr result = gen->new_register(T_INT); - __ cmove(lir_cond_equal, LIR_OprFact::intConst(1), LIR_OprFact::intConst(0), - result, T_INT); + __ append(new LIR_OpShenandoahCompareAndSwap(addr, cmp_value.result(), new_value.result(), t1, t2, result)); return result; } } diff -r 04dcc65c9d58 src/hotspot/cpu/x86/gc/shenandoah/shenandoah_x86_64.ad --- a/src/hotspot/cpu/x86/gc/shenandoah/shenandoah_x86_64.ad Fri Jan 25 18:13:25 2019 +0000 +++ b/src/hotspot/cpu/x86/gc/shenandoah/shenandoah_x86_64.ad Fri Jan 25 19:20:15 2019 -0500 @@ -83,7 +83,7 @@ ShenandoahBarrierSet::assembler()->cmpxchg_oop(&_masm, $res$$Register, $mem_ptr$$Address, $oldval$$Register, $newval$$Register, false, // swap - false, $tmp1$$Register, $tmp2$$Register + $tmp1$$Register, $tmp2$$Register ); %} ins_pipe( pipe_cmpxchg ); @@ -104,7 +104,7 @@ ShenandoahBarrierSet::assembler()->cmpxchg_oop(&_masm, $res$$Register, $mem_ptr$$Address, $oldval$$Register, $newval$$Register, false, // swap - false, $tmp1$$Register, $tmp2$$Register + $tmp1$$Register, $tmp2$$Register ); %} ins_pipe( pipe_cmpxchg ); @@ -123,7 +123,7 @@ ShenandoahBarrierSet::assembler()->cmpxchg_oop(&_masm, NULL, $mem_ptr$$Address, $oldval$$Register, $newval$$Register, true, // exchange - false, $tmp1$$Register, $tmp2$$Register + $tmp1$$Register, $tmp2$$Register ); %} ins_pipe( pipe_cmpxchg ); @@ -145,7 +145,7 @@ ShenandoahBarrierSet::assembler()->cmpxchg_oop(&_masm, NULL, $mem_ptr$$Address, $oldval$$Register, $newval$$Register, true, // exchange - false, $tmp1$$Register, $tmp2$$Register + $tmp1$$Register, $tmp2$$Register ); %} ins_pipe( pipe_cmpxchg ); diff -r 04dcc65c9d58 src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp --- a/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp Fri Jan 25 18:13:25 2019 +0000 +++ b/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp Fri Jan 25 19:20:15 2019 -0500 @@ -777,11 +777,19 @@ if (adr->bottom_type()->is_ptr_to_narrowoop()) { Node *newval_enc = kit->gvn().transform(new EncodePNode(new_val, new_val->bottom_type()->make_narrowoop())); Node *oldval_enc = kit->gvn().transform(new EncodePNode(expected_val, expected_val->bottom_type()->make_narrowoop())); - load_store = kit->gvn().transform(new ShenandoahCompareAndExchangeNNode(kit->control(), mem, adr, newval_enc, oldval_enc, adr_type, value_type->make_narrowoop(), mo)); + if (ShenandoahCASBarrier) { + load_store = kit->gvn().transform(new ShenandoahCompareAndExchangeNNode(kit->control(), mem, adr, newval_enc, oldval_enc, adr_type, value_type->make_narrowoop(), mo)); + } else { + load_store = kit->gvn().transform(new CompareAndExchangeNNode(kit->control(), mem, adr, newval_enc, oldval_enc, adr_type, value_type->make_narrowoop(), mo)); + } } else #endif { - load_store = kit->gvn().transform(new ShenandoahCompareAndExchangePNode(kit->control(), mem, adr, new_val, expected_val, adr_type, value_type->is_oopptr(), mo)); + if (ShenandoahCASBarrier) { + load_store = kit->gvn().transform(new ShenandoahCompareAndExchangePNode(kit->control(), mem, adr, new_val, expected_val, adr_type, value_type->is_oopptr(), mo)); + } else { + load_store = kit->gvn().transform(new CompareAndExchangePNode(kit->control(), mem, adr, new_val, expected_val, adr_type, value_type->is_oopptr(), mo)); + } } access.set_raw_access(load_store); @@ -815,18 +823,34 @@ if (adr->bottom_type()->is_ptr_to_narrowoop()) { Node *newval_enc = kit->gvn().transform(new EncodePNode(new_val, new_val->bottom_type()->make_narrowoop())); Node *oldval_enc = kit->gvn().transform(new EncodePNode(expected_val, expected_val->bottom_type()->make_narrowoop())); - if (is_weak_cas) { - load_store = kit->gvn().transform(new ShenandoahWeakCompareAndSwapNNode(kit->control(), mem, adr, newval_enc, oldval_enc, mo)); + if (ShenandoahCASBarrier) { + if (is_weak_cas) { + load_store = kit->gvn().transform(new ShenandoahWeakCompareAndSwapNNode(kit->control(), mem, adr, newval_enc, oldval_enc, mo)); + } else { + load_store = kit->gvn().transform(new ShenandoahCompareAndSwapNNode(kit->control(), mem, adr, newval_enc, oldval_enc, mo)); + } } else { - load_store = kit->gvn().transform(new ShenandoahCompareAndSwapNNode(kit->control(), mem, adr, newval_enc, oldval_enc, mo)); + if (is_weak_cas) { + load_store = kit->gvn().transform(new WeakCompareAndSwapNNode(kit->control(), mem, adr, newval_enc, oldval_enc, mo)); + } else { + load_store = kit->gvn().transform(new CompareAndSwapNNode(kit->control(), mem, adr, newval_enc, oldval_enc, mo)); + } } } else #endif { - if (is_weak_cas) { - load_store = kit->gvn().transform(new ShenandoahWeakCompareAndSwapPNode(kit->control(), mem, adr, new_val, expected_val, mo)); + if (ShenandoahCASBarrier) { + if (is_weak_cas) { + load_store = kit->gvn().transform(new ShenandoahWeakCompareAndSwapPNode(kit->control(), mem, adr, new_val, expected_val, mo)); + } else { + load_store = kit->gvn().transform(new ShenandoahCompareAndSwapPNode(kit->control(), mem, adr, new_val, expected_val, mo)); + } } else { - load_store = kit->gvn().transform(new ShenandoahCompareAndSwapPNode(kit->control(), mem, adr, new_val, expected_val, mo)); + if (is_weak_cas) { + load_store = kit->gvn().transform(new WeakCompareAndSwapPNode(kit->control(), mem, adr, new_val, expected_val, mo)); + } else { + load_store = kit->gvn().transform(new CompareAndSwapPNode(kit->control(), mem, adr, new_val, expected_val, mo)); + } } } access.set_raw_access(load_store);