--- old/src/cpu/aarch64/vm/aarch64.ad 2016-11-16 17:45:40.137724453 +0000 +++ new/src/cpu/aarch64/vm/aarch64.ad 2016-11-16 17:45:39.907719423 +0000 @@ -4282,14 +4282,14 @@ %} - enc_class aarch64_enc_cmpxchg_oop_shenandoah(iRegINoSp res, memory mem, iRegLNoSp oldval, iRegLNoSp newval, iRegP tmp) %{ + enc_class aarch64_enc_cmpxchg_oop_shenandoah(memory mem, iRegLNoSp oldval, iRegLNoSp newval, iRegP tmp) %{ MacroAssembler _masm(&cbuf); guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding"); Register tmp = $tmp$$Register; __ shenandoah_store_check($mem$$base$$Register, $newval$$Register); __ mov(tmp, $oldval$$Register); // Must not clobber oldval. - __ cmpxchg_oop_shenandoah($res$$base$$Register, $mem$$base$$Register, tmp, $newval$$Register, - false, /*acquire*/ true, /*release*/ true); + __ cmpxchg_oop_shenandoah($mem$$base$$Register, tmp, $newval$$Register, + Assembler::xword, /*acquire*/ false, /*release*/ true, /*weak*/ false); %} // The only difference between aarch64_enc_cmpxchg and @@ -4315,14 +4315,14 @@ %} - enc_class aarch64_enc_cmpxchg_acq_oop_shenandoah(iRegINoSp res, memory mem, iRegLNoSp oldval, iRegLNoSp newval, iRegP tmp) %{ + enc_class aarch64_enc_cmpxchg_acq_oop_shenandoah(memory mem, iRegLNoSp oldval, iRegLNoSp newval, iRegP tmp) %{ MacroAssembler _masm(&cbuf); guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding"); Register tmp = $tmp$$Register; __ shenandoah_store_check($mem$$base$$Register, $newval$$Register); __ mov(tmp, $oldval$$Register); // Must not clobber oldval. - __ cmpxchg_oop_shenandoah($res$$base$$Register, $mem$$base$$Register, tmp, $newval$$Register, - false, /*acquire*/ true, /*release*/ true); + __ cmpxchg_oop_shenandoah($mem$$base$$Register, tmp, $newval$$Register, + Assembler::xword, /*acquire*/ true, /*release*/ true, /*weak*/ false); %} // auxiliary used for CompareAndSwapX to set result register @@ -9658,11 +9658,11 @@ effect(TEMP tmp, KILL cr); format %{ - "cmpxchg_oop_shenandoah $res, $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval with temp $tmp" + "cmpxchg_oop_shenandoah $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval with temp $tmp" "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)" %} - ins_encode(aarch64_enc_cmpxchg_oop_shenandoah(res, mem, oldval, newval, tmp), + ins_encode(aarch64_enc_cmpxchg_oop_shenandoah(mem, oldval, newval, tmp), aarch64_enc_cset_eq(res)); ins_pipe(pipe_slow); @@ -9696,14 +9696,14 @@ effect(TEMP tmp, KILL cr); format %{ - "cmpxchg_narrow_oop_shenandoah $res, $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval with temp $tmp" + "cmpxchg_narrow_oop_shenandoah $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval with temp $tmp" "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)" %} ins_encode %{ Register tmp = $tmp$$Register; __ mov(tmp, $oldval$$Register); // Must not clobber oldval. - __ cmpxchg_oop_shenandoah($res$$base$$Register, $mem$$base$$Register, tmp, $newval$$Register, true, /*acquire*/ true, /*release*/ true); + __ cmpxchg_oop_shenandoah($mem$$base$$Register, tmp, $newval$$Register, Assembler::word, /*acquire*/ false, /*release*/ true, /*weak*/ false); __ cset($res$$Register, Assembler::EQ); %} @@ -9778,11 +9778,11 @@ effect(TEMP tmp, KILL cr); format %{ - "cmpxchg_acq_oop_shenandoah $res,$mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval with temp $tmp" + "cmpxchg_acq_oop_shenandoah $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval with temp $tmp" "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)" %} - ins_encode(aarch64_enc_cmpxchg_acq_oop_shenandoah(res, mem, oldval, newval, tmp), + ins_encode(aarch64_enc_cmpxchg_acq_oop_shenandoah(mem, oldval, newval, tmp), aarch64_enc_cset_eq(res)); ins_pipe(pipe_slow); @@ -9816,14 +9816,14 @@ effect(TEMP tmp, KILL cr); format %{ - "cmpxchg_narrow_oop_shenandoah $res, $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval with temp $tmp" + "cmpxchg_narrow_oop_shenandoah $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval with temp $tmp" "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)" %} ins_encode %{ Register tmp = $tmp$$Register; __ mov(tmp, $oldval$$Register); // Must not clobber oldval. - __ cmpxchg_oop_shenandoah($res$$base$$Register, $mem$$base$$Register, tmp, $newval$$Register, true, /*acquire*/ true, /*release*/ true); + __ cmpxchg_oop_shenandoah($mem$$base$$Register, tmp, $newval$$Register, Assembler::word, /*acquire*/ true, /*release*/ true, /*weak*/ false); __ cset($res$$Register, Assembler::EQ); %} @@ -9906,6 +9906,7 @@ %} instruct compareAndExchangeN(iRegN_R0 res, indirect mem, iRegN_R2 oldval, iRegN_R3 newval, rFlagsReg cr) %{ + predicate(! UseShenandoahGC); match(Set res (CompareAndExchangeN mem (Binary oldval newval))); ins_cost(2 * VOLATILE_REF_COST); effect(KILL cr); @@ -9920,7 +9921,26 @@ ins_pipe(pipe_slow); %} +instruct compareAndExchangeN_shenandoah(iRegN_R0 res, indirect mem, iRegN_R2 oldval, iRegN_R3 newval, iRegP tmp, rFlagsReg cr) %{ + predicate(UseShenandoahGC); + match(Set res (CompareAndExchangeN mem (Binary oldval newval))); + ins_cost(3 * VOLATILE_REF_COST); + effect(TEMP tmp, KILL cr); + format %{ + "cmpxchg_oop_shenandoah $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval" + %} + ins_encode %{ + Register tmp = $tmp$$Register; + __ mov(tmp, $oldval$$Register); // Must not clobber oldval. + __ cmpxchg_oop_shenandoah($mem$$Register, tmp, $newval$$Register, + Assembler::word, /*acquire*/ false, /*release*/ true, + /*weak*/ false, $res$$Register); + %} + ins_pipe(pipe_slow); +%} + instruct compareAndExchangeP(iRegP_R0 res, indirect mem, iRegP_R2 oldval, iRegP_R3 newval, rFlagsReg cr) %{ + predicate(!UseShenandoahGC || n->in(3)->in(1)->bottom_type() == TypePtr::NULL_PTR); match(Set res (CompareAndExchangeP mem (Binary oldval newval))); ins_cost(2 * VOLATILE_REF_COST); effect(KILL cr); @@ -9935,6 +9955,24 @@ ins_pipe(pipe_slow); %} +instruct compareAndExchangeP_shenandoah(iRegP_R0 res, indirect mem, iRegP_R2 oldval, iRegP_R3 newval, iRegP tmp, rFlagsReg cr) %{ + predicate(UseShenandoahGC); + match(Set res (CompareAndExchangeP mem (Binary oldval newval))); + ins_cost(3 * VOLATILE_REF_COST); + effect(TEMP tmp, KILL cr); + format %{ + "cmpxchg_oop_shenandoah $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval with temp $tmp" + %} + ins_encode %{ + Register tmp = $tmp$$Register; + __ mov(tmp, $oldval$$Register); // Must not clobber oldval. + __ cmpxchg_oop_shenandoah($mem$$Register, tmp, $newval$$Register, + Assembler::xword, /*acquire*/ false, /*release*/ true, + /*weak*/ false, $res$$Register); + %} + ins_pipe(pipe_slow); +%} + instruct weakCompareAndSwapB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{ match(Set res (WeakCompareAndSwapB mem (Binary oldval newval))); ins_cost(2 * VOLATILE_REF_COST); @@ -10006,6 +10044,7 @@ %} instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{ + predicate(! UseShenandoahGC); match(Set res (WeakCompareAndSwapN mem (Binary oldval newval))); ins_cost(2 * VOLATILE_REF_COST); effect(KILL cr); @@ -10022,7 +10061,28 @@ ins_pipe(pipe_slow); %} +instruct weakCompareAndSwapN_shenandoah(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, iRegP tmp, rFlagsReg cr) %{ + predicate(UseShenandoahGC); + match(Set res (WeakCompareAndSwapN mem (Binary oldval newval))); + ins_cost(3 * VOLATILE_REF_COST); + effect(TEMP tmp, KILL cr); + format %{ + "cmpxchg_oop_shenandoah $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval" + "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)" + %} + ins_encode %{ + Register tmp = $tmp$$Register; + __ mov(tmp, $oldval$$Register); // Must not clobber oldval. + __ cmpxchg_oop_shenandoah($mem$$Register, tmp, $newval$$Register, + Assembler::word, /*acquire*/ false, /*release*/ true, + /*weak*/ true); + __ csetw($res$$Register, Assembler::EQ); + %} + ins_pipe(pipe_slow); +%} + instruct weakCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{ + predicate(!UseShenandoahGC || n->in(3)->in(1)->bottom_type() == TypePtr::NULL_PTR); match(Set res (WeakCompareAndSwapP mem (Binary oldval newval))); ins_cost(2 * VOLATILE_REF_COST); effect(KILL cr); @@ -10037,6 +10097,26 @@ __ csetw($res$$Register, Assembler::EQ); %} ins_pipe(pipe_slow); +%} + +instruct weakCompareAndSwapP_shenandoah(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, iRegP tmp, rFlagsReg cr) %{ + predicate(UseShenandoahGC); + match(Set res (WeakCompareAndSwapP mem (Binary oldval newval))); + ins_cost(3 * VOLATILE_REF_COST); + effect(TEMP tmp, KILL cr); + format %{ + "cmpxchg_oop_shenandoah $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval" + "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)" + %} + ins_encode %{ + Register tmp = $tmp$$Register; + __ mov(tmp, $oldval$$Register); // Must not clobber oldval. + __ cmpxchg_oop_shenandoah($mem$$Register, tmp, $newval$$Register, + Assembler::xword, /*acquire*/ false, /*release*/ true, + /*weak*/ true); + __ csetw($res$$Register, Assembler::EQ); + %} + ins_pipe(pipe_slow); %} // --------------------------------------------------------------------- --- old/src/cpu/aarch64/vm/c1_LIRAssembler_aarch64.cpp 2016-11-16 17:45:42.107767532 +0000 +++ new/src/cpu/aarch64/vm/c1_LIRAssembler_aarch64.cpp 2016-11-16 17:45:41.877762503 +0000 @@ -1617,7 +1617,8 @@ Register t2 = op->tmp2()->as_register(); __ encode_heap_oop(t2, newval); newval = t2; - __ cmpxchg_oop_shenandoah(res, addr, cmpval, newval, true, true, true); + __ cmpxchg_oop_shenandoah(addr, cmpval, newval, Assembler::word, true, true, false); + __ csetw(res, Assembler::EQ); } else { __ encode_heap_oop(t1, cmpval); cmpval = t1; @@ -1628,7 +1629,8 @@ } } else { if (UseShenandoahGC) { - __ cmpxchg_oop_shenandoah(res, addr, cmpval, newval, false, true, true); + __ cmpxchg_oop_shenandoah(addr, cmpval, newval, Assembler::xword, true, true, false); + __ csetw(res, Assembler::EQ); } else { casl(addr, newval, cmpval); __ eorw (res, r8, 1); --- old/src/cpu/aarch64/vm/macroAssembler_aarch64.cpp 2016-11-16 17:45:43.587799896 +0000 +++ new/src/cpu/aarch64/vm/macroAssembler_aarch64.cpp 2016-11-16 17:45:43.357794866 +0000 @@ -2197,46 +2197,50 @@ } } -void MacroAssembler::cmpxchg_oop_shenandoah(Register res, Register addr, Register expected, +void MacroAssembler::cmpxchg_oop_shenandoah(Register addr, Register expected, Register new_val, - bool narrow, + enum operand_size size, bool acquire, bool release, - Register tmp1, Register tmp2) { + bool weak, + Register result, Register tmp2) { assert(UseShenandoahGC, "only for shenandoah"); - assert_different_registers(res, addr, expected, new_val, tmp1, tmp2); + assert_different_registers(addr, expected, new_val, result, tmp2); Label retry, done, fail; - mov(res, 0); - // CAS, using LL/SC pair. bind(retry); - load_exclusive(tmp1, addr, narrow ? word : xword, true); - if (narrow) cmpw(tmp1, expected); - else cmp(tmp1, expected); + load_exclusive(result, addr, size, acquire); + if (size == xword) { + cmp(result, expected); + } else { + cmpw(result, expected); + } br(Assembler::NE, fail); - store_exclusive(tmp2, new_val, addr, narrow ? word : xword, true); - cbnzw(tmp2, retry); - mov(res, 1); + store_exclusive(tmp2, new_val, addr, size, release); + if (weak) { + cmpw(tmp2, 0u); // If the store fails, return NE to our caller + } else { + cbnzw(tmp2, retry); + } b(done); bind(fail); - // Check if rb(expected)==rb(tmp1) + // Check if rb(expected)==rb(result) // Shuffle registers so that we have memory value ready for next expected. mov(tmp2, expected); - mov(expected, tmp1); - if (narrow) { - decode_heap_oop(tmp1, tmp1); + mov(expected, result); + if (size == word) { + decode_heap_oop(result, result); decode_heap_oop(tmp2, tmp2); } - oopDesc::bs()->interpreter_read_barrier(this, tmp1); + oopDesc::bs()->interpreter_read_barrier(this, result); oopDesc::bs()->interpreter_read_barrier(this, tmp2); - cmp(tmp1, tmp2); + cmp(result, tmp2); // Retry with expected now being the value we just loaded from addr. br(Assembler::EQ, retry); bind(done); - membar(AnyAny); } static bool different(Register a, RegisterOrConstant b, Register c) { --- old/src/cpu/aarch64/vm/macroAssembler_aarch64.hpp 2016-11-16 17:45:45.127833572 +0000 +++ new/src/cpu/aarch64/vm/macroAssembler_aarch64.hpp 2016-11-16 17:45:44.887828324 +0000 @@ -1021,10 +1021,10 @@ bool acquire, bool release, bool weak, Register result); - void cmpxchg_oop_shenandoah(Register res, Register addr, Register expected, Register new_val, - bool narrow, - bool acquire, bool release, - Register tmp1 = rscratch1, Register tmp2 = rscratch2); + void cmpxchg_oop_shenandoah(Register addr, Register expected, Register new_val, + enum operand_size size, + bool acquire, bool release, bool weak, + Register result = rscratch1, Register tmp2 = rscratch2); // Calls address trampoline_call(Address entry, CodeBuffer *cbuf = NULL);