< prev index next >
src/cpu/aarch64/vm/aarch64.ad
Print this page
rev 8905 : 8135157: DMB elimination in AArch64 C2 synchronization implementation
Summary: Reduce memory barrier usage in C2 fast lock and unlock.
Reviewed-by: kvn
Contributed-by: wei.tang@linaro.org, aph@redhat.com
*** 3801,3886 ****
__ cmpw(rscratch1, zr);
%}
enc_class aarch64_enc_cmpxchg(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
MacroAssembler _masm(&cbuf);
! Register old_reg = as_Register($oldval$$reg);
! Register new_reg = as_Register($newval$$reg);
! Register base = as_Register($mem$$base);
! Register addr_reg;
! int index = $mem$$index;
! int scale = $mem$$scale;
! int disp = $mem$$disp;
! if (index == -1) {
! if (disp != 0) {
! __ lea(rscratch2, Address(base, disp));
! addr_reg = rscratch2;
! } else {
! // TODO
! // should we ever get anything other than this case?
! addr_reg = base;
! }
! } else {
! Register index_reg = as_Register(index);
! if (disp == 0) {
! __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
! addr_reg = rscratch2;
! } else {
! __ lea(rscratch2, Address(base, disp));
! __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
! addr_reg = rscratch2;
! }
! }
! Label retry_load, done;
! __ bind(retry_load);
! __ ldxr(rscratch1, addr_reg);
! __ cmp(rscratch1, old_reg);
! __ br(Assembler::NE, done);
! __ stlxr(rscratch1, new_reg, addr_reg);
! __ cbnzw(rscratch1, retry_load);
! __ bind(done);
%}
enc_class aarch64_enc_cmpxchgw(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
MacroAssembler _masm(&cbuf);
! Register old_reg = as_Register($oldval$$reg);
! Register new_reg = as_Register($newval$$reg);
! Register base = as_Register($mem$$base);
! Register addr_reg;
! int index = $mem$$index;
! int scale = $mem$$scale;
! int disp = $mem$$disp;
! if (index == -1) {
! if (disp != 0) {
! __ lea(rscratch2, Address(base, disp));
! addr_reg = rscratch2;
! } else {
! // TODO
! // should we ever get anything other than this case?
! addr_reg = base;
! }
! } else {
! Register index_reg = as_Register(index);
! if (disp == 0) {
! __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
! addr_reg = rscratch2;
! } else {
! __ lea(rscratch2, Address(base, disp));
! __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
! addr_reg = rscratch2;
! }
! }
! Label retry_load, done;
! __ bind(retry_load);
! __ ldxrw(rscratch1, addr_reg);
! __ cmpw(rscratch1, old_reg);
! __ br(Assembler::NE, done);
! __ stlxrw(rscratch1, new_reg, addr_reg);
! __ cbnzw(rscratch1, retry_load);
! __ bind(done);
%}
// auxiliary used for CompareAndSwapX to set result register
enc_class aarch64_enc_cset_eq(iRegINoSp res) %{
MacroAssembler _masm(&cbuf);
Register res_reg = as_Register($res$$reg);
__ cset(res_reg, Assembler::EQ);
--- 3801,3842 ----
__ cmpw(rscratch1, zr);
%}
enc_class aarch64_enc_cmpxchg(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
MacroAssembler _masm(&cbuf);
! guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
! __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
! &Assembler::ldxr, &MacroAssembler::cmp, &Assembler::stlxr);
%}
enc_class aarch64_enc_cmpxchgw(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
MacroAssembler _masm(&cbuf);
! guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
! __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
! &Assembler::ldxrw, &MacroAssembler::cmpw, &Assembler::stlxrw);
! %}
!
!
! // The only difference between aarch64_enc_cmpxchg and
! // aarch64_enc_cmpxchg_acq is that we use load-acquire in the
! // CompareAndSwap sequence to serve as a barrier on acquiring a
! // lock.
! enc_class aarch64_enc_cmpxchg_acq(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
! MacroAssembler _masm(&cbuf);
! guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
! __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
! &Assembler::ldaxr, &MacroAssembler::cmp, &Assembler::stlxr);
! %}
!
! enc_class aarch64_enc_cmpxchgw_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
! MacroAssembler _masm(&cbuf);
! guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
! __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
! &Assembler::ldaxrw, &MacroAssembler::cmpw, &Assembler::stlxrw);
%}
+
// auxiliary used for CompareAndSwapX to set result register
enc_class aarch64_enc_cset_eq(iRegINoSp res) %{
MacroAssembler _masm(&cbuf);
Register res_reg = as_Register($res$$reg);
__ cset(res_reg, Assembler::EQ);
*** 4396,4412 ****
// Initialize the box. (Must happen before we update the object mark!)
__ str(disp_hdr, Address(box, BasicLock::displaced_header_offset_in_bytes()));
// Compare object markOop with mark and if equal exchange scratch1
// with object markOop.
- // Note that this is simply a CAS: it does not generate any
- // barriers. These are separately generated by
- // membar_acquire_lock().
{
Label retry_load;
__ bind(retry_load);
! __ ldxr(tmp, oop);
__ cmp(tmp, disp_hdr);
__ br(Assembler::NE, cas_failed);
// use stlxr to ensure update is immediately visible
__ stlxr(tmp, box, oop);
__ cbzw(tmp, cont);
--- 4352,4365 ----
// Initialize the box. (Must happen before we update the object mark!)
__ str(disp_hdr, Address(box, BasicLock::displaced_header_offset_in_bytes()));
// Compare object markOop with mark and if equal exchange scratch1
// with object markOop.
{
Label retry_load;
__ bind(retry_load);
! __ ldaxr(tmp, oop);
__ cmp(tmp, disp_hdr);
__ br(Assembler::NE, cas_failed);
// use stlxr to ensure update is immediately visible
__ stlxr(tmp, box, oop);
__ cbzw(tmp, cont);
*** 4452,4462 ****
__ mov(disp_hdr, zr);
{
Label retry_load, fail;
__ bind(retry_load);
! __ ldxr(rscratch1, tmp);
__ cmp(disp_hdr, rscratch1);
__ br(Assembler::NE, fail);
// use stlxr to ensure update is immediately visible
__ stlxr(rscratch1, rthread, tmp);
__ cbnzw(rscratch1, retry_load);
--- 4405,4415 ----
__ mov(disp_hdr, zr);
{
Label retry_load, fail;
__ bind(retry_load);
! __ ldaxr(rscratch1, tmp);
__ cmp(disp_hdr, rscratch1);
__ br(Assembler::NE, fail);
// use stlxr to ensure update is immediately visible
__ stlxr(rscratch1, rthread, tmp);
__ cbnzw(rscratch1, retry_load);
*** 8015,8028 ****
instruct membar_acquire_lock() %{
match(MemBarAcquireLock);
ins_cost(VOLATILE_REF_COST);
! format %{ "membar_acquire_lock" %}
ins_encode %{
! __ membar(Assembler::LoadLoad|Assembler::LoadStore);
%}
ins_pipe(pipe_serial);
%}
--- 7968,7981 ----
instruct membar_acquire_lock() %{
match(MemBarAcquireLock);
ins_cost(VOLATILE_REF_COST);
! format %{ "membar_acquire_lock (elided)" %}
ins_encode %{
! __ block_comment("membar_acquire_lock (elided)");
%}
ins_pipe(pipe_serial);
%}
*** 8078,8091 ****
instruct membar_release_lock() %{
match(MemBarReleaseLock);
ins_cost(VOLATILE_REF_COST);
! format %{ "membar_release_lock" %}
ins_encode %{
! __ membar(Assembler::LoadStore|Assembler::StoreStore);
%}
ins_pipe(pipe_serial);
%}
--- 8031,8044 ----
instruct membar_release_lock() %{
match(MemBarReleaseLock);
ins_cost(VOLATILE_REF_COST);
! format %{ "membar_release_lock (elided)" %}
ins_encode %{
! __ block_comment("membar_release_lock (elided)");
%}
ins_pipe(pipe_serial);
%}
*** 8367,8377 ****
ins_encode(aarch64_enc_stlxr(newval, heap_top_ptr));
ins_pipe(pipe_serial);
%}
! // this has to be implemented as a CAS
instruct storeLConditional(indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr)
%{
match(Set cr (StoreLConditional mem (Binary oldval newval)));
ins_cost(VOLATILE_REF_COST);
--- 8320,8334 ----
ins_encode(aarch64_enc_stlxr(newval, heap_top_ptr));
ins_pipe(pipe_serial);
%}
!
! // storeLConditional is used by PhaseMacroExpand::expand_lock_node
! // when attempting to rebias a lock towards the current thread. We
! // must use the acquire form of cmpxchg in order to guarantee acquire
! // semantics in this case.
instruct storeLConditional(indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr)
%{
match(Set cr (StoreLConditional mem (Binary oldval newval)));
ins_cost(VOLATILE_REF_COST);
*** 8379,8394 ****
format %{
"cmpxchg rscratch1, $mem, $oldval, $newval, $mem\t# if $mem == $oldval then $mem <-- $newval"
"cmpw rscratch1, zr\t# EQ on successful write"
%}
! ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval));
ins_pipe(pipe_slow);
%}
! // this has to be implemented as a CAS
instruct storeIConditional(indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr)
%{
match(Set cr (StoreIConditional mem (Binary oldval newval)));
ins_cost(VOLATILE_REF_COST);
--- 8336,8353 ----
format %{
"cmpxchg rscratch1, $mem, $oldval, $newval, $mem\t# if $mem == $oldval then $mem <-- $newval"
"cmpw rscratch1, zr\t# EQ on successful write"
%}
! ins_encode(aarch64_enc_cmpxchg_acq(newval, heap_top_ptr));
ins_pipe(pipe_slow);
%}
! // storeIConditional also has acquire semantics, for no better reason
! // than matching storeLConditional. At the time of writing this
! // comment storeIConditional was not used anywhere by AArch64.
instruct storeIConditional(indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr)
%{
match(Set cr (StoreIConditional mem (Binary oldval newval)));
ins_cost(VOLATILE_REF_COST);
*** 8396,8406 ****
format %{
"cmpxchgw rscratch1, $mem, $oldval, $newval, $mem\t# if $mem == $oldval then $mem <-- $newval"
"cmpw rscratch1, zr\t# EQ on successful write"
%}
! ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval));
ins_pipe(pipe_slow);
%}
// XXX No flag versions for CompareAndSwap{I,L,P,N} because matcher
--- 8355,8365 ----
format %{
"cmpxchgw rscratch1, $mem, $oldval, $newval, $mem\t# if $mem == $oldval then $mem <-- $newval"
"cmpw rscratch1, zr\t# EQ on successful write"
%}
! ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval));
ins_pipe(pipe_slow);
%}
// XXX No flag versions for CompareAndSwap{I,L,P,N} because matcher
< prev index next >