# HG changeset patch # User enevill # Date 1456172372 0 # Mon Feb 22 20:19:32 2016 +0000 # Node ID 54b136698526c855ab36bb95880712616512c3a4 # Parent db7934bcad3b70cfcaf0aef93dbe540898d99b44 8150394: aarch64: add support for 8.1 LSE CAS instructions Reviewed-by: duke Contributed-by: ananth.jasty@caviumnetworks.com, edward.nevill@gmail.com diff --git a/src/cpu/aarch64/vm/aarch64.ad b/src/cpu/aarch64/vm/aarch64.ad --- a/src/cpu/aarch64/vm/aarch64.ad +++ b/src/cpu/aarch64/vm/aarch64.ad @@ -4679,7 +4679,12 @@ // Compare object markOop with mark and if equal exchange scratch1 // with object markOop. - { + if (UseLSE) { + __ mov(tmp, disp_hdr); + __ casal(disp_hdr, box, oop); + __ cmp(tmp, disp_hdr); + __ br(Assembler::EQ, cont); + } else { Label retry_load; __ bind(retry_load); __ ldaxr(tmp, oop); @@ -4729,7 +4734,11 @@ __ add(tmp, disp_hdr, (ObjectMonitor::owner_offset_in_bytes()-markOopDesc::monitor_value)); __ mov(disp_hdr, zr); - { + if (UseLSE) { + __ mov(rscratch1, disp_hdr); + __ casal(disp_hdr, rthread, tmp); + __ cmp(rscratch1, disp_hdr); + } else { Label retry_load, fail; __ bind(retry_load); __ ldaxr(rscratch1, tmp); @@ -4818,7 +4827,11 @@ // see the stack address of the basicLock in the markOop of the // object. - { + if (UseLSE) { + __ mov(tmp, box); + __ casl(box, disp_hdr, oop); + __ cmp(tmp, box); + } else { Label retry_load; __ bind(retry_load); __ ldxr(tmp, oop); diff --git a/src/cpu/aarch64/vm/assembler_aarch64.hpp b/src/cpu/aarch64/vm/assembler_aarch64.hpp --- a/src/cpu/aarch64/vm/assembler_aarch64.hpp +++ b/src/cpu/aarch64/vm/assembler_aarch64.hpp @@ -1169,6 +1169,60 @@ #undef INSN4 #undef INSN_FOO + // 8.1 Compare and swap extensions + void compare_and_swap(Register Rs, Register Rt, Register Rn, + enum operand_size sz, int op, int a, int r) { + starti; + f(sz, 31, 30), f(0b001000, 29, 24), f(1, 23), f(a, 22), f(1, 21); + rf(Rs, 16), f(r, 15), f(0b11111, 14, 10), rf(Rn, 5), rf(Rt, 0); + } + + // CAS +#define INSN(NAME, sz, a, r) \ + void NAME(Register Rs, Register Rt, Register Rn) { \ + assert(Rs != Rn && Rs != Rt, "unpredictable instruction"); \ + compare_and_swap(Rs, Rt, Rn, sz, 1, a, r); \ + } + INSN(casb, byte, 0, 0) + INSN(cash, halfword, 0, 0) + INSN(casw, word, 0, 0) + INSN(cas, xword, 0, 0) + INSN(casab, byte, 1, 0) + INSN(casah, halfword, 1, 0) + INSN(casaw, word, 1, 0) + INSN(casa, xword, 1, 0) + INSN(caslb, byte, 0, 1) + INSN(caslh, halfword, 0, 1) + INSN(caslw, word, 0, 1) + INSN(casl, xword, 0, 1) + INSN(casalb, byte, 1, 1) + INSN(casalh, halfword, 1, 1) + INSN(casalw, word, 1, 1) + INSN(casal, xword, 1, 1) +#undef INSN + + // CASP +#define INSN(NAME, sz, a, r) \ + void NAME(Register Rs, Register Rs1, \ + Register Rt, Register Rt1, Register Rn) { \ + assert((Rs->encoding() & 1) == 0 && (Rt->encoding() & 1) == 0 && \ + Rs->successor() == Rs1 && Rt->successor() == Rt1 && \ + Rs != Rn && Rs1 != Rn && Rs != Rt, "invalid registers"); \ + assert(sz == word || sz == xword, "invalid size"); \ + /* The size bit is in bit 30, not 31 as in ldx/stx above */ \ + compare_and_swap(Rs, Rt, Rn, \ + (operand_size)(sz == word ? 0b00:0b01), 0, a, r); \ + } + INSN(caspw, word, 0, 0) + INSN(casp, xword, 0, 0) + INSN(caspaw, word, 1, 0) + INSN(caspa, xword, 1, 0) + INSN(casplw, word, 0, 1) + INSN(caspl, xword, 0, 1) + INSN(caspalw, word, 1, 1) + INSN(caspal, xword, 1, 1) +#undef INSN + // Load register (literal) #define INSN(NAME, opc, V) \ void NAME(Register Rt, address dest) { \ diff --git a/src/cpu/aarch64/vm/c1_LIRAssembler_aarch64.cpp b/src/cpu/aarch64/vm/c1_LIRAssembler_aarch64.cpp --- a/src/cpu/aarch64/vm/c1_LIRAssembler_aarch64.cpp +++ b/src/cpu/aarch64/vm/c1_LIRAssembler_aarch64.cpp @@ -1556,38 +1556,52 @@ } void LIR_Assembler::casw(Register addr, Register newval, Register cmpval) { - Label retry_load, nope; - // flush and load exclusive from the memory location - // and fail if it is not what we expect - __ bind(retry_load); - __ ldaxrw(rscratch1, addr); - __ cmpw(rscratch1, cmpval); - __ cset(rscratch1, Assembler::NE); - __ br(Assembler::NE, nope); - // if we store+flush with no intervening write rscratch1 wil be zero - __ stlxrw(rscratch1, newval, addr); - // retry so we only ever return after a load fails to compare - // ensures we don't return a stale value after a failed write. - __ cbnzw(rscratch1, retry_load); - __ bind(nope); + if (UseLSE) { + __ mov(rscratch1, cmpval); + __ casalw(cmpval, newval, addr); + __ cmpw(rscratch1, cmpval); + __ cset(rscratch1, Assembler::NE); + } else { + Label retry_load, nope; + // flush and load exclusive from the memory location + // and fail if it is not what we expect + __ bind(retry_load); + __ ldaxrw(rscratch1, addr); + __ cmpw(rscratch1, cmpval); + __ cset(rscratch1, Assembler::NE); + __ br(Assembler::NE, nope); + // if we store+flush with no intervening write rscratch1 wil be zero + __ stlxrw(rscratch1, newval, addr); + // retry so we only ever return after a load fails to compare + // ensures we don't return a stale value after a failed write. + __ cbnzw(rscratch1, retry_load); + __ bind(nope); + } __ membar(__ AnyAny); } void LIR_Assembler::casl(Register addr, Register newval, Register cmpval) { - Label retry_load, nope; - // flush and load exclusive from the memory location - // and fail if it is not what we expect - __ bind(retry_load); - __ ldaxr(rscratch1, addr); - __ cmp(rscratch1, cmpval); - __ cset(rscratch1, Assembler::NE); - __ br(Assembler::NE, nope); - // if we store+flush with no intervening write rscratch1 wil be zero - __ stlxr(rscratch1, newval, addr); - // retry so we only ever return after a load fails to compare - // ensures we don't return a stale value after a failed write. - __ cbnz(rscratch1, retry_load); - __ bind(nope); + if (UseLSE) { + __ mov(rscratch1, cmpval); + __ casal(cmpval, newval, addr); + __ cmp(rscratch1, cmpval); + __ cset(rscratch1, Assembler::NE); + } else { + Label retry_load, nope; + // flush and load exclusive from the memory location + // and fail if it is not what we expect + __ bind(retry_load); + __ ldaxr(rscratch1, addr); + __ cmp(rscratch1, cmpval); + __ cset(rscratch1, Assembler::NE); + __ br(Assembler::NE, nope); + // if we store+flush with no intervening write rscratch1 wil be zero + __ stlxr(rscratch1, newval, addr); + // retry so we only ever return after a load fails to compare + // ensures we don't return a stale value after a failed write. + __ cbnz(rscratch1, retry_load); + __ bind(nope); + } __ membar(__ AnyAny); } diff --git a/src/cpu/aarch64/vm/globals_aarch64.hpp b/src/cpu/aarch64/vm/globals_aarch64.hpp --- a/src/cpu/aarch64/vm/globals_aarch64.hpp +++ b/src/cpu/aarch64/vm/globals_aarch64.hpp @@ -101,6 +101,9 @@ \ product(bool, UseCRC32, false, \ "Use CRC32 instructions for CRC32 computation") \ + \ + product(bool, UseLSE, false, \ + "Use LSE instructions") \ // Don't attempt to use Neon on builtin sim until builtin sim supports it #define UseCRC32 false @@ -121,6 +124,8 @@ "Use Neon for CRC32 computation") \ product(bool, UseCRC32, false, \ "Use CRC32 instructions for CRC32 computation") \ + product(bool, UseLSE, false, \ + "Use LSE instructions") \ product(bool, TraceTraps, false, "Trace all traps the signal handler") #endif diff --git a/src/cpu/aarch64/vm/macroAssembler_aarch64.cpp b/src/cpu/aarch64/vm/macroAssembler_aarch64.cpp --- a/src/cpu/aarch64/vm/macroAssembler_aarch64.cpp +++ b/src/cpu/aarch64/vm/macroAssembler_aarch64.cpp @@ -2071,24 +2071,32 @@ // newv holds value to write in exchange // addr identifies memory word to compare against/update // tmp returns 0/1 for success/failure - Label retry_load, nope; - - bind(retry_load); - // flush and load exclusive from the memory location - // and fail if it is not what we expect - ldaxr(tmp, addr); - cmp(tmp, oldv); - br(Assembler::NE, nope); - // if we store+flush with no intervening write tmp wil be zero - stlxr(tmp, newv, addr); - cbzw(tmp, succeed); - // retry so we only ever return after a load fails to compare - // ensures we don't return a stale value after a failed write. - b(retry_load); - // if the memory word differs we return it in oldv and signal a fail - bind(nope); - membar(AnyAny); - mov(oldv, tmp); + if (UseLSE) { + mov(tmp, oldv); + casal(oldv, newv, addr); + cmp(tmp, oldv); + br(Assembler::EQ, succeed); + membar(AnyAny); + } else { + Label retry_load, nope; + + bind(retry_load); + // flush and load exclusive from the memory location + // and fail if it is not what we expect + ldaxr(tmp, addr); + cmp(tmp, oldv); + br(Assembler::NE, nope); + // if we store+flush with no intervening write tmp wil be zero + stlxr(tmp, newv, addr); + cbzw(tmp, succeed); + // retry so we only ever return after a load fails to compare + // ensures we don't return a stale value after a failed write. + b(retry_load); + // if the memory word differs we return it in oldv and signal a fail + bind(nope); + membar(AnyAny); + mov(oldv, tmp); + } if (fail) b(*fail); } @@ -2099,24 +2107,32 @@ // newv holds value to write in exchange // addr identifies memory word to compare against/update // tmp returns 0/1 for success/failure - Label retry_load, nope; - - bind(retry_load); - // flush and load exclusive from the memory location - // and fail if it is not what we expect - ldaxrw(tmp, addr); - cmp(tmp, oldv); - br(Assembler::NE, nope); - // if we store+flush with no intervening write tmp wil be zero - stlxrw(tmp, newv, addr); - cbzw(tmp, succeed); - // retry so we only ever return after a load fails to compare - // ensures we don't return a stale value after a failed write. - b(retry_load); - // if the memory word differs we return it in oldv and signal a fail - bind(nope); - membar(AnyAny); - mov(oldv, tmp); + if (UseLSE) { + mov(tmp, oldv); + casalw(oldv, newv, addr); + cmp(tmp, oldv); + br(Assembler::EQ, succeed); + membar(AnyAny); + } else { + Label retry_load, nope; + + bind(retry_load); + // flush and load exclusive from the memory location + // and fail if it is not what we expect + ldaxrw(tmp, addr); + cmp(tmp, oldv); + br(Assembler::NE, nope); + // if we store+flush with no intervening write tmp wil be zero + stlxrw(tmp, newv, addr); + cbzw(tmp, succeed); + // retry so we only ever return after a load fails to compare + // ensures we don't return a stale value after a failed write. + b(retry_load); + // if the memory word differs we return it in oldv and signal a fail + bind(nope); + membar(AnyAny); + mov(oldv, tmp); + } if (fail) b(*fail); } diff --git a/src/cpu/aarch64/vm/vm_version_aarch64.cpp b/src/cpu/aarch64/vm/vm_version_aarch64.cpp --- a/src/cpu/aarch64/vm/vm_version_aarch64.cpp +++ b/src/cpu/aarch64/vm/vm_version_aarch64.cpp @@ -61,6 +61,10 @@ #define HWCAP_CRC32 (1<<7) #endif +#ifndef HWCAP_ATOMICS +#define HWCAP_ATOMICS (1<<8) +#endif + int VM_Version::_cpu; int VM_Version::_model; int VM_Version::_model2; @@ -172,6 +176,7 @@ if (auxv & HWCAP_AES) strcat(buf, ", aes"); if (auxv & HWCAP_SHA1) strcat(buf, ", sha1"); if (auxv & HWCAP_SHA2) strcat(buf, ", sha256"); + if (auxv & HWCAP_ATOMICS) strcat(buf, ", lse"); _features_string = os::strdup(buf); @@ -191,6 +196,15 @@ FLAG_SET_DEFAULT(UseVectorizedMismatchIntrinsic, false); } + if (auxv & HWCAP_ATOMICS) { + if (FLAG_IS_DEFAULT(UseLSE)) + FLAG_SET_DEFAULT(UseLSE, true); + } else { + if (UseLSE) { + warning("UseLSE specified, but not supported on this CPU"); + } + } + if (auxv & HWCAP_AES) { UseAES = UseAES || FLAG_IS_DEFAULT(UseAES); UseAESIntrinsics =