4265
4266 enc_class aarch64_enc_cmpxchg(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
4267 MacroAssembler _masm(&cbuf);
4268 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
4269 __ shenandoah_store_addr_check($mem$$base$$Register);
4270 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
4271 Assembler::xword, /*acquire*/ false, /*release*/ true,
4272 /*weak*/ false, noreg);
4273 %}
4274
4275 enc_class aarch64_enc_cmpxchgw(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
4276 MacroAssembler _masm(&cbuf);
4277 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
4278 __ shenandoah_store_addr_check($mem$$base$$Register);
4279 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
4280 Assembler::word, /*acquire*/ false, /*release*/ true,
4281 /*weak*/ false, noreg);
4282 %}
4283
4284
4285 enc_class aarch64_enc_cmpxchg_oop_shenandoah(iRegINoSp res, memory mem, iRegLNoSp oldval, iRegLNoSp newval, iRegP tmp) %{
4286 MacroAssembler _masm(&cbuf);
4287 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
4288 Register tmp = $tmp$$Register;
4289 __ shenandoah_store_check($mem$$base$$Register, $newval$$Register);
4290 __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
4291 __ cmpxchg_oop_shenandoah($res$$base$$Register, $mem$$base$$Register, tmp, $newval$$Register,
4292 false, /*acquire*/ true, /*release*/ true);
4293 %}
4294
4295 // The only difference between aarch64_enc_cmpxchg and
4296 // aarch64_enc_cmpxchg_acq is that we use load-acquire in the
4297 // CompareAndSwap sequence to serve as a barrier on acquiring a
4298 // lock.
4299 enc_class aarch64_enc_cmpxchg_acq(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
4300 MacroAssembler _masm(&cbuf);
4301 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
4302 __ shenandoah_store_addr_check($mem$$base$$Register);
4303 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
4304 Assembler::xword, /*acquire*/ true, /*release*/ true,
4305 /*weak*/ false, noreg);
4306 %}
4307
4308 enc_class aarch64_enc_cmpxchgw_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
4309 MacroAssembler _masm(&cbuf);
4310 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
4311 __ shenandoah_store_addr_check($mem$$base$$Register);
4312 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
4313 Assembler::word, /*acquire*/ true, /*release*/ true,
4314 /*weak*/ false, noreg);
4315 %}
4316
4317
4318 enc_class aarch64_enc_cmpxchg_acq_oop_shenandoah(iRegINoSp res, memory mem, iRegLNoSp oldval, iRegLNoSp newval, iRegP tmp) %{
4319 MacroAssembler _masm(&cbuf);
4320 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
4321 Register tmp = $tmp$$Register;
4322 __ shenandoah_store_check($mem$$base$$Register, $newval$$Register);
4323 __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
4324 __ cmpxchg_oop_shenandoah($res$$base$$Register, $mem$$base$$Register, tmp, $newval$$Register,
4325 false, /*acquire*/ true, /*release*/ true);
4326 %}
4327
4328 // auxiliary used for CompareAndSwapX to set result register
4329 enc_class aarch64_enc_cset_eq(iRegINoSp res) %{
4330 MacroAssembler _masm(&cbuf);
4331 Register res_reg = as_Register($res$$reg);
4332 __ cset(res_reg, Assembler::EQ);
4333 %}
4334
4335 // prefetch encodings
4336
4337 enc_class aarch64_enc_prefetchw(memory mem) %{
4338 MacroAssembler _masm(&cbuf);
4339 Register base = as_Register($mem$$base);
4340 int index = $mem$$index;
4341 int scale = $mem$$scale;
4342 int disp = $mem$$disp;
4343 if (index == -1) {
4344 __ prfm(Address(base, disp), PSTL1KEEP);
4345 } else {
9641
9642 format %{
9643 "cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
9644 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9645 %}
9646
9647 ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
9648 aarch64_enc_cset_eq(res));
9649
9650 ins_pipe(pipe_slow);
9651 %}
9652 instruct compareAndSwapP_shenandoah(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, iRegP tmp, rFlagsReg cr) %{
9653
9654 predicate(UseShenandoahGC);
9655 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
9656 ins_cost(3 * VOLATILE_REF_COST);
9657
9658 effect(TEMP tmp, KILL cr);
9659
9660 format %{
9661 "cmpxchg_oop_shenandoah $res, $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval with temp $tmp"
9662 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9663 %}
9664
9665 ins_encode(aarch64_enc_cmpxchg_oop_shenandoah(res, mem, oldval, newval, tmp),
9666 aarch64_enc_cset_eq(res));
9667
9668 ins_pipe(pipe_slow);
9669 %}
9670
9671 instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
9672
9673 predicate(!UseShenandoahGC);
9674 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
9675 ins_cost(2 * VOLATILE_REF_COST);
9676
9677 effect(KILL cr);
9678
9679 format %{
9680 "cmpxchgw $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
9681 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9682 %}
9683
9684 ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
9685 aarch64_enc_cset_eq(res));
9686
9687 ins_pipe(pipe_slow);
9688 %}
9689
9690 instruct compareAndSwapN_shenandoah(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, iRegP tmp, rFlagsReg cr) %{
9691
9692 predicate(UseShenandoahGC);
9693 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
9694 ins_cost(2 * VOLATILE_REF_COST);
9695
9696 effect(TEMP tmp, KILL cr);
9697
9698 format %{
9699 "cmpxchg_narrow_oop_shenandoah $res, $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval with temp $tmp"
9700 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9701 %}
9702
9703 ins_encode %{
9704 Register tmp = $tmp$$Register;
9705 __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
9706 __ cmpxchg_oop_shenandoah($res$$base$$Register, $mem$$base$$Register, tmp, $newval$$Register, true, /*acquire*/ true, /*release*/ true);
9707 __ cset($res$$Register, Assembler::EQ);
9708 %}
9709
9710 ins_pipe(pipe_slow);
9711 %}
9712
9713 // alternative CompareAndSwapX when we are eliding barriers
9714
9715 instruct compareAndSwapIAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
9716
9717 predicate(needs_acquiring_load_exclusive(n));
9718 match(Set res (CompareAndSwapI mem (Binary oldval newval)));
9719 ins_cost(VOLATILE_REF_COST);
9720
9721 effect(KILL cr);
9722
9723 format %{
9724 "cmpxchgw_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
9725 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9726 %}
9761 format %{
9762 "cmpxchg_acq $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
9763 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9764 %}
9765
9766 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
9767 aarch64_enc_cset_eq(res));
9768
9769 ins_pipe(pipe_slow);
9770 %}
9771
9772 instruct compareAndSwapPAcq_shenandoah(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, iRegP tmp, rFlagsReg cr) %{
9773
9774 predicate(needs_acquiring_load_exclusive(n) && UseShenandoahGC);
9775 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
9776 ins_cost(2 * VOLATILE_REF_COST);
9777
9778 effect(TEMP tmp, KILL cr);
9779
9780 format %{
9781 "cmpxchg_acq_oop_shenandoah $res,$mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval with temp $tmp"
9782 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9783 %}
9784
9785 ins_encode(aarch64_enc_cmpxchg_acq_oop_shenandoah(res, mem, oldval, newval, tmp),
9786 aarch64_enc_cset_eq(res));
9787
9788 ins_pipe(pipe_slow);
9789 %}
9790
9791 instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
9792
9793 predicate(needs_acquiring_load_exclusive(n) && ! UseShenandoahGC);
9794 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
9795 ins_cost(VOLATILE_REF_COST);
9796
9797 effect(KILL cr);
9798
9799 format %{
9800 "cmpxchgw_acq $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
9801 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9802 %}
9803
9804 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
9805 aarch64_enc_cset_eq(res));
9806
9807 ins_pipe(pipe_slow);
9808 %}
9809
9810 instruct compareAndSwapNAcq_shenandoah(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, iRegP tmp, rFlagsReg cr) %{
9811
9812 predicate(needs_acquiring_load_exclusive(n) && UseShenandoahGC);
9813 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
9814 ins_cost(2 * VOLATILE_REF_COST);
9815
9816 effect(TEMP tmp, KILL cr);
9817
9818 format %{
9819 "cmpxchg_narrow_oop_shenandoah $res, $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval with temp $tmp"
9820 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9821 %}
9822
9823 ins_encode %{
9824 Register tmp = $tmp$$Register;
9825 __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
9826 __ cmpxchg_oop_shenandoah($res$$base$$Register, $mem$$base$$Register, tmp, $newval$$Register, true, /*acquire*/ true, /*release*/ true);
9827 __ cset($res$$Register, Assembler::EQ);
9828 %}
9829
9830 ins_pipe(pipe_slow);
9831 %}
9832
9833 // ---------------------------------------------------------------------
9834 // Sundry CAS operations. Note that release is always true,
9835 // regardless of the memory ordering of the CAS. This is because we
9836 // need the volatile case to be sequentially consistent but there is
9837 // no trailing StoreLoad barrier emitted by C2. Unfortunately we
9838 // can't check the type of memory ordering here, so we always emit a
9839 // STLXR.
9840
9841 // This section is generated from aarch64_ad_cas.m4
9842
9843
9844 instruct compareAndExchangeB(iRegI_R0 res, indirect mem, iRegI_R2 oldval, iRegI_R3 newval, rFlagsReg cr) %{
9845 match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
9846 ins_cost(2 * VOLATILE_REF_COST);
9889 %}
9890 ins_pipe(pipe_slow);
9891 %}
9892
9893 instruct compareAndExchangeL(iRegL_R0 res, indirect mem, iRegL_R2 oldval, iRegL_R3 newval, rFlagsReg cr) %{
9894 match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
9895 ins_cost(2 * VOLATILE_REF_COST);
9896 effect(KILL cr);
9897 format %{
9898 "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
9899 %}
9900 ins_encode %{
9901 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9902 Assembler::xword, /*acquire*/ false, /*release*/ true,
9903 /*weak*/ false, $res$$Register);
9904 %}
9905 ins_pipe(pipe_slow);
9906 %}
9907
9908 instruct compareAndExchangeN(iRegN_R0 res, indirect mem, iRegN_R2 oldval, iRegN_R3 newval, rFlagsReg cr) %{
9909 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
9910 ins_cost(2 * VOLATILE_REF_COST);
9911 effect(KILL cr);
9912 format %{
9913 "cmpxchg $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
9914 %}
9915 ins_encode %{
9916 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9917 Assembler::word, /*acquire*/ false, /*release*/ true,
9918 /*weak*/ false, $res$$Register);
9919 %}
9920 ins_pipe(pipe_slow);
9921 %}
9922
9923 instruct compareAndExchangeP(iRegP_R0 res, indirect mem, iRegP_R2 oldval, iRegP_R3 newval, rFlagsReg cr) %{
9924 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
9925 ins_cost(2 * VOLATILE_REF_COST);
9926 effect(KILL cr);
9927 format %{
9928 "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
9929 %}
9930 ins_encode %{
9931 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9932 Assembler::xword, /*acquire*/ false, /*release*/ true,
9933 /*weak*/ false, $res$$Register);
9934 %}
9935 ins_pipe(pipe_slow);
9936 %}
9937
9938 instruct weakCompareAndSwapB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9939 match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
9940 ins_cost(2 * VOLATILE_REF_COST);
9941 effect(KILL cr);
9942 format %{
9943 "cmpxchg $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
9944 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9945 %}
9946 ins_encode %{
9947 __ uxtbw(rscratch2, $oldval$$Register);
9948 __ cmpxchg($mem$$Register, rscratch2, $newval$$Register,
9949 Assembler::byte, /*acquire*/ false, /*release*/ true,
9950 /*weak*/ true, noreg);
9951 __ csetw($res$$Register, Assembler::EQ);
9952 %}
9953 ins_pipe(pipe_slow);
9954 %}
9955
9956 instruct weakCompareAndSwapS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9957 match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
9989 %}
9990
9991 instruct weakCompareAndSwapL(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
9992 match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
9993 ins_cost(2 * VOLATILE_REF_COST);
9994 effect(KILL cr);
9995 format %{
9996 "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
9997 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9998 %}
9999 ins_encode %{
10000 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
10001 Assembler::xword, /*acquire*/ false, /*release*/ true,
10002 /*weak*/ true, noreg);
10003 __ csetw($res$$Register, Assembler::EQ);
10004 %}
10005 ins_pipe(pipe_slow);
10006 %}
10007
10008 instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
10009 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
10010 ins_cost(2 * VOLATILE_REF_COST);
10011 effect(KILL cr);
10012 format %{
10013 "cmpxchg $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
10014 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
10015 %}
10016 ins_encode %{
10017 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
10018 Assembler::word, /*acquire*/ false, /*release*/ true,
10019 /*weak*/ true, noreg);
10020 __ csetw($res$$Register, Assembler::EQ);
10021 %}
10022 ins_pipe(pipe_slow);
10023 %}
10024
10025 instruct weakCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
10026 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
10027 ins_cost(2 * VOLATILE_REF_COST);
10028 effect(KILL cr);
10029 format %{
10030 "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
10031 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
10032 %}
10033 ins_encode %{
10034 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
10035 Assembler::xword, /*acquire*/ false, /*release*/ true,
10036 /*weak*/ true, noreg);
10037 __ csetw($res$$Register, Assembler::EQ);
10038 %}
10039 ins_pipe(pipe_slow);
10040 %}
10041 // ---------------------------------------------------------------------
10042
10043 instruct get_and_setI(indirect mem, iRegINoSp newv, iRegI prev) %{
10044 match(Set prev (GetAndSetI mem newv));
10045 format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
10046 ins_encode %{
10047 __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
10048 %}
10049 ins_pipe(pipe_serial);
10050 %}
10051
10052 instruct get_and_setL(indirect mem, iRegLNoSp newv, iRegL prev) %{
10053 match(Set prev (GetAndSetL mem newv));
10054 format %{ "atomic_xchg $prev, $newv, [$mem]" %}
10055 ins_encode %{
10056 __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
|
4265
4266 enc_class aarch64_enc_cmpxchg(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
4267 MacroAssembler _masm(&cbuf);
4268 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
4269 __ shenandoah_store_addr_check($mem$$base$$Register);
4270 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
4271 Assembler::xword, /*acquire*/ false, /*release*/ true,
4272 /*weak*/ false, noreg);
4273 %}
4274
4275 enc_class aarch64_enc_cmpxchgw(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
4276 MacroAssembler _masm(&cbuf);
4277 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
4278 __ shenandoah_store_addr_check($mem$$base$$Register);
4279 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
4280 Assembler::word, /*acquire*/ false, /*release*/ true,
4281 /*weak*/ false, noreg);
4282 %}
4283
4284
4285 enc_class aarch64_enc_cmpxchg_oop_shenandoah(memory mem, iRegLNoSp oldval, iRegLNoSp newval, iRegP tmp) %{
4286 MacroAssembler _masm(&cbuf);
4287 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
4288 Register tmp = $tmp$$Register;
4289 __ shenandoah_store_check($mem$$base$$Register, $newval$$Register);
4290 __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
4291 __ cmpxchg_oop_shenandoah($mem$$base$$Register, tmp, $newval$$Register,
4292 Assembler::xword, /*acquire*/ false, /*release*/ true, /*weak*/ false);
4293 %}
4294
4295 // The only difference between aarch64_enc_cmpxchg and
4296 // aarch64_enc_cmpxchg_acq is that we use load-acquire in the
4297 // CompareAndSwap sequence to serve as a barrier on acquiring a
4298 // lock.
4299 enc_class aarch64_enc_cmpxchg_acq(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
4300 MacroAssembler _masm(&cbuf);
4301 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
4302 __ shenandoah_store_addr_check($mem$$base$$Register);
4303 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
4304 Assembler::xword, /*acquire*/ true, /*release*/ true,
4305 /*weak*/ false, noreg);
4306 %}
4307
4308 enc_class aarch64_enc_cmpxchgw_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
4309 MacroAssembler _masm(&cbuf);
4310 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
4311 __ shenandoah_store_addr_check($mem$$base$$Register);
4312 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
4313 Assembler::word, /*acquire*/ true, /*release*/ true,
4314 /*weak*/ false, noreg);
4315 %}
4316
4317
4318 enc_class aarch64_enc_cmpxchg_acq_oop_shenandoah(memory mem, iRegLNoSp oldval, iRegLNoSp newval, iRegP tmp) %{
4319 MacroAssembler _masm(&cbuf);
4320 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
4321 Register tmp = $tmp$$Register;
4322 __ shenandoah_store_check($mem$$base$$Register, $newval$$Register);
4323 __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
4324 __ cmpxchg_oop_shenandoah($mem$$base$$Register, tmp, $newval$$Register,
4325 Assembler::xword, /*acquire*/ true, /*release*/ true, /*weak*/ false);
4326 %}
4327
4328 // auxiliary used for CompareAndSwapX to set result register
4329 enc_class aarch64_enc_cset_eq(iRegINoSp res) %{
4330 MacroAssembler _masm(&cbuf);
4331 Register res_reg = as_Register($res$$reg);
4332 __ cset(res_reg, Assembler::EQ);
4333 %}
4334
4335 // prefetch encodings
4336
4337 enc_class aarch64_enc_prefetchw(memory mem) %{
4338 MacroAssembler _masm(&cbuf);
4339 Register base = as_Register($mem$$base);
4340 int index = $mem$$index;
4341 int scale = $mem$$scale;
4342 int disp = $mem$$disp;
4343 if (index == -1) {
4344 __ prfm(Address(base, disp), PSTL1KEEP);
4345 } else {
9641
9642 format %{
9643 "cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
9644 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9645 %}
9646
9647 ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
9648 aarch64_enc_cset_eq(res));
9649
9650 ins_pipe(pipe_slow);
9651 %}
9652 instruct compareAndSwapP_shenandoah(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, iRegP tmp, rFlagsReg cr) %{
9653
9654 predicate(UseShenandoahGC);
9655 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
9656 ins_cost(3 * VOLATILE_REF_COST);
9657
9658 effect(TEMP tmp, KILL cr);
9659
9660 format %{
9661 "cmpxchg_oop_shenandoah $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval with temp $tmp"
9662 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9663 %}
9664
9665 ins_encode(aarch64_enc_cmpxchg_oop_shenandoah(mem, oldval, newval, tmp),
9666 aarch64_enc_cset_eq(res));
9667
9668 ins_pipe(pipe_slow);
9669 %}
9670
9671 instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
9672
9673 predicate(!UseShenandoahGC);
9674 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
9675 ins_cost(2 * VOLATILE_REF_COST);
9676
9677 effect(KILL cr);
9678
9679 format %{
9680 "cmpxchgw $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
9681 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9682 %}
9683
9684 ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
9685 aarch64_enc_cset_eq(res));
9686
9687 ins_pipe(pipe_slow);
9688 %}
9689
9690 instruct compareAndSwapN_shenandoah(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, iRegP tmp, rFlagsReg cr) %{
9691
9692 predicate(UseShenandoahGC);
9693 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
9694 ins_cost(3 * VOLATILE_REF_COST);
9695
9696 effect(TEMP tmp, KILL cr);
9697
9698 format %{
9699 "cmpxchg_narrow_oop_shenandoah $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval with temp $tmp"
9700 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9701 %}
9702
9703 ins_encode %{
9704 Register tmp = $tmp$$Register;
9705 __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
9706 __ cmpxchg_oop_shenandoah($mem$$base$$Register, tmp, $newval$$Register, Assembler::word, /*acquire*/ false, /*release*/ true, /*weak*/ false);
9707 __ cset($res$$Register, Assembler::EQ);
9708 %}
9709
9710 ins_pipe(pipe_slow);
9711 %}
9712
9713 // alternative CompareAndSwapX when we are eliding barriers
9714
9715 instruct compareAndSwapIAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
9716
9717 predicate(needs_acquiring_load_exclusive(n));
9718 match(Set res (CompareAndSwapI mem (Binary oldval newval)));
9719 ins_cost(VOLATILE_REF_COST);
9720
9721 effect(KILL cr);
9722
9723 format %{
9724 "cmpxchgw_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
9725 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9726 %}
9761 format %{
9762 "cmpxchg_acq $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
9763 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9764 %}
9765
9766 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
9767 aarch64_enc_cset_eq(res));
9768
9769 ins_pipe(pipe_slow);
9770 %}
9771
9772 instruct compareAndSwapPAcq_shenandoah(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, iRegP tmp, rFlagsReg cr) %{
9773
9774 predicate(needs_acquiring_load_exclusive(n) && UseShenandoahGC);
9775 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
9776 ins_cost(2 * VOLATILE_REF_COST);
9777
9778 effect(TEMP tmp, KILL cr);
9779
9780 format %{
9781 "cmpxchg_acq_oop_shenandoah $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval with temp $tmp"
9782 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9783 %}
9784
9785 ins_encode(aarch64_enc_cmpxchg_acq_oop_shenandoah(mem, oldval, newval, tmp),
9786 aarch64_enc_cset_eq(res));
9787
9788 ins_pipe(pipe_slow);
9789 %}
9790
9791 instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
9792
9793 predicate(needs_acquiring_load_exclusive(n) && ! UseShenandoahGC);
9794 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
9795 ins_cost(VOLATILE_REF_COST);
9796
9797 effect(KILL cr);
9798
9799 format %{
9800 "cmpxchgw_acq $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
9801 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9802 %}
9803
9804 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
9805 aarch64_enc_cset_eq(res));
9806
9807 ins_pipe(pipe_slow);
9808 %}
9809
9810 instruct compareAndSwapNAcq_shenandoah(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, iRegP tmp, rFlagsReg cr) %{
9811
9812 predicate(needs_acquiring_load_exclusive(n) && UseShenandoahGC);
9813 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
9814 ins_cost(3 * VOLATILE_REF_COST);
9815
9816 effect(TEMP tmp, KILL cr);
9817
9818 format %{
9819 "cmpxchg_narrow_oop_shenandoah $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval with temp $tmp"
9820 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9821 %}
9822
9823 ins_encode %{
9824 Register tmp = $tmp$$Register;
9825 __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
9826 __ cmpxchg_oop_shenandoah($mem$$base$$Register, tmp, $newval$$Register, Assembler::word, /*acquire*/ true, /*release*/ true, /*weak*/ false);
9827 __ cset($res$$Register, Assembler::EQ);
9828 %}
9829
9830 ins_pipe(pipe_slow);
9831 %}
9832
9833 // ---------------------------------------------------------------------
9834 // Sundry CAS operations. Note that release is always true,
9835 // regardless of the memory ordering of the CAS. This is because we
9836 // need the volatile case to be sequentially consistent but there is
9837 // no trailing StoreLoad barrier emitted by C2. Unfortunately we
9838 // can't check the type of memory ordering here, so we always emit a
9839 // STLXR.
9840
9841 // This section is generated from aarch64_ad_cas.m4
9842
9843
9844 instruct compareAndExchangeB(iRegI_R0 res, indirect mem, iRegI_R2 oldval, iRegI_R3 newval, rFlagsReg cr) %{
9845 match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
9846 ins_cost(2 * VOLATILE_REF_COST);
9889 %}
9890 ins_pipe(pipe_slow);
9891 %}
9892
9893 instruct compareAndExchangeL(iRegL_R0 res, indirect mem, iRegL_R2 oldval, iRegL_R3 newval, rFlagsReg cr) %{
9894 match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
9895 ins_cost(2 * VOLATILE_REF_COST);
9896 effect(KILL cr);
9897 format %{
9898 "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
9899 %}
9900 ins_encode %{
9901 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9902 Assembler::xword, /*acquire*/ false, /*release*/ true,
9903 /*weak*/ false, $res$$Register);
9904 %}
9905 ins_pipe(pipe_slow);
9906 %}
9907
9908 instruct compareAndExchangeN(iRegN_R0 res, indirect mem, iRegN_R2 oldval, iRegN_R3 newval, rFlagsReg cr) %{
9909 predicate(! UseShenandoahGC);
9910 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
9911 ins_cost(2 * VOLATILE_REF_COST);
9912 effect(KILL cr);
9913 format %{
9914 "cmpxchg $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
9915 %}
9916 ins_encode %{
9917 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9918 Assembler::word, /*acquire*/ false, /*release*/ true,
9919 /*weak*/ false, $res$$Register);
9920 %}
9921 ins_pipe(pipe_slow);
9922 %}
9923
9924 instruct compareAndExchangeN_shenandoah(iRegN_R0 res, indirect mem, iRegN_R2 oldval, iRegN_R3 newval, iRegP tmp, rFlagsReg cr) %{
9925 predicate(UseShenandoahGC);
9926 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
9927 ins_cost(3 * VOLATILE_REF_COST);
9928 effect(TEMP tmp, KILL cr);
9929 format %{
9930 "cmpxchg_oop_shenandoah $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
9931 %}
9932 ins_encode %{
9933 Register tmp = $tmp$$Register;
9934 __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
9935 __ cmpxchg_oop_shenandoah($mem$$Register, tmp, $newval$$Register,
9936 Assembler::word, /*acquire*/ false, /*release*/ true,
9937 /*weak*/ false, $res$$Register);
9938 %}
9939 ins_pipe(pipe_slow);
9940 %}
9941
9942 instruct compareAndExchangeP(iRegP_R0 res, indirect mem, iRegP_R2 oldval, iRegP_R3 newval, rFlagsReg cr) %{
9943 predicate(!UseShenandoahGC || n->in(3)->in(1)->bottom_type() == TypePtr::NULL_PTR);
9944 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
9945 ins_cost(2 * VOLATILE_REF_COST);
9946 effect(KILL cr);
9947 format %{
9948 "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
9949 %}
9950 ins_encode %{
9951 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9952 Assembler::xword, /*acquire*/ false, /*release*/ true,
9953 /*weak*/ false, $res$$Register);
9954 %}
9955 ins_pipe(pipe_slow);
9956 %}
9957
9958 instruct compareAndExchangeP_shenandoah(iRegP_R0 res, indirect mem, iRegP_R2 oldval, iRegP_R3 newval, iRegP tmp, rFlagsReg cr) %{
9959 predicate(UseShenandoahGC);
9960 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
9961 ins_cost(3 * VOLATILE_REF_COST);
9962 effect(TEMP tmp, KILL cr);
9963 format %{
9964 "cmpxchg_oop_shenandoah $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval with temp $tmp"
9965 %}
9966 ins_encode %{
9967 Register tmp = $tmp$$Register;
9968 __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
9969 __ cmpxchg_oop_shenandoah($mem$$Register, tmp, $newval$$Register,
9970 Assembler::xword, /*acquire*/ false, /*release*/ true,
9971 /*weak*/ false, $res$$Register);
9972 %}
9973 ins_pipe(pipe_slow);
9974 %}
9975
9976 instruct weakCompareAndSwapB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9977 match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
9978 ins_cost(2 * VOLATILE_REF_COST);
9979 effect(KILL cr);
9980 format %{
9981 "cmpxchg $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
9982 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9983 %}
9984 ins_encode %{
9985 __ uxtbw(rscratch2, $oldval$$Register);
9986 __ cmpxchg($mem$$Register, rscratch2, $newval$$Register,
9987 Assembler::byte, /*acquire*/ false, /*release*/ true,
9988 /*weak*/ true, noreg);
9989 __ csetw($res$$Register, Assembler::EQ);
9990 %}
9991 ins_pipe(pipe_slow);
9992 %}
9993
9994 instruct weakCompareAndSwapS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9995 match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
10027 %}
10028
10029 instruct weakCompareAndSwapL(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
10030 match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
10031 ins_cost(2 * VOLATILE_REF_COST);
10032 effect(KILL cr);
10033 format %{
10034 "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
10035 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
10036 %}
10037 ins_encode %{
10038 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
10039 Assembler::xword, /*acquire*/ false, /*release*/ true,
10040 /*weak*/ true, noreg);
10041 __ csetw($res$$Register, Assembler::EQ);
10042 %}
10043 ins_pipe(pipe_slow);
10044 %}
10045
10046 instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
10047 predicate(! UseShenandoahGC);
10048 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
10049 ins_cost(2 * VOLATILE_REF_COST);
10050 effect(KILL cr);
10051 format %{
10052 "cmpxchg $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
10053 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
10054 %}
10055 ins_encode %{
10056 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
10057 Assembler::word, /*acquire*/ false, /*release*/ true,
10058 /*weak*/ true, noreg);
10059 __ csetw($res$$Register, Assembler::EQ);
10060 %}
10061 ins_pipe(pipe_slow);
10062 %}
10063
10064 instruct weakCompareAndSwapN_shenandoah(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, iRegP tmp, rFlagsReg cr) %{
10065 predicate(UseShenandoahGC);
10066 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
10067 ins_cost(3 * VOLATILE_REF_COST);
10068 effect(TEMP tmp, KILL cr);
10069 format %{
10070 "cmpxchg_oop_shenandoah $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
10071 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
10072 %}
10073 ins_encode %{
10074 Register tmp = $tmp$$Register;
10075 __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
10076 __ cmpxchg_oop_shenandoah($mem$$Register, tmp, $newval$$Register,
10077 Assembler::word, /*acquire*/ false, /*release*/ true,
10078 /*weak*/ true);
10079 __ csetw($res$$Register, Assembler::EQ);
10080 %}
10081 ins_pipe(pipe_slow);
10082 %}
10083
10084 instruct weakCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
10085 predicate(!UseShenandoahGC || n->in(3)->in(1)->bottom_type() == TypePtr::NULL_PTR);
10086 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
10087 ins_cost(2 * VOLATILE_REF_COST);
10088 effect(KILL cr);
10089 format %{
10090 "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
10091 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
10092 %}
10093 ins_encode %{
10094 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
10095 Assembler::xword, /*acquire*/ false, /*release*/ true,
10096 /*weak*/ true, noreg);
10097 __ csetw($res$$Register, Assembler::EQ);
10098 %}
10099 ins_pipe(pipe_slow);
10100 %}
10101
10102 instruct weakCompareAndSwapP_shenandoah(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, iRegP tmp, rFlagsReg cr) %{
10103 predicate(UseShenandoahGC);
10104 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
10105 ins_cost(3 * VOLATILE_REF_COST);
10106 effect(TEMP tmp, KILL cr);
10107 format %{
10108 "cmpxchg_oop_shenandoah $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
10109 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
10110 %}
10111 ins_encode %{
10112 Register tmp = $tmp$$Register;
10113 __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
10114 __ cmpxchg_oop_shenandoah($mem$$Register, tmp, $newval$$Register,
10115 Assembler::xword, /*acquire*/ false, /*release*/ true,
10116 /*weak*/ true);
10117 __ csetw($res$$Register, Assembler::EQ);
10118 %}
10119 ins_pipe(pipe_slow);
10120 %}
10121 // ---------------------------------------------------------------------
10122
10123 instruct get_and_setI(indirect mem, iRegINoSp newv, iRegI prev) %{
10124 match(Set prev (GetAndSetI mem newv));
10125 format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
10126 ins_encode %{
10127 __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
10128 %}
10129 ins_pipe(pipe_serial);
10130 %}
10131
10132 instruct get_and_setL(indirect mem, iRegLNoSp newv, iRegL prev) %{
10133 match(Set prev (GetAndSetL mem newv));
10134 format %{ "atomic_xchg $prev, $newv, [$mem]" %}
10135 ins_encode %{
10136 __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
|