4265
4266 enc_class aarch64_enc_cmpxchg(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
4267 MacroAssembler _masm(&cbuf);
4268 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
4269 __ shenandoah_store_addr_check($mem$$base$$Register);
4270 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
4271 Assembler::xword, /*acquire*/ false, /*release*/ true,
4272 /*weak*/ false, noreg);
4273 %}
4274
4275 enc_class aarch64_enc_cmpxchgw(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
4276 MacroAssembler _masm(&cbuf);
4277 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
4278 __ shenandoah_store_addr_check($mem$$base$$Register);
4279 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
4280 Assembler::word, /*acquire*/ false, /*release*/ true,
4281 /*weak*/ false, noreg);
4282 %}
4283
4284
4285 enc_class aarch64_enc_cmpxchg_oop_shenandoah(memory mem, iRegLNoSp oldval, iRegLNoSp newval, iRegP tmp) %{
4286 MacroAssembler _masm(&cbuf);
4287 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
4288 Register tmp = $tmp$$Register;
4289 __ shenandoah_store_check($mem$$base$$Register, $newval$$Register);
4290 __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
4291 __ cmpxchg_oop_shenandoah($mem$$base$$Register, tmp, $newval$$Register,
4292 Assembler::xword, /*acquire*/ false, /*release*/ true, /*weak*/ false);
4293 %}
4294
4295 // The only difference between aarch64_enc_cmpxchg and
4296 // aarch64_enc_cmpxchg_acq is that we use load-acquire in the
4297 // CompareAndSwap sequence to serve as a barrier on acquiring a
4298 // lock.
4299 enc_class aarch64_enc_cmpxchg_acq(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
4300 MacroAssembler _masm(&cbuf);
4301 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
4302 __ shenandoah_store_addr_check($mem$$base$$Register);
4303 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
4304 Assembler::xword, /*acquire*/ true, /*release*/ true,
4305 /*weak*/ false, noreg);
4306 %}
4307
4308 enc_class aarch64_enc_cmpxchgw_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
4309 MacroAssembler _masm(&cbuf);
4310 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
4311 __ shenandoah_store_addr_check($mem$$base$$Register);
4312 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
4313 Assembler::word, /*acquire*/ true, /*release*/ true,
4314 /*weak*/ false, noreg);
4315 %}
4316
4317
4318 enc_class aarch64_enc_cmpxchg_acq_oop_shenandoah(memory mem, iRegLNoSp oldval, iRegLNoSp newval, iRegP tmp) %{
4319 MacroAssembler _masm(&cbuf);
4320 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
4321 Register tmp = $tmp$$Register;
4322 __ shenandoah_store_check($mem$$base$$Register, $newval$$Register);
4323 __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
4324 __ cmpxchg_oop_shenandoah($mem$$base$$Register, tmp, $newval$$Register,
4325 Assembler::xword, /*acquire*/ true, /*release*/ true, /*weak*/ false);
4326 %}
4327
4328 // auxiliary used for CompareAndSwapX to set result register
4329 enc_class aarch64_enc_cset_eq(iRegINoSp res) %{
4330 MacroAssembler _masm(&cbuf);
4331 Register res_reg = as_Register($res$$reg);
4332 __ cset(res_reg, Assembler::EQ);
4333 %}
4334
4335 // prefetch encodings
4336
4337 enc_class aarch64_enc_prefetchw(memory mem) %{
4338 MacroAssembler _masm(&cbuf);
4339 Register base = as_Register($mem$$base);
4340 int index = $mem$$index;
4341 int scale = $mem$$scale;
4342 int disp = $mem$$disp;
4343 if (index == -1) {
4344 __ prfm(Address(base, disp), PSTL1KEEP);
9632 %}
9633
9634 instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
9635
9636 predicate(!UseShenandoahGC || n->in(3)->in(1)->bottom_type() == TypePtr::NULL_PTR);
9637 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
9638 ins_cost(2 * VOLATILE_REF_COST);
9639
9640 effect(KILL cr);
9641
9642 format %{
9643 "cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
9644 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9645 %}
9646
9647 ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
9648 aarch64_enc_cset_eq(res));
9649
9650 ins_pipe(pipe_slow);
9651 %}
9652 instruct compareAndSwapP_shenandoah(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, iRegP tmp, rFlagsReg cr) %{
9653
9654 predicate(UseShenandoahGC);
9655 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
9656 ins_cost(3 * VOLATILE_REF_COST);
9657
9658 effect(TEMP tmp, KILL cr);
9659
9660 format %{
9661 "cmpxchg_oop_shenandoah $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval with temp $tmp"
9662 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9663 %}
9664
9665 ins_encode(aarch64_enc_cmpxchg_oop_shenandoah(mem, oldval, newval, tmp),
9666 aarch64_enc_cset_eq(res));
9667
9668 ins_pipe(pipe_slow);
9669 %}
9670
9671 instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
9672
9673 predicate(!UseShenandoahGC);
9674 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
9675 ins_cost(2 * VOLATILE_REF_COST);
9676
9677 effect(KILL cr);
9678
9679 format %{
9680 "cmpxchgw $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
9681 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9682 %}
9683
9684 ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
9685 aarch64_enc_cset_eq(res));
9686
9687 ins_pipe(pipe_slow);
9688 %}
9689
9690 instruct compareAndSwapN_shenandoah(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, iRegP tmp, rFlagsReg cr) %{
9691
9692 predicate(UseShenandoahGC);
9693 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
9694 ins_cost(3 * VOLATILE_REF_COST);
9695
9696 effect(TEMP tmp, KILL cr);
9697
9698 format %{
9699 "cmpxchg_narrow_oop_shenandoah $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval with temp $tmp"
9700 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9701 %}
9702
9703 ins_encode %{
9704 Register tmp = $tmp$$Register;
9705 __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
9706 __ cmpxchg_oop_shenandoah($mem$$base$$Register, tmp, $newval$$Register, Assembler::word, /*acquire*/ false, /*release*/ true, /*weak*/ false);
9707 __ cset($res$$Register, Assembler::EQ);
9708 %}
9709
9710 ins_pipe(pipe_slow);
9711 %}
9712
9713 // alternative CompareAndSwapX when we are eliding barriers
9714
9715 instruct compareAndSwapIAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
9716
9717 predicate(needs_acquiring_load_exclusive(n));
9718 match(Set res (CompareAndSwapI mem (Binary oldval newval)));
9719 ins_cost(VOLATILE_REF_COST);
9720
9721 effect(KILL cr);
9722
9723 format %{
9724 "cmpxchgw_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
9725 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9726 %}
9752
9753 instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
9754
9755 predicate(needs_acquiring_load_exclusive(n) && (!UseShenandoahGC || n->in(3)->in(1)->bottom_type() == TypePtr::NULL_PTR));
9756 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
9757 ins_cost(VOLATILE_REF_COST);
9758
9759 effect(KILL cr);
9760
9761 format %{
9762 "cmpxchg_acq $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
9763 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9764 %}
9765
9766 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
9767 aarch64_enc_cset_eq(res));
9768
9769 ins_pipe(pipe_slow);
9770 %}
9771
9772 instruct compareAndSwapPAcq_shenandoah(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, iRegP tmp, rFlagsReg cr) %{
9773
9774 predicate(needs_acquiring_load_exclusive(n) && UseShenandoahGC);
9775 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
9776 ins_cost(2 * VOLATILE_REF_COST);
9777
9778 effect(TEMP tmp, KILL cr);
9779
9780 format %{
9781 "cmpxchg_acq_oop_shenandoah $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval with temp $tmp"
9782 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9783 %}
9784
9785 ins_encode(aarch64_enc_cmpxchg_acq_oop_shenandoah(mem, oldval, newval, tmp),
9786 aarch64_enc_cset_eq(res));
9787
9788 ins_pipe(pipe_slow);
9789 %}
9790
9791 instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
9792
9793 predicate(needs_acquiring_load_exclusive(n) && ! UseShenandoahGC);
9794 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
9795 ins_cost(VOLATILE_REF_COST);
9796
9797 effect(KILL cr);
9798
9799 format %{
9800 "cmpxchgw_acq $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
9801 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9802 %}
9803
9804 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
9805 aarch64_enc_cset_eq(res));
9806
9807 ins_pipe(pipe_slow);
9808 %}
9809
9810 instruct compareAndSwapNAcq_shenandoah(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, iRegP tmp, rFlagsReg cr) %{
9811
9812 predicate(needs_acquiring_load_exclusive(n) && UseShenandoahGC);
9813 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
9814 ins_cost(3 * VOLATILE_REF_COST);
9815
9816 effect(TEMP tmp, KILL cr);
9817
9818 format %{
9819 "cmpxchg_narrow_oop_shenandoah $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval with temp $tmp"
9820 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9821 %}
9822
9823 ins_encode %{
9824 Register tmp = $tmp$$Register;
9825 __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
9826 __ cmpxchg_oop_shenandoah($mem$$base$$Register, tmp, $newval$$Register, Assembler::word, /*acquire*/ true, /*release*/ true, /*weak*/ false);
9827 __ cset($res$$Register, Assembler::EQ);
9828 %}
9829
9830 ins_pipe(pipe_slow);
9831 %}
9832
9833 // ---------------------------------------------------------------------
9834 // Sundry CAS operations. Note that release is always true,
9835 // regardless of the memory ordering of the CAS. This is because we
9836 // need the volatile case to be sequentially consistent but there is
9837 // no trailing StoreLoad barrier emitted by C2. Unfortunately we
9838 // can't check the type of memory ordering here, so we always emit a
9839 // STLXR.
9840
9841 // This section is generated from aarch64_ad_cas.m4
9842
9843
9844 instruct compareAndExchangeB(iRegI_R0 res, indirect mem, iRegI_R2 oldval, iRegI_R3 newval, rFlagsReg cr) %{
9845 match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
9846 ins_cost(2 * VOLATILE_REF_COST);
9847 effect(KILL cr);
9848 format %{
9849 "cmpxchg $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
9850 %}
9851 ins_encode %{
9852 __ uxtbw(rscratch2, $oldval$$Register);
9853 __ cmpxchg($mem$$Register, rscratch2, $newval$$Register,
9854 Assembler::byte, /*acquire*/ false, /*release*/ true,
9855 /*weak*/ false, $res$$Register);
9856 __ sxtbw($res$$Register, $res$$Register);
9857 %}
9858 ins_pipe(pipe_slow);
9859 %}
9860
9861 instruct compareAndExchangeS(iRegI_R0 res, indirect mem, iRegI_R2 oldval, iRegI_R3 newval, rFlagsReg cr) %{
9862 match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
9863 ins_cost(2 * VOLATILE_REF_COST);
9864 effect(KILL cr);
9865 format %{
9866 "cmpxchg $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
9867 %}
9868 ins_encode %{
9869 __ uxthw(rscratch2, $oldval$$Register);
9870 __ cmpxchg($mem$$Register, rscratch2, $newval$$Register,
9871 Assembler::halfword, /*acquire*/ false, /*release*/ true,
9872 /*weak*/ false, $res$$Register);
9873 __ sxthw($res$$Register, $res$$Register);
9874 %}
9875 ins_pipe(pipe_slow);
9876 %}
9877
9878 instruct compareAndExchangeI(iRegI_R0 res, indirect mem, iRegI_R2 oldval, iRegI_R3 newval, rFlagsReg cr) %{
9879 match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
9880 ins_cost(2 * VOLATILE_REF_COST);
9881 effect(KILL cr);
9882 format %{
9883 "cmpxchg $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
9884 %}
9885 ins_encode %{
9886 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9887 Assembler::word, /*acquire*/ false, /*release*/ true,
9888 /*weak*/ false, $res$$Register);
9889 %}
9890 ins_pipe(pipe_slow);
9891 %}
9892
9893 instruct compareAndExchangeL(iRegL_R0 res, indirect mem, iRegL_R2 oldval, iRegL_R3 newval, rFlagsReg cr) %{
9894 match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
9895 ins_cost(2 * VOLATILE_REF_COST);
9896 effect(KILL cr);
9897 format %{
9898 "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
9899 %}
9900 ins_encode %{
9901 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9902 Assembler::xword, /*acquire*/ false, /*release*/ true,
9903 /*weak*/ false, $res$$Register);
9904 %}
9905 ins_pipe(pipe_slow);
9906 %}
9907
9908 instruct compareAndExchangeN(iRegN_R0 res, indirect mem, iRegN_R2 oldval, iRegN_R3 newval, rFlagsReg cr) %{
9909 predicate(! UseShenandoahGC);
9910 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
9911 ins_cost(2 * VOLATILE_REF_COST);
9912 effect(KILL cr);
9913 format %{
9914 "cmpxchg $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
9915 %}
9916 ins_encode %{
9917 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9918 Assembler::word, /*acquire*/ false, /*release*/ true,
9919 /*weak*/ false, $res$$Register);
9920 %}
9921 ins_pipe(pipe_slow);
9922 %}
9923
9924 instruct compareAndExchangeN_shenandoah(iRegN_R0 res, indirect mem, iRegN_R2 oldval, iRegN_R3 newval, iRegP tmp, rFlagsReg cr) %{
9925 predicate(UseShenandoahGC);
9926 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
9927 ins_cost(3 * VOLATILE_REF_COST);
9928 effect(TEMP tmp, KILL cr);
9929 format %{
9930 "cmpxchg_oop_shenandoah $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
9931 %}
9932 ins_encode %{
9933 Register tmp = $tmp$$Register;
9934 __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
9935 __ cmpxchg_oop_shenandoah($mem$$Register, tmp, $newval$$Register,
9936 Assembler::word, /*acquire*/ false, /*release*/ true,
9937 /*weak*/ false, $res$$Register);
9938 %}
9939 ins_pipe(pipe_slow);
9940 %}
9941
9942 instruct compareAndExchangeP(iRegP_R0 res, indirect mem, iRegP_R2 oldval, iRegP_R3 newval, rFlagsReg cr) %{
9943 predicate(!UseShenandoahGC || n->in(3)->in(1)->bottom_type() == TypePtr::NULL_PTR);
9944 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
9945 ins_cost(2 * VOLATILE_REF_COST);
9946 effect(KILL cr);
9947 format %{
9948 "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
9949 %}
9950 ins_encode %{
9951 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9952 Assembler::xword, /*acquire*/ false, /*release*/ true,
9953 /*weak*/ false, $res$$Register);
9954 %}
9955 ins_pipe(pipe_slow);
9956 %}
9957
9958 instruct compareAndExchangeP_shenandoah(iRegP_R0 res, indirect mem, iRegP_R2 oldval, iRegP_R3 newval, iRegP tmp, rFlagsReg cr) %{
9959 predicate(UseShenandoahGC);
9960 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
9961 ins_cost(3 * VOLATILE_REF_COST);
9962 effect(TEMP tmp, KILL cr);
9963 format %{
9964 "cmpxchg_oop_shenandoah $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval with temp $tmp"
9965 %}
9966 ins_encode %{
9967 Register tmp = $tmp$$Register;
9968 __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
9969 __ cmpxchg_oop_shenandoah($mem$$Register, tmp, $newval$$Register,
9970 Assembler::xword, /*acquire*/ false, /*release*/ true,
9971 /*weak*/ false, $res$$Register);
9972 %}
9973 ins_pipe(pipe_slow);
9974 %}
9975
9976 instruct weakCompareAndSwapB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9977 match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
9978 ins_cost(2 * VOLATILE_REF_COST);
9979 effect(KILL cr);
9980 format %{
9981 "cmpxchg $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
9982 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9983 %}
9984 ins_encode %{
9985 __ uxtbw(rscratch2, $oldval$$Register);
9986 __ cmpxchg($mem$$Register, rscratch2, $newval$$Register,
9987 Assembler::byte, /*acquire*/ false, /*release*/ true,
9988 /*weak*/ true, noreg);
9989 __ csetw($res$$Register, Assembler::EQ);
9990 %}
9991 ins_pipe(pipe_slow);
10044 %}
10045
10046 instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
10047 predicate(! UseShenandoahGC);
10048 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
10049 ins_cost(2 * VOLATILE_REF_COST);
10050 effect(KILL cr);
10051 format %{
10052 "cmpxchg $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
10053 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
10054 %}
10055 ins_encode %{
10056 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
10057 Assembler::word, /*acquire*/ false, /*release*/ true,
10058 /*weak*/ true, noreg);
10059 __ csetw($res$$Register, Assembler::EQ);
10060 %}
10061 ins_pipe(pipe_slow);
10062 %}
10063
10064 instruct weakCompareAndSwapN_shenandoah(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, iRegP tmp, rFlagsReg cr) %{
10065 predicate(UseShenandoahGC);
10066 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
10067 ins_cost(3 * VOLATILE_REF_COST);
10068 effect(TEMP tmp, KILL cr);
10069 format %{
10070 "cmpxchg_oop_shenandoah $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
10071 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
10072 %}
10073 ins_encode %{
10074 Register tmp = $tmp$$Register;
10075 __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
10076 __ cmpxchg_oop_shenandoah($mem$$Register, tmp, $newval$$Register,
10077 Assembler::word, /*acquire*/ false, /*release*/ true,
10078 /*weak*/ true);
10079 __ csetw($res$$Register, Assembler::EQ);
10080 %}
10081 ins_pipe(pipe_slow);
10082 %}
10083
10084 instruct weakCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
10085 predicate(!UseShenandoahGC || n->in(3)->in(1)->bottom_type() == TypePtr::NULL_PTR);
10086 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
10087 ins_cost(2 * VOLATILE_REF_COST);
10088 effect(KILL cr);
10089 format %{
10090 "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
10091 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
10092 %}
10093 ins_encode %{
10094 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
10095 Assembler::xword, /*acquire*/ false, /*release*/ true,
10096 /*weak*/ true, noreg);
10097 __ csetw($res$$Register, Assembler::EQ);
10098 %}
10099 ins_pipe(pipe_slow);
10100 %}
10101
10102 instruct weakCompareAndSwapP_shenandoah(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, iRegP tmp, rFlagsReg cr) %{
10103 predicate(UseShenandoahGC);
10104 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
10105 ins_cost(3 * VOLATILE_REF_COST);
10106 effect(TEMP tmp, KILL cr);
10107 format %{
10108 "cmpxchg_oop_shenandoah $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
10109 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
10110 %}
10111 ins_encode %{
10112 Register tmp = $tmp$$Register;
10113 __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
10114 __ cmpxchg_oop_shenandoah($mem$$Register, tmp, $newval$$Register,
10115 Assembler::xword, /*acquire*/ false, /*release*/ true,
10116 /*weak*/ true);
10117 __ csetw($res$$Register, Assembler::EQ);
10118 %}
10119 ins_pipe(pipe_slow);
10120 %}
10121 // ---------------------------------------------------------------------
10122
10123 instruct get_and_setI(indirect mem, iRegINoSp newv, iRegI prev) %{
10124 match(Set prev (GetAndSetI mem newv));
10125 format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
10126 ins_encode %{
10127 __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
10128 %}
10129 ins_pipe(pipe_serial);
10130 %}
10131
10132 instruct get_and_setL(indirect mem, iRegLNoSp newv, iRegL prev) %{
10133 match(Set prev (GetAndSetL mem newv));
10134 format %{ "atomic_xchg $prev, $newv, [$mem]" %}
10135 ins_encode %{
10136 __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
|
4265
4266 enc_class aarch64_enc_cmpxchg(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
4267 MacroAssembler _masm(&cbuf);
4268 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
4269 __ shenandoah_store_addr_check($mem$$base$$Register);
4270 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
4271 Assembler::xword, /*acquire*/ false, /*release*/ true,
4272 /*weak*/ false, noreg);
4273 %}
4274
4275 enc_class aarch64_enc_cmpxchgw(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
4276 MacroAssembler _masm(&cbuf);
4277 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
4278 __ shenandoah_store_addr_check($mem$$base$$Register);
4279 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
4280 Assembler::word, /*acquire*/ false, /*release*/ true,
4281 /*weak*/ false, noreg);
4282 %}
4283
4284
4285 enc_class aarch64_enc_cmpxchg_oop_shenandoah(memory mem, iRegP oldval, iRegP newval, iRegPNoSp tmp) %{
4286 MacroAssembler _masm(&cbuf);
4287 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
4288 Register tmp = $tmp$$Register;
4289 __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
4290 __ cmpxchg_oop_shenandoah($mem$$Register, tmp, $newval$$Register,
4291 Assembler::xword, /*acquire*/ false, /*release*/ true, /*weak*/ false);
4292 %}
4293
4294 // The only difference between aarch64_enc_cmpxchg and
4295 // aarch64_enc_cmpxchg_acq is that we use load-acquire in the
4296 // CompareAndSwap sequence to serve as a barrier on acquiring a
4297 // lock.
4298 enc_class aarch64_enc_cmpxchg_acq(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
4299 MacroAssembler _masm(&cbuf);
4300 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
4301 __ shenandoah_store_addr_check($mem$$base$$Register);
4302 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
4303 Assembler::xword, /*acquire*/ true, /*release*/ true,
4304 /*weak*/ false, noreg);
4305 %}
4306
4307 enc_class aarch64_enc_cmpxchgw_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
4308 MacroAssembler _masm(&cbuf);
4309 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
4310 __ shenandoah_store_addr_check($mem$$base$$Register);
4311 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
4312 Assembler::word, /*acquire*/ true, /*release*/ true,
4313 /*weak*/ false, noreg);
4314 %}
4315
4316
4317 enc_class aarch64_enc_cmpxchg_acq_oop_shenandoah(memory mem, iRegP oldval, iRegP newval, iRegPNoSp tmp) %{
4318 MacroAssembler _masm(&cbuf);
4319 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
4320 Register tmp = $tmp$$Register;
4321 __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
4322 __ cmpxchg_oop_shenandoah($mem$$Register, tmp, $newval$$Register,
4323 Assembler::xword, /*acquire*/ true, /*release*/ true, /*weak*/ false);
4324 %}
4325
4326 // auxiliary used for CompareAndSwapX to set result register
4327 enc_class aarch64_enc_cset_eq(iRegINoSp res) %{
4328 MacroAssembler _masm(&cbuf);
4329 Register res_reg = as_Register($res$$reg);
4330 __ cset(res_reg, Assembler::EQ);
4331 %}
4332
4333 // prefetch encodings
4334
4335 enc_class aarch64_enc_prefetchw(memory mem) %{
4336 MacroAssembler _masm(&cbuf);
4337 Register base = as_Register($mem$$base);
4338 int index = $mem$$index;
4339 int scale = $mem$$scale;
4340 int disp = $mem$$disp;
4341 if (index == -1) {
4342 __ prfm(Address(base, disp), PSTL1KEEP);
9630 %}
9631
9632 instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
9633
9634 predicate(!UseShenandoahGC || n->in(3)->in(1)->bottom_type() == TypePtr::NULL_PTR);
9635 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
9636 ins_cost(2 * VOLATILE_REF_COST);
9637
9638 effect(KILL cr);
9639
9640 format %{
9641 "cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
9642 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9643 %}
9644
9645 ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
9646 aarch64_enc_cset_eq(res));
9647
9648 ins_pipe(pipe_slow);
9649 %}
9650 instruct compareAndSwapP_shenandoah(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp, rFlagsReg cr) %{
9651
9652 predicate(UseShenandoahGC);
9653 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
9654 ins_cost(3 * VOLATILE_REF_COST);
9655
9656 effect(TEMP tmp, KILL cr);
9657
9658 format %{
9659 "cmpxchg_oop_shenandoah $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval with temp $tmp"
9660 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9661 %}
9662
9663 ins_encode(aarch64_enc_cmpxchg_oop_shenandoah(mem, oldval, newval, tmp),
9664 aarch64_enc_cset_eq(res));
9665
9666 ins_pipe(pipe_slow);
9667 %}
9668
9669 instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
9670
9671 predicate(!UseShenandoahGC);
9672 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
9673 ins_cost(2 * VOLATILE_REF_COST);
9674
9675 effect(KILL cr);
9676
9677 format %{
9678 "cmpxchgw $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
9679 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9680 %}
9681
9682 ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
9683 aarch64_enc_cset_eq(res));
9684
9685 ins_pipe(pipe_slow);
9686 %}
9687
9688 instruct compareAndSwapN_shenandoah(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, iRegNNoSp tmp, rFlagsReg cr) %{
9689
9690 predicate(UseShenandoahGC);
9691 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
9692 ins_cost(3 * VOLATILE_REF_COST);
9693
9694 effect(TEMP tmp, KILL cr);
9695
9696 format %{
9697 "cmpxchg_narrow_oop_shenandoah $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval with temp $tmp"
9698 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9699 %}
9700
9701 ins_encode %{
9702 Register tmp = $tmp$$Register;
9703 __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
9704 __ cmpxchg_oop_shenandoah($mem$$Register, tmp, $newval$$Register, Assembler::word, /*acquire*/ false, /*release*/ true, /*weak*/ false);
9705 __ cset($res$$Register, Assembler::EQ);
9706 %}
9707
9708 ins_pipe(pipe_slow);
9709 %}
9710
9711 // alternative CompareAndSwapX when we are eliding barriers
9712
9713 instruct compareAndSwapIAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
9714
9715 predicate(needs_acquiring_load_exclusive(n));
9716 match(Set res (CompareAndSwapI mem (Binary oldval newval)));
9717 ins_cost(VOLATILE_REF_COST);
9718
9719 effect(KILL cr);
9720
9721 format %{
9722 "cmpxchgw_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
9723 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9724 %}
9750
9751 instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
9752
9753 predicate(needs_acquiring_load_exclusive(n) && (!UseShenandoahGC || n->in(3)->in(1)->bottom_type() == TypePtr::NULL_PTR));
9754 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
9755 ins_cost(VOLATILE_REF_COST);
9756
9757 effect(KILL cr);
9758
9759 format %{
9760 "cmpxchg_acq $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
9761 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9762 %}
9763
9764 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
9765 aarch64_enc_cset_eq(res));
9766
9767 ins_pipe(pipe_slow);
9768 %}
9769
9770 instruct compareAndSwapPAcq_shenandoah(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp, rFlagsReg cr) %{
9771
9772 predicate(needs_acquiring_load_exclusive(n) && UseShenandoahGC);
9773 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
9774 ins_cost(2 * VOLATILE_REF_COST);
9775
9776 effect(TEMP tmp, KILL cr);
9777
9778 format %{
9779 "cmpxchg_acq_oop_shenandoah $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval with temp $tmp"
9780 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9781 %}
9782
9783 ins_encode(aarch64_enc_cmpxchg_acq_oop_shenandoah(mem, oldval, newval, tmp),
9784 aarch64_enc_cset_eq(res));
9785
9786 ins_pipe(pipe_slow);
9787 %}
9788
9789 instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
9790
9791 predicate(needs_acquiring_load_exclusive(n) && ! UseShenandoahGC);
9792 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
9793 ins_cost(VOLATILE_REF_COST);
9794
9795 effect(KILL cr);
9796
9797 format %{
9798 "cmpxchgw_acq $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
9799 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9800 %}
9801
9802 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
9803 aarch64_enc_cset_eq(res));
9804
9805 ins_pipe(pipe_slow);
9806 %}
9807
9808 instruct compareAndSwapNAcq_shenandoah(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, iRegNNoSp tmp, rFlagsReg cr) %{
9809
9810 predicate(needs_acquiring_load_exclusive(n) && UseShenandoahGC);
9811 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
9812 ins_cost(3 * VOLATILE_REF_COST);
9813
9814 effect(TEMP tmp, KILL cr);
9815
9816 format %{
9817 "cmpxchg_narrow_oop_shenandoah $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval with temp $tmp"
9818 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9819 %}
9820
9821 ins_encode %{
9822 Register tmp = $tmp$$Register;
9823 __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
9824 __ cmpxchg_oop_shenandoah($mem$$Register, tmp, $newval$$Register, Assembler::word, /*acquire*/ true, /*release*/ true, /*weak*/ false);
9825 __ cset($res$$Register, Assembler::EQ);
9826 %}
9827
9828 ins_pipe(pipe_slow);
9829 %}
9830
9831 // ---------------------------------------------------------------------
9832 // Sundry CAS operations. Note that release is always true,
9833 // regardless of the memory ordering of the CAS. This is because we
9834 // need the volatile case to be sequentially consistent but there is
9835 // no trailing StoreLoad barrier emitted by C2. Unfortunately we
9836 // can't check the type of memory ordering here, so we always emit a
9837 // STLXR.
9838
9839 // This section is generated from aarch64_ad_cas.m4
9840
9841
9842 instruct compareAndExchangeB(iRegI_R0 res, indirect mem, iRegI_R2 oldval, iRegI_R3 newval, rFlagsReg cr) %{
9843 match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
9844 ins_cost(2 * VOLATILE_REF_COST);
9845 effect(TEMP_DEF res, KILL cr);
9846 format %{
9847 "cmpxchg $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
9848 %}
9849 ins_encode %{
9850 __ uxtbw(rscratch2, $oldval$$Register);
9851 __ cmpxchg($mem$$Register, rscratch2, $newval$$Register,
9852 Assembler::byte, /*acquire*/ false, /*release*/ true,
9853 /*weak*/ false, $res$$Register);
9854 __ sxtbw($res$$Register, $res$$Register);
9855 %}
9856 ins_pipe(pipe_slow);
9857 %}
9858
9859 instruct compareAndExchangeS(iRegI_R0 res, indirect mem, iRegI_R2 oldval, iRegI_R3 newval, rFlagsReg cr) %{
9860 match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
9861 ins_cost(2 * VOLATILE_REF_COST);
9862 effect(TEMP_DEF res, KILL cr);
9863 format %{
9864 "cmpxchg $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
9865 %}
9866 ins_encode %{
9867 __ uxthw(rscratch2, $oldval$$Register);
9868 __ cmpxchg($mem$$Register, rscratch2, $newval$$Register,
9869 Assembler::halfword, /*acquire*/ false, /*release*/ true,
9870 /*weak*/ false, $res$$Register);
9871 __ sxthw($res$$Register, $res$$Register);
9872 %}
9873 ins_pipe(pipe_slow);
9874 %}
9875
9876 instruct compareAndExchangeI(iRegI_R0 res, indirect mem, iRegI_R2 oldval, iRegI_R3 newval, rFlagsReg cr) %{
9877 match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
9878 ins_cost(2 * VOLATILE_REF_COST);
9879 effect(TEMP_DEF res, KILL cr);
9880 format %{
9881 "cmpxchg $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
9882 %}
9883 ins_encode %{
9884 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9885 Assembler::word, /*acquire*/ false, /*release*/ true,
9886 /*weak*/ false, $res$$Register);
9887 %}
9888 ins_pipe(pipe_slow);
9889 %}
9890
9891 instruct compareAndExchangeL(iRegL_R0 res, indirect mem, iRegL_R2 oldval, iRegL_R3 newval, rFlagsReg cr) %{
9892 match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
9893 ins_cost(2 * VOLATILE_REF_COST);
9894 effect(TEMP_DEF res, KILL cr);
9895 format %{
9896 "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
9897 %}
9898 ins_encode %{
9899 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9900 Assembler::xword, /*acquire*/ false, /*release*/ true,
9901 /*weak*/ false, $res$$Register);
9902 %}
9903 ins_pipe(pipe_slow);
9904 %}
9905
9906 instruct compareAndExchangeN(iRegN_R0 res, indirect mem, iRegN_R2 oldval, iRegN_R3 newval, rFlagsReg cr) %{
9907 predicate(! UseShenandoahGC);
9908 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
9909 ins_cost(2 * VOLATILE_REF_COST);
9910 effect(TEMP_DEF res, KILL cr);
9911 format %{
9912 "cmpxchg $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
9913 %}
9914 ins_encode %{
9915 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9916 Assembler::word, /*acquire*/ false, /*release*/ true,
9917 /*weak*/ false, $res$$Register);
9918 %}
9919 ins_pipe(pipe_slow);
9920 %}
9921
9922 instruct compareAndExchangeN_shenandoah(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, iRegNNoSp tmp, rFlagsReg cr) %{
9923 predicate(UseShenandoahGC);
9924 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
9925 ins_cost(3 * VOLATILE_REF_COST);
9926 effect(TEMP_DEF res, TEMP tmp, KILL cr);
9927 format %{
9928 "cmpxchg_oop_shenandoah $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
9929 %}
9930 ins_encode %{
9931 Register tmp = $tmp$$Register;
9932 __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
9933 __ cmpxchg_oop_shenandoah($mem$$Register, tmp, $newval$$Register,
9934 Assembler::word, /*acquire*/ false, /*release*/ true, /*weak*/ false, $res$$Register);
9935 %}
9936 ins_pipe(pipe_slow);
9937 %}
9938
9939 instruct compareAndExchangeP(iRegP_R0 res, indirect mem, iRegP_R2 oldval, iRegP_R3 newval, rFlagsReg cr) %{
9940 predicate(!UseShenandoahGC || n->in(3)->in(1)->bottom_type() == TypePtr::NULL_PTR);
9941 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
9942 ins_cost(2 * VOLATILE_REF_COST);
9943 effect(TEMP_DEF res, KILL cr);
9944 format %{
9945 "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
9946 %}
9947 ins_encode %{
9948 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9949 Assembler::xword, /*acquire*/ false, /*release*/ true,
9950 /*weak*/ false, $res$$Register);
9951 %}
9952 ins_pipe(pipe_slow);
9953 %}
9954
9955 instruct compareAndExchangeP_shenandoah(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp, rFlagsReg cr) %{
9956 predicate(UseShenandoahGC);
9957 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
9958 ins_cost(3 * VOLATILE_REF_COST);
9959 effect(TEMP_DEF res, TEMP tmp, KILL cr);
9960 format %{
9961 "cmpxchg_oop_shenandoah $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval with temp $tmp"
9962 %}
9963 ins_encode %{
9964 Register tmp = $tmp$$Register;
9965 __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
9966 __ cmpxchg_oop_shenandoah($mem$$Register, tmp, $newval$$Register,
9967 Assembler::xword, /*acquire*/ false, /*release*/ true, /*weak*/ false, $res$$Register);
9968 %}
9969 ins_pipe(pipe_slow);
9970 %}
9971
9972 instruct weakCompareAndSwapB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9973 match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
9974 ins_cost(2 * VOLATILE_REF_COST);
9975 effect(KILL cr);
9976 format %{
9977 "cmpxchg $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
9978 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9979 %}
9980 ins_encode %{
9981 __ uxtbw(rscratch2, $oldval$$Register);
9982 __ cmpxchg($mem$$Register, rscratch2, $newval$$Register,
9983 Assembler::byte, /*acquire*/ false, /*release*/ true,
9984 /*weak*/ true, noreg);
9985 __ csetw($res$$Register, Assembler::EQ);
9986 %}
9987 ins_pipe(pipe_slow);
10040 %}
10041
10042 instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
10043 predicate(! UseShenandoahGC);
10044 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
10045 ins_cost(2 * VOLATILE_REF_COST);
10046 effect(KILL cr);
10047 format %{
10048 "cmpxchg $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
10049 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
10050 %}
10051 ins_encode %{
10052 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
10053 Assembler::word, /*acquire*/ false, /*release*/ true,
10054 /*weak*/ true, noreg);
10055 __ csetw($res$$Register, Assembler::EQ);
10056 %}
10057 ins_pipe(pipe_slow);
10058 %}
10059
10060 instruct weakCompareAndSwapN_shenandoah(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, iRegNNoSp tmp, rFlagsReg cr) %{
10061 predicate(UseShenandoahGC);
10062 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
10063 ins_cost(3 * VOLATILE_REF_COST);
10064 effect(TEMP tmp, KILL cr);
10065 format %{
10066 "cmpxchg_oop_shenandoah $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
10067 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
10068 %}
10069 ins_encode %{
10070 Register tmp = $tmp$$Register;
10071 __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
10072 __ cmpxchg_oop_shenandoah($mem$$Register, tmp, $newval$$Register,
10073 Assembler::word, /*acquire*/ false, /*release*/ true, /*weak*/ true);
10074 __ csetw($res$$Register, Assembler::EQ);
10075 %}
10076 ins_pipe(pipe_slow);
10077 %}
10078
10079 instruct weakCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
10080 predicate(!UseShenandoahGC || n->in(3)->in(1)->bottom_type() == TypePtr::NULL_PTR);
10081 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
10082 ins_cost(2 * VOLATILE_REF_COST);
10083 effect(KILL cr);
10084 format %{
10085 "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
10086 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
10087 %}
10088 ins_encode %{
10089 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
10090 Assembler::xword, /*acquire*/ false, /*release*/ true,
10091 /*weak*/ true, noreg);
10092 __ csetw($res$$Register, Assembler::EQ);
10093 %}
10094 ins_pipe(pipe_slow);
10095 %}
10096
10097 instruct weakCompareAndSwapP_shenandoah(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp, rFlagsReg cr) %{
10098 predicate(UseShenandoahGC);
10099 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
10100 ins_cost(3 * VOLATILE_REF_COST);
10101 effect(TEMP tmp, KILL cr);
10102 format %{
10103 "cmpxchg_oop_shenandoah $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
10104 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
10105 %}
10106 ins_encode %{
10107 Register tmp = $tmp$$Register;
10108 __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
10109 __ cmpxchg_oop_shenandoah($mem$$Register, tmp, $newval$$Register,
10110 Assembler::xword, /*acquire*/ false, /*release*/ true, /*weak*/ true);
10111 __ csetw($res$$Register, Assembler::EQ);
10112 %}
10113 ins_pipe(pipe_slow);
10114 %}
10115 // ---------------------------------------------------------------------
10116
10117 instruct get_and_setI(indirect mem, iRegINoSp newv, iRegI prev) %{
10118 match(Set prev (GetAndSetI mem newv));
10119 format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
10120 ins_encode %{
10121 __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
10122 %}
10123 ins_pipe(pipe_serial);
10124 %}
10125
10126 instruct get_and_setL(indirect mem, iRegLNoSp newv, iRegL prev) %{
10127 match(Set prev (GetAndSetL mem newv));
10128 format %{ "atomic_xchg $prev, $newv, [$mem]" %}
10129 ins_encode %{
10130 __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
|