9350 ins_cost(INSN_COST);
9351 format %{ "mov dst, $src\t# compressed ptr -> int" %}
9352 ins_encode %{
9353 __ movw($dst$$Register, $src$$Register);
9354 %}
9355
9356 ins_pipe(ialu_reg);
9357 %}
9358
9359 instruct shenandoahRB(iRegPNoSp dst, iRegP src, rFlagsReg cr) %{
9360 match(Set dst (ShenandoahReadBarrier src));
9361 format %{ "shenandoah_rb $dst,$src" %}
9362 ins_encode %{
9363 Register s = $src$$Register;
9364 Register d = $dst$$Register;
9365 __ ldr(d, Address(s, BrooksPointer::byte_offset()));
9366 %}
9367 ins_pipe(pipe_class_memory);
9368 %}
9369
9370 instruct shenandoahWB(iRegP_R0 dst, iRegP src, rFlagsReg cr) %{
9371 match(Set dst (ShenandoahWriteBarrier src));
9372 effect(KILL cr);
9373
9374 format %{ "shenandoah_wb $dst,$src" %}
9375 ins_encode %{
9376 Label done;
9377 Register s = $src$$Register;
9378 Register d = $dst$$Register;
9379 assert(d == r0, "result in r0");
9380 __ block_comment("Shenandoah write barrier {");
9381 // We need that first read barrier in order to trigger a SEGV/NPE on incoming NULL.
9382 // Also, it brings s into d in preparation for the call to shenandoah_write_barrier().
9383 __ ldr(d, Address(s, BrooksPointer::byte_offset()));
9384 __ shenandoah_write_barrier(d);
9385 __ block_comment("} Shenandoah write barrier");
9386 %}
9387 ins_pipe(pipe_slow);
9388 %}
9389
9390 // Convert oop pointer into compressed form
9391 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
9392 predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
9393 match(Set dst (EncodeP src));
9394 effect(KILL cr);
9395 ins_cost(INSN_COST * 3);
9396 format %{ "encode_heap_oop $dst, $src" %}
9397 ins_encode %{
9398 Register s = $src$$Register;
9399 Register d = $dst$$Register;
9400 __ encode_heap_oop(d, s);
9401 %}
9402 ins_pipe(ialu_reg);
9403 %}
9404
9405 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
9406 predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
9407 match(Set dst (EncodeP src));
9408 ins_cost(INSN_COST * 3);
9409 format %{ "encode_heap_oop_not_null $dst, $src" %}
9662
9663 predicate(!UseShenandoahGC || !ShenandoahCASBarrier || n->in(3)->in(1)->bottom_type() == TypePtr::NULL_PTR);
9664 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
9665 ins_cost(2 * VOLATILE_REF_COST);
9666
9667 effect(KILL cr);
9668
9669 format %{
9670 "cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
9671 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9672 %}
9673
9674 ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
9675 aarch64_enc_cset_eq(res));
9676
9677 ins_pipe(pipe_slow);
9678 %}
9679
9680 instruct compareAndSwapP_shenandoah(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp, rFlagsReg cr) %{
9681
9682 predicate(UseShenandoahGC && ShenandoahCASBarrier);
9683 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
9684 ins_cost(3 * VOLATILE_REF_COST);
9685
9686 effect(TEMP tmp, KILL cr);
9687
9688 format %{
9689 "cmpxchg_oop_shenandoah $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval with temp $tmp"
9690 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9691 %}
9692
9693 ins_encode(aarch64_enc_cmpxchg_oop_shenandoah(mem, oldval, newval, tmp),
9694 aarch64_enc_cset_eq(res));
9695
9696 ins_pipe(pipe_slow);
9697 %}
9698
9699 instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
9700
9701 predicate(!UseShenandoahGC || !ShenandoahCASBarrier);
9702 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
9703 ins_cost(2 * VOLATILE_REF_COST);
9704
9705 effect(KILL cr);
9706
9707 format %{
9708 "cmpxchgw $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
9709 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9710 %}
9711
9712 ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
9713 aarch64_enc_cset_eq(res));
9714
9715 ins_pipe(pipe_slow);
9716 %}
9717
9718 instruct compareAndSwapN_shenandoah(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, iRegNNoSp tmp, rFlagsReg cr) %{
9719
9720 predicate(UseShenandoahGC && ShenandoahCASBarrier);
9721 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
9722 ins_cost(3 * VOLATILE_REF_COST);
9723
9724 effect(TEMP tmp, KILL cr);
9725
9726 format %{
9727 "cmpxchg_narrow_oop_shenandoah $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval with temp $tmp"
9728 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9729 %}
9730
9731 ins_encode %{
9732 Register tmp = $tmp$$Register;
9733 __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
9734 __ cmpxchg_oop_shenandoah($mem$$Register, tmp, $newval$$Register, Assembler::word, /*acquire*/ false, /*release*/ true, /*weak*/ false);
9735 __ cset($res$$Register, Assembler::EQ);
9736 %}
9737
9738 ins_pipe(pipe_slow);
9739 %}
9740
9741 // alternative CompareAndSwapX when we are eliding barriers
9742
9743 instruct compareAndSwapIAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
9744
9745 predicate(needs_acquiring_load_exclusive(n));
9746 match(Set res (CompareAndSwapI mem (Binary oldval newval)));
9747 ins_cost(VOLATILE_REF_COST);
9782
9783 predicate(needs_acquiring_load_exclusive(n) && (!UseShenandoahGC || !ShenandoahCASBarrier || n->in(3)->in(1)->bottom_type() == TypePtr::NULL_PTR));
9784 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
9785 ins_cost(VOLATILE_REF_COST);
9786
9787 effect(KILL cr);
9788
9789 format %{
9790 "cmpxchg_acq $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
9791 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9792 %}
9793
9794 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
9795 aarch64_enc_cset_eq(res));
9796
9797 ins_pipe(pipe_slow);
9798 %}
9799
9800 instruct compareAndSwapPAcq_shenandoah(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp, rFlagsReg cr) %{
9801
9802 predicate(needs_acquiring_load_exclusive(n) && UseShenandoahGC && ShenandoahCASBarrier);
9803 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
9804 ins_cost(2 * VOLATILE_REF_COST);
9805
9806 effect(TEMP tmp, KILL cr);
9807
9808 format %{
9809 "cmpxchg_acq_oop_shenandoah $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval with temp $tmp"
9810 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9811 %}
9812
9813 ins_encode(aarch64_enc_cmpxchg_acq_oop_shenandoah(mem, oldval, newval, tmp),
9814 aarch64_enc_cset_eq(res));
9815
9816 ins_pipe(pipe_slow);
9817 %}
9818
9819 instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
9820
9821 predicate(needs_acquiring_load_exclusive(n) && (!UseShenandoahGC || !ShenandoahCASBarrier));
9822 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
9823 ins_cost(VOLATILE_REF_COST);
9824
9825 effect(KILL cr);
9826
9827 format %{
9828 "cmpxchgw_acq $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
9829 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9830 %}
9831
9832 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
9833 aarch64_enc_cset_eq(res));
9834
9835 ins_pipe(pipe_slow);
9836 %}
9837
9838 instruct compareAndSwapNAcq_shenandoah(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, iRegNNoSp tmp, rFlagsReg cr) %{
9839
9840 predicate(needs_acquiring_load_exclusive(n) && UseShenandoahGC && ShenandoahCASBarrier);
9841 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
9842 ins_cost(3 * VOLATILE_REF_COST);
9843
9844 effect(TEMP tmp, KILL cr);
9845
9846 format %{
9847 "cmpxchg_narrow_oop_shenandoah $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval with temp $tmp"
9848 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9849 %}
9850
9851 ins_encode %{
9852 Register tmp = $tmp$$Register;
9853 __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
9854 __ cmpxchg_oop_shenandoah($mem$$Register, tmp, $newval$$Register, Assembler::word, /*acquire*/ true, /*release*/ true, /*weak*/ false);
9855 __ cset($res$$Register, Assembler::EQ);
9856 %}
9857
9858 ins_pipe(pipe_slow);
9859 %}
9860
9861 instruct get_and_setI(indirect mem, iRegI newv, iRegINoSp prev) %{
9862 match(Set prev (GetAndSetI mem newv));
9863 format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
9864 ins_encode %{
9865 __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9866 %}
9867 ins_pipe(pipe_serial);
|
9350 ins_cost(INSN_COST);
9351 format %{ "mov dst, $src\t# compressed ptr -> int" %}
9352 ins_encode %{
9353 __ movw($dst$$Register, $src$$Register);
9354 %}
9355
9356 ins_pipe(ialu_reg);
9357 %}
9358
9359 instruct shenandoahRB(iRegPNoSp dst, iRegP src, rFlagsReg cr) %{
9360 match(Set dst (ShenandoahReadBarrier src));
9361 format %{ "shenandoah_rb $dst,$src" %}
9362 ins_encode %{
9363 Register s = $src$$Register;
9364 Register d = $dst$$Register;
9365 __ ldr(d, Address(s, BrooksPointer::byte_offset()));
9366 %}
9367 ins_pipe(pipe_class_memory);
9368 %}
9369
9370 // Convert oop pointer into compressed form
9371 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
9372 predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
9373 match(Set dst (EncodeP src));
9374 effect(KILL cr);
9375 ins_cost(INSN_COST * 3);
9376 format %{ "encode_heap_oop $dst, $src" %}
9377 ins_encode %{
9378 Register s = $src$$Register;
9379 Register d = $dst$$Register;
9380 __ encode_heap_oop(d, s);
9381 %}
9382 ins_pipe(ialu_reg);
9383 %}
9384
9385 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
9386 predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
9387 match(Set dst (EncodeP src));
9388 ins_cost(INSN_COST * 3);
9389 format %{ "encode_heap_oop_not_null $dst, $src" %}
9642
9643 predicate(!UseShenandoahGC || !ShenandoahCASBarrier || n->in(3)->in(1)->bottom_type() == TypePtr::NULL_PTR);
9644 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
9645 ins_cost(2 * VOLATILE_REF_COST);
9646
9647 effect(KILL cr);
9648
9649 format %{
9650 "cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
9651 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9652 %}
9653
9654 ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
9655 aarch64_enc_cset_eq(res));
9656
9657 ins_pipe(pipe_slow);
9658 %}
9659
9660 instruct compareAndSwapP_shenandoah(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp, rFlagsReg cr) %{
9661
9662 predicate(UseShenandoahGC && ShenandoahCASBarrier && n->in(3)->in(1)->bottom_type() != TypePtr::NULL_PTR);
9663 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
9664 ins_cost(2 * VOLATILE_REF_COST);
9665
9666 effect(TEMP tmp, KILL cr);
9667
9668 format %{
9669 "cmpxchg_shenandoah_oop $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval with temp $tmp"
9670 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9671 %}
9672
9673 ins_encode(aarch64_enc_cmpxchg_oop_shenandoah(mem, oldval, newval, tmp),
9674 aarch64_enc_cset_eq(res));
9675
9676 ins_pipe(pipe_slow);
9677 %}
9678
9679 instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
9680
9681 predicate(!UseShenandoahGC || !ShenandoahCASBarrier || n->in(3)->in(1)->bottom_type() == TypeNarrowOop::NULL_PTR);
9682 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
9683 ins_cost(2 * VOLATILE_REF_COST);
9684
9685 effect(KILL cr);
9686
9687 format %{
9688 "cmpxchgw $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
9689 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9690 %}
9691
9692 ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
9693 aarch64_enc_cset_eq(res));
9694
9695 ins_pipe(pipe_slow);
9696 %}
9697
9698 instruct compareAndSwapN_shenandoah(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, iRegNNoSp tmp, rFlagsReg cr) %{
9699
9700 predicate(UseShenandoahGC && ShenandoahCASBarrier && n->in(3)->in(1)->bottom_type() != TypeNarrowOop::NULL_PTR);
9701 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
9702 ins_cost(2 * VOLATILE_REF_COST);
9703
9704 effect(TEMP tmp, KILL cr);
9705
9706 format %{
9707 "cmpxchgw_shenandoah_narrow_oop $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval with temp $tmp"
9708 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9709 %}
9710
9711 ins_encode %{
9712 Register tmp = $tmp$$Register;
9713 __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
9714 __ cmpxchg_oop_shenandoah($mem$$Register, tmp, $newval$$Register, Assembler::word, /*acquire*/ false, /*release*/ true, /*weak*/ false);
9715 __ cset($res$$Register, Assembler::EQ);
9716 %}
9717
9718 ins_pipe(pipe_slow);
9719 %}
9720
9721 // alternative CompareAndSwapX when we are eliding barriers
9722
9723 instruct compareAndSwapIAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
9724
9725 predicate(needs_acquiring_load_exclusive(n));
9726 match(Set res (CompareAndSwapI mem (Binary oldval newval)));
9727 ins_cost(VOLATILE_REF_COST);
9762
9763 predicate(needs_acquiring_load_exclusive(n) && (!UseShenandoahGC || !ShenandoahCASBarrier || n->in(3)->in(1)->bottom_type() == TypePtr::NULL_PTR));
9764 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
9765 ins_cost(VOLATILE_REF_COST);
9766
9767 effect(KILL cr);
9768
9769 format %{
9770 "cmpxchg_acq $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
9771 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9772 %}
9773
9774 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
9775 aarch64_enc_cset_eq(res));
9776
9777 ins_pipe(pipe_slow);
9778 %}
9779
9780 instruct compareAndSwapPAcq_shenandoah(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp, rFlagsReg cr) %{
9781
9782 predicate(needs_acquiring_load_exclusive(n) && UseShenandoahGC && ShenandoahCASBarrier && n->in(3)->in(1)->bottom_type() != TypePtr::NULL_PTR);
9783 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
9784 ins_cost(VOLATILE_REF_COST);
9785
9786 effect(TEMP tmp, KILL cr);
9787
9788 format %{
9789 "cmpxchg_acq_shenandoah_oop $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval with temp $tmp"
9790 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9791 %}
9792
9793 ins_encode(aarch64_enc_cmpxchg_acq_oop_shenandoah(mem, oldval, newval, tmp),
9794 aarch64_enc_cset_eq(res));
9795
9796 ins_pipe(pipe_slow);
9797 %}
9798
9799 instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
9800
9801 predicate(needs_acquiring_load_exclusive(n) && (!UseShenandoahGC || !ShenandoahCASBarrier|| n->in(3)->in(1)->bottom_type() == TypeNarrowOop::NULL_PTR));
9802 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
9803 ins_cost(VOLATILE_REF_COST);
9804
9805 effect(KILL cr);
9806
9807 format %{
9808 "cmpxchgw_acq $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
9809 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9810 %}
9811
9812 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
9813 aarch64_enc_cset_eq(res));
9814
9815 ins_pipe(pipe_slow);
9816 %}
9817
9818 instruct compareAndSwapNAcq_shenandoah(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, iRegNNoSp tmp, rFlagsReg cr) %{
9819
9820 predicate(needs_acquiring_load_exclusive(n) && UseShenandoahGC && ShenandoahCASBarrier && n->in(3)->in(1)->bottom_type() != TypeNarrowOop::NULL_PTR);
9821 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
9822 ins_cost(VOLATILE_REF_COST);
9823
9824 effect(TEMP tmp, KILL cr);
9825
9826 format %{
9827 "cmpxchgw_acq_shenandoah_narrow_oop $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval with temp $tmp"
9828 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9829 %}
9830
9831 ins_encode %{
9832 Register tmp = $tmp$$Register;
9833 __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
9834 __ cmpxchg_oop_shenandoah($mem$$Register, tmp, $newval$$Register, Assembler::word, /*acquire*/ true, /*release*/ true, /*weak*/ false);
9835 __ cset($res$$Register, Assembler::EQ);
9836 %}
9837
9838 ins_pipe(pipe_slow);
9839 %}
9840
9841 instruct get_and_setI(indirect mem, iRegI newv, iRegINoSp prev) %{
9842 match(Set prev (GetAndSetI mem newv));
9843 format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
9844 ins_encode %{
9845 __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9846 %}
9847 ins_pipe(pipe_serial);
|