3786 // TODO
3787 // should we ever get anything other than this case?
3788 __ stlxr(rscratch1, src_reg, base);
3789 }
3790 } else {
3791 Register index_reg = as_Register(index);
3792 if (disp == 0) {
3793 __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
3794 __ stlxr(rscratch1, src_reg, rscratch2);
3795 } else {
3796 __ lea(rscratch2, Address(base, disp));
3797 __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
3798 __ stlxr(rscratch1, src_reg, rscratch2);
3799 }
3800 }
3801 __ cmpw(rscratch1, zr);
3802 %}
3803
3804 enc_class aarch64_enc_cmpxchg(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
3805 MacroAssembler _masm(&cbuf);
3806 Register old_reg = as_Register($oldval$$reg);
3807 Register new_reg = as_Register($newval$$reg);
3808 Register base = as_Register($mem$$base);
3809 Register addr_reg;
3810 int index = $mem$$index;
3811 int scale = $mem$$scale;
3812 int disp = $mem$$disp;
3813 if (index == -1) {
3814 if (disp != 0) {
3815 __ lea(rscratch2, Address(base, disp));
3816 addr_reg = rscratch2;
3817 } else {
3818 // TODO
3819 // should we ever get anything other than this case?
3820 addr_reg = base;
3821 }
3822 } else {
3823 Register index_reg = as_Register(index);
3824 if (disp == 0) {
3825 __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
3826 addr_reg = rscratch2;
3827 } else {
3828 __ lea(rscratch2, Address(base, disp));
3829 __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
3830 addr_reg = rscratch2;
3831 }
3832 }
3833 Label retry_load, done;
3834 __ bind(retry_load);
3835 __ ldxr(rscratch1, addr_reg);
3836 __ cmp(rscratch1, old_reg);
3837 __ br(Assembler::NE, done);
3838 __ stlxr(rscratch1, new_reg, addr_reg);
3839 __ cbnzw(rscratch1, retry_load);
3840 __ bind(done);
3841 %}
3842
3843 enc_class aarch64_enc_cmpxchgw(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3844 MacroAssembler _masm(&cbuf);
3845 Register old_reg = as_Register($oldval$$reg);
3846 Register new_reg = as_Register($newval$$reg);
3847 Register base = as_Register($mem$$base);
3848 Register addr_reg;
3849 int index = $mem$$index;
3850 int scale = $mem$$scale;
3851 int disp = $mem$$disp;
3852 if (index == -1) {
3853 if (disp != 0) {
3854 __ lea(rscratch2, Address(base, disp));
3855 addr_reg = rscratch2;
3856 } else {
3857 // TODO
3858 // should we ever get anything other than this case?
3859 addr_reg = base;
3860 }
3861 } else {
3862 Register index_reg = as_Register(index);
3863 if (disp == 0) {
3864 __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
3865 addr_reg = rscratch2;
3866 } else {
3867 __ lea(rscratch2, Address(base, disp));
3868 __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
3869 addr_reg = rscratch2;
3870 }
3871 }
3872 Label retry_load, done;
3873 __ bind(retry_load);
3874 __ ldxrw(rscratch1, addr_reg);
3875 __ cmpw(rscratch1, old_reg);
3876 __ br(Assembler::NE, done);
3877 __ stlxrw(rscratch1, new_reg, addr_reg);
3878 __ cbnzw(rscratch1, retry_load);
3879 __ bind(done);
3880 %}
3881
3882 // auxiliary used for CompareAndSwapX to set result register
3883 enc_class aarch64_enc_cset_eq(iRegINoSp res) %{
3884 MacroAssembler _masm(&cbuf);
3885 Register res_reg = as_Register($res$$reg);
3886 __ cset(res_reg, Assembler::EQ);
3887 %}
3888
3889 // prefetch encodings
3890
3891 enc_class aarch64_enc_prefetchw(memory mem) %{
3892 MacroAssembler _masm(&cbuf);
3893 Register base = as_Register($mem$$base);
3894 int index = $mem$$index;
3895 int scale = $mem$$scale;
3896 int disp = $mem$$disp;
3897 if (index == -1) {
3898 __ prfm(Address(base, disp), PSTL1KEEP);
3899 __ nop();
3900 } else {
3901 Register index_reg = as_Register(index);
4381 if ((EmitSync & 0x02) == 0) {
4382 // we can use AArch64's bit test and branch here but
4383 // markoopDesc does not define a bit index just the bit value
4384 // so assert in case the bit pos changes
4385 # define __monitor_value_log2 1
4386 assert(markOopDesc::monitor_value == (1 << __monitor_value_log2), "incorrect bit position");
4387 __ tbnz(disp_hdr, __monitor_value_log2, object_has_monitor);
4388 # undef __monitor_value_log2
4389 }
4390
4391 // Set displaced_header to be (markOop of object | UNLOCK_VALUE).
4392 __ orr(disp_hdr, disp_hdr, markOopDesc::unlocked_value);
4393
4394 // Load Compare Value application register.
4395
4396 // Initialize the box. (Must happen before we update the object mark!)
4397 __ str(disp_hdr, Address(box, BasicLock::displaced_header_offset_in_bytes()));
4398
4399 // Compare object markOop with mark and if equal exchange scratch1
4400 // with object markOop.
4401 // Note that this is simply a CAS: it does not generate any
4402 // barriers. These are separately generated by
4403 // membar_acquire_lock().
4404 {
4405 Label retry_load;
4406 __ bind(retry_load);
4407 __ ldxr(tmp, oop);
4408 __ cmp(tmp, disp_hdr);
4409 __ br(Assembler::NE, cas_failed);
4410 // use stlxr to ensure update is immediately visible
4411 __ stlxr(tmp, box, oop);
4412 __ cbzw(tmp, cont);
4413 __ b(retry_load);
4414 }
4415
4416 // Formerly:
4417 // __ cmpxchgptr(/*oldv=*/disp_hdr,
4418 // /*newv=*/box,
4419 // /*addr=*/oop,
4420 // /*tmp=*/tmp,
4421 // cont,
4422 // /*fail*/NULL);
4423
4424 assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
4425
4426 // If the compare-and-exchange succeeded, then we found an unlocked
4427 // object, will have now locked it will continue at label cont
4437 // If condition is true we are cont and hence we can store 0 as the
4438 // displaced header in the box, which indicates that it is a recursive lock.
4439 __ ands(tmp/*==0?*/, disp_hdr, tmp);
4440 __ str(tmp/*==0, perhaps*/, Address(box, BasicLock::displaced_header_offset_in_bytes()));
4441
4442 // Handle existing monitor.
4443 if ((EmitSync & 0x02) == 0) {
4444 __ b(cont);
4445
4446 __ bind(object_has_monitor);
4447 // The object's monitor m is unlocked iff m->owner == NULL,
4448 // otherwise m->owner may contain a thread or a stack address.
4449 //
4450 // Try to CAS m->owner from NULL to current thread.
4451 __ add(tmp, disp_hdr, (ObjectMonitor::owner_offset_in_bytes()-markOopDesc::monitor_value));
4452 __ mov(disp_hdr, zr);
4453
4454 {
4455 Label retry_load, fail;
4456 __ bind(retry_load);
4457 __ ldxr(rscratch1, tmp);
4458 __ cmp(disp_hdr, rscratch1);
4459 __ br(Assembler::NE, fail);
4460 // use stlxr to ensure update is immediately visible
4461 __ stlxr(rscratch1, rthread, tmp);
4462 __ cbnzw(rscratch1, retry_load);
4463 __ bind(fail);
4464 }
4465
4466 // Label next;
4467 // __ cmpxchgptr(/*oldv=*/disp_hdr,
4468 // /*newv=*/rthread,
4469 // /*addr=*/tmp,
4470 // /*tmp=*/rscratch1,
4471 // /*succeed*/next,
4472 // /*fail*/NULL);
4473 // __ bind(next);
4474
4475 // store a non-null value into the box.
4476 __ str(box, Address(box, BasicLock::displaced_header_offset_in_bytes()));
4477
8000
8001 instruct membar_acquire() %{
8002 match(MemBarAcquire);
8003 ins_cost(VOLATILE_REF_COST);
8004
8005 format %{ "membar_acquire" %}
8006
8007 ins_encode %{
8008 __ block_comment("membar_acquire");
8009 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
8010 %}
8011
8012 ins_pipe(pipe_serial);
8013 %}
8014
8015
8016 instruct membar_acquire_lock() %{
8017 match(MemBarAcquireLock);
8018 ins_cost(VOLATILE_REF_COST);
8019
8020 format %{ "membar_acquire_lock" %}
8021
8022 ins_encode %{
8023 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
8024 %}
8025
8026 ins_pipe(pipe_serial);
8027 %}
8028
8029 instruct store_fence() %{
8030 match(StoreFence);
8031 ins_cost(VOLATILE_REF_COST);
8032
8033 format %{ "store_fence" %}
8034
8035 ins_encode %{
8036 __ membar(Assembler::LoadStore|Assembler::StoreStore);
8037 %}
8038 ins_pipe(pipe_serial);
8039 %}
8040
8041 instruct unnecessary_membar_release() %{
8042 predicate(unnecessary_release(n));
8043 match(MemBarRelease);
8063 %}
8064 ins_pipe(pipe_serial);
8065 %}
8066
8067 instruct membar_storestore() %{
8068 match(MemBarStoreStore);
8069 ins_cost(VOLATILE_REF_COST);
8070
8071 format %{ "MEMBAR-store-store" %}
8072
8073 ins_encode %{
8074 __ membar(Assembler::StoreStore);
8075 %}
8076 ins_pipe(pipe_serial);
8077 %}
8078
8079 instruct membar_release_lock() %{
8080 match(MemBarReleaseLock);
8081 ins_cost(VOLATILE_REF_COST);
8082
8083 format %{ "membar_release_lock" %}
8084
8085 ins_encode %{
8086 __ membar(Assembler::LoadStore|Assembler::StoreStore);
8087 %}
8088
8089 ins_pipe(pipe_serial);
8090 %}
8091
8092 instruct unnecessary_membar_volatile() %{
8093 predicate(unnecessary_volatile(n));
8094 match(MemBarVolatile);
8095 ins_cost(0);
8096
8097 format %{ "membar_volatile (elided)" %}
8098
8099 ins_encode %{
8100 __ block_comment("membar_volatile (elided)");
8101 %}
8102
8103 ins_pipe(pipe_serial);
8104 %}
8105
8106 instruct membar_volatile() %{
8352 instruct storePConditional(memory heap_top_ptr, iRegP oldval, iRegP newval, rFlagsReg cr)
8353 %{
8354 match(Set cr (StorePConditional heap_top_ptr (Binary oldval newval)));
8355
8356 ins_cost(VOLATILE_REF_COST);
8357
8358 // TODO
8359 // do we need to do a store-conditional release or can we just use a
8360 // plain store-conditional?
8361
8362 format %{
8363 "stlxr rscratch1, $newval, $heap_top_ptr\t# ptr cond release"
8364 "cmpw rscratch1, zr\t# EQ on successful write"
8365 %}
8366
8367 ins_encode(aarch64_enc_stlxr(newval, heap_top_ptr));
8368
8369 ins_pipe(pipe_serial);
8370 %}
8371
8372 // this has to be implemented as a CAS
8373 instruct storeLConditional(indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr)
8374 %{
8375 match(Set cr (StoreLConditional mem (Binary oldval newval)));
8376
8377 ins_cost(VOLATILE_REF_COST);
8378
8379 format %{
8380 "cmpxchg rscratch1, $mem, $oldval, $newval, $mem\t# if $mem == $oldval then $mem <-- $newval"
8381 "cmpw rscratch1, zr\t# EQ on successful write"
8382 %}
8383
8384 ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval));
8385
8386 ins_pipe(pipe_slow);
8387 %}
8388
8389 // this has to be implemented as a CAS
8390 instruct storeIConditional(indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr)
8391 %{
8392 match(Set cr (StoreIConditional mem (Binary oldval newval)));
8393
8394 ins_cost(VOLATILE_REF_COST);
8395
8396 format %{
8397 "cmpxchgw rscratch1, $mem, $oldval, $newval, $mem\t# if $mem == $oldval then $mem <-- $newval"
8398 "cmpw rscratch1, zr\t# EQ on successful write"
8399 %}
8400
8401 ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval));
8402
8403 ins_pipe(pipe_slow);
8404 %}
8405
8406 // XXX No flag versions for CompareAndSwap{I,L,P,N} because matcher
8407 // can't match them
8408
8409 instruct compareAndSwapI(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8410
8411 match(Set res (CompareAndSwapI mem (Binary oldval newval)));
8412
8413 effect(KILL cr);
8414
8415 format %{
8416 "cmpxchgw $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8417 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8418 %}
8419
8420 ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
8421 aarch64_enc_cset_eq(res));
|
3786 // TODO
3787 // should we ever get anything other than this case?
3788 __ stlxr(rscratch1, src_reg, base);
3789 }
3790 } else {
3791 Register index_reg = as_Register(index);
3792 if (disp == 0) {
3793 __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
3794 __ stlxr(rscratch1, src_reg, rscratch2);
3795 } else {
3796 __ lea(rscratch2, Address(base, disp));
3797 __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
3798 __ stlxr(rscratch1, src_reg, rscratch2);
3799 }
3800 }
3801 __ cmpw(rscratch1, zr);
3802 %}
3803
3804 enc_class aarch64_enc_cmpxchg(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
3805 MacroAssembler _masm(&cbuf);
3806 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3807 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3808 &Assembler::ldxr, &MacroAssembler::cmp, &Assembler::stlxr);
3809 %}
3810
3811 enc_class aarch64_enc_cmpxchgw(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3812 MacroAssembler _masm(&cbuf);
3813 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3814 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3815 &Assembler::ldxrw, &MacroAssembler::cmpw, &Assembler::stlxrw);
3816 %}
3817
3818
3819 // The only difference between aarch64_enc_cmpxchg and
3820 // aarch64_enc_cmpxchg_acq is that we use load-acquire in the
3821 // CompareAndSwap sequence to serve as a barrier on acquiring a
3822 // lock.
3823 enc_class aarch64_enc_cmpxchg_acq(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
3824 MacroAssembler _masm(&cbuf);
3825 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3826 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3827 &Assembler::ldaxr, &MacroAssembler::cmp, &Assembler::stlxr);
3828 %}
3829
3830 enc_class aarch64_enc_cmpxchgw_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3831 MacroAssembler _masm(&cbuf);
3832 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3833 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3834 &Assembler::ldaxrw, &MacroAssembler::cmpw, &Assembler::stlxrw);
3835 %}
3836
3837
3838 // auxiliary used for CompareAndSwapX to set result register
3839 enc_class aarch64_enc_cset_eq(iRegINoSp res) %{
3840 MacroAssembler _masm(&cbuf);
3841 Register res_reg = as_Register($res$$reg);
3842 __ cset(res_reg, Assembler::EQ);
3843 %}
3844
3845 // prefetch encodings
3846
3847 enc_class aarch64_enc_prefetchw(memory mem) %{
3848 MacroAssembler _masm(&cbuf);
3849 Register base = as_Register($mem$$base);
3850 int index = $mem$$index;
3851 int scale = $mem$$scale;
3852 int disp = $mem$$disp;
3853 if (index == -1) {
3854 __ prfm(Address(base, disp), PSTL1KEEP);
3855 __ nop();
3856 } else {
3857 Register index_reg = as_Register(index);
4337 if ((EmitSync & 0x02) == 0) {
4338 // we can use AArch64's bit test and branch here but
4339 // markoopDesc does not define a bit index just the bit value
4340 // so assert in case the bit pos changes
4341 # define __monitor_value_log2 1
4342 assert(markOopDesc::monitor_value == (1 << __monitor_value_log2), "incorrect bit position");
4343 __ tbnz(disp_hdr, __monitor_value_log2, object_has_monitor);
4344 # undef __monitor_value_log2
4345 }
4346
4347 // Set displaced_header to be (markOop of object | UNLOCK_VALUE).
4348 __ orr(disp_hdr, disp_hdr, markOopDesc::unlocked_value);
4349
4350 // Load Compare Value application register.
4351
4352 // Initialize the box. (Must happen before we update the object mark!)
4353 __ str(disp_hdr, Address(box, BasicLock::displaced_header_offset_in_bytes()));
4354
4355 // Compare object markOop with mark and if equal exchange scratch1
4356 // with object markOop.
4357 {
4358 Label retry_load;
4359 __ bind(retry_load);
4360 __ ldaxr(tmp, oop);
4361 __ cmp(tmp, disp_hdr);
4362 __ br(Assembler::NE, cas_failed);
4363 // use stlxr to ensure update is immediately visible
4364 __ stlxr(tmp, box, oop);
4365 __ cbzw(tmp, cont);
4366 __ b(retry_load);
4367 }
4368
4369 // Formerly:
4370 // __ cmpxchgptr(/*oldv=*/disp_hdr,
4371 // /*newv=*/box,
4372 // /*addr=*/oop,
4373 // /*tmp=*/tmp,
4374 // cont,
4375 // /*fail*/NULL);
4376
4377 assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
4378
4379 // If the compare-and-exchange succeeded, then we found an unlocked
4380 // object, will have now locked it will continue at label cont
4390 // If condition is true we are cont and hence we can store 0 as the
4391 // displaced header in the box, which indicates that it is a recursive lock.
4392 __ ands(tmp/*==0?*/, disp_hdr, tmp);
4393 __ str(tmp/*==0, perhaps*/, Address(box, BasicLock::displaced_header_offset_in_bytes()));
4394
4395 // Handle existing monitor.
4396 if ((EmitSync & 0x02) == 0) {
4397 __ b(cont);
4398
4399 __ bind(object_has_monitor);
4400 // The object's monitor m is unlocked iff m->owner == NULL,
4401 // otherwise m->owner may contain a thread or a stack address.
4402 //
4403 // Try to CAS m->owner from NULL to current thread.
4404 __ add(tmp, disp_hdr, (ObjectMonitor::owner_offset_in_bytes()-markOopDesc::monitor_value));
4405 __ mov(disp_hdr, zr);
4406
4407 {
4408 Label retry_load, fail;
4409 __ bind(retry_load);
4410 __ ldaxr(rscratch1, tmp);
4411 __ cmp(disp_hdr, rscratch1);
4412 __ br(Assembler::NE, fail);
4413 // use stlxr to ensure update is immediately visible
4414 __ stlxr(rscratch1, rthread, tmp);
4415 __ cbnzw(rscratch1, retry_load);
4416 __ bind(fail);
4417 }
4418
4419 // Label next;
4420 // __ cmpxchgptr(/*oldv=*/disp_hdr,
4421 // /*newv=*/rthread,
4422 // /*addr=*/tmp,
4423 // /*tmp=*/rscratch1,
4424 // /*succeed*/next,
4425 // /*fail*/NULL);
4426 // __ bind(next);
4427
4428 // store a non-null value into the box.
4429 __ str(box, Address(box, BasicLock::displaced_header_offset_in_bytes()));
4430
7953
7954 instruct membar_acquire() %{
7955 match(MemBarAcquire);
7956 ins_cost(VOLATILE_REF_COST);
7957
7958 format %{ "membar_acquire" %}
7959
7960 ins_encode %{
7961 __ block_comment("membar_acquire");
7962 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7963 %}
7964
7965 ins_pipe(pipe_serial);
7966 %}
7967
7968
7969 instruct membar_acquire_lock() %{
7970 match(MemBarAcquireLock);
7971 ins_cost(VOLATILE_REF_COST);
7972
7973 format %{ "membar_acquire_lock (elided)" %}
7974
7975 ins_encode %{
7976 __ block_comment("membar_acquire_lock (elided)");
7977 %}
7978
7979 ins_pipe(pipe_serial);
7980 %}
7981
7982 instruct store_fence() %{
7983 match(StoreFence);
7984 ins_cost(VOLATILE_REF_COST);
7985
7986 format %{ "store_fence" %}
7987
7988 ins_encode %{
7989 __ membar(Assembler::LoadStore|Assembler::StoreStore);
7990 %}
7991 ins_pipe(pipe_serial);
7992 %}
7993
7994 instruct unnecessary_membar_release() %{
7995 predicate(unnecessary_release(n));
7996 match(MemBarRelease);
8016 %}
8017 ins_pipe(pipe_serial);
8018 %}
8019
8020 instruct membar_storestore() %{
8021 match(MemBarStoreStore);
8022 ins_cost(VOLATILE_REF_COST);
8023
8024 format %{ "MEMBAR-store-store" %}
8025
8026 ins_encode %{
8027 __ membar(Assembler::StoreStore);
8028 %}
8029 ins_pipe(pipe_serial);
8030 %}
8031
8032 instruct membar_release_lock() %{
8033 match(MemBarReleaseLock);
8034 ins_cost(VOLATILE_REF_COST);
8035
8036 format %{ "membar_release_lock (elided)" %}
8037
8038 ins_encode %{
8039 __ block_comment("membar_release_lock (elided)");
8040 %}
8041
8042 ins_pipe(pipe_serial);
8043 %}
8044
8045 instruct unnecessary_membar_volatile() %{
8046 predicate(unnecessary_volatile(n));
8047 match(MemBarVolatile);
8048 ins_cost(0);
8049
8050 format %{ "membar_volatile (elided)" %}
8051
8052 ins_encode %{
8053 __ block_comment("membar_volatile (elided)");
8054 %}
8055
8056 ins_pipe(pipe_serial);
8057 %}
8058
8059 instruct membar_volatile() %{
8305 instruct storePConditional(memory heap_top_ptr, iRegP oldval, iRegP newval, rFlagsReg cr)
8306 %{
8307 match(Set cr (StorePConditional heap_top_ptr (Binary oldval newval)));
8308
8309 ins_cost(VOLATILE_REF_COST);
8310
8311 // TODO
8312 // do we need to do a store-conditional release or can we just use a
8313 // plain store-conditional?
8314
8315 format %{
8316 "stlxr rscratch1, $newval, $heap_top_ptr\t# ptr cond release"
8317 "cmpw rscratch1, zr\t# EQ on successful write"
8318 %}
8319
8320 ins_encode(aarch64_enc_stlxr(newval, heap_top_ptr));
8321
8322 ins_pipe(pipe_serial);
8323 %}
8324
8325
8326 // storeLConditional is used by PhaseMacroExpand::expand_lock_node
8327 // when attempting to rebias a lock towards the current thread. We
8328 // must use the acquire form of cmpxchg in order to guarantee acquire
8329 // semantics in this case.
8330 instruct storeLConditional(indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr)
8331 %{
8332 match(Set cr (StoreLConditional mem (Binary oldval newval)));
8333
8334 ins_cost(VOLATILE_REF_COST);
8335
8336 format %{
8337 "cmpxchg rscratch1, $mem, $oldval, $newval, $mem\t# if $mem == $oldval then $mem <-- $newval"
8338 "cmpw rscratch1, zr\t# EQ on successful write"
8339 %}
8340
8341 ins_encode(aarch64_enc_cmpxchg_acq(newval, heap_top_ptr));
8342
8343 ins_pipe(pipe_slow);
8344 %}
8345
8346 // storeIConditional also has acquire semantics, for no better reason
8347 // than matching storeLConditional. At the time of writing this
8348 // comment storeIConditional was not used anywhere by AArch64.
8349 instruct storeIConditional(indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr)
8350 %{
8351 match(Set cr (StoreIConditional mem (Binary oldval newval)));
8352
8353 ins_cost(VOLATILE_REF_COST);
8354
8355 format %{
8356 "cmpxchgw rscratch1, $mem, $oldval, $newval, $mem\t# if $mem == $oldval then $mem <-- $newval"
8357 "cmpw rscratch1, zr\t# EQ on successful write"
8358 %}
8359
8360 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval));
8361
8362 ins_pipe(pipe_slow);
8363 %}
8364
8365 // XXX No flag versions for CompareAndSwap{I,L,P,N} because matcher
8366 // can't match them
8367
8368 instruct compareAndSwapI(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8369
8370 match(Set res (CompareAndSwapI mem (Binary oldval newval)));
8371
8372 effect(KILL cr);
8373
8374 format %{
8375 "cmpxchgw $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8376 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8377 %}
8378
8379 ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
8380 aarch64_enc_cset_eq(res));
|