3503 }
3504 __ bind(retry_load);
3505 __ ldaxr(rscratch1, tmp);
3506 __ cmp(disp_hdr, rscratch1);
3507 __ br(Assembler::NE, fail);
3508 // use stlxr to ensure update is immediately visible
3509 __ stlxr(rscratch1, rthread, tmp);
3510 __ cbnzw(rscratch1, retry_load);
3511 __ bind(fail);
3512 }
3513
3514 // Label next;
3515 // __ cmpxchgptr(/*oldv=*/disp_hdr,
3516 // /*newv=*/rthread,
3517 // /*addr=*/tmp,
3518 // /*tmp=*/rscratch1,
3519 // /*succeed*/next,
3520 // /*fail*/NULL);
3521 // __ bind(next);
3522
3523 // store a non-null value into the box.
3524 __ str(box, Address(box, BasicLock::displaced_header_offset_in_bytes()));
3525
3526 // PPC port checks the following invariants
3527 // #ifdef ASSERT
3528 // bne(flag, cont);
3529 // We have acquired the monitor, check some invariants.
3530 // addw(/*monitor=*/tmp, tmp, -ObjectMonitor::owner_offset_in_bytes());
3531 // Invariant 1: _recursions should be 0.
3532 // assert(ObjectMonitor::recursions_size_in_bytes() == 8, "unexpected size");
3533 // assert_mem8_is_zero(ObjectMonitor::recursions_offset_in_bytes(), tmp,
3534 // "monitor->_recursions should be 0", -1);
3535 // Invariant 2: OwnerIsThread shouldn't be 0.
3536 // assert(ObjectMonitor::OwnerIsThread_size_in_bytes() == 4, "unexpected size");
3537 //assert_mem4_isnot_zero(ObjectMonitor::OwnerIsThread_offset_in_bytes(), tmp,
3538 // "monitor->OwnerIsThread shouldn't be 0", -1);
3539 // #endif
3540
3541 __ bind(cont);
3542 // flag == EQ indicates success
3543 // flag == NE indicates failure
3544
3545 %}
3546
3547 // TODO
3548 // reimplement this with custom cmpxchgptr code
3549 // which avoids some of the unnecessary branching
3550 enc_class aarch64_enc_fast_unlock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{
3551 MacroAssembler _masm(&cbuf);
3552 Register oop = as_Register($object$$reg);
3553 Register box = as_Register($box$$reg);
3554 Register disp_hdr = as_Register($tmp$$reg);
3555 Register tmp = as_Register($tmp2$$reg);
3556 Label cont;
3557 Label object_has_monitor;
3558 Label cas_failed;
3559
3560 assert_different_registers(oop, box, tmp, disp_hdr);
3561
3562 if (UseBiasedLocking && !UseOptoBiasInlining) {
3563 __ biased_locking_exit(oop, tmp, cont);
3564 }
3565
3566 // Find the lock address and load the displaced header from the stack.
3567 __ ldr(disp_hdr, Address(box, BasicLock::displaced_header_offset_in_bytes()));
3568
3569 // If the displaced header is 0, we have a recursive unlock.
3570 __ cmp(disp_hdr, zr);
3571 __ br(Assembler::EQ, cont);
3572
3573
3574 // Handle existing monitor.
3575 __ ldr(tmp, Address(oop, oopDesc::mark_offset_in_bytes()));
3576 __ tbnz(disp_hdr, exact_log2(markOopDesc::monitor_value), object_has_monitor);
3577
3578 // Check if it is still a light weight lock, this is is true if we
3579 // see the stack address of the basicLock in the markOop of the
3580 // object.
3581
3582 if (UseLSE) {
3583 __ mov(tmp, box);
3584 __ casl(Assembler::xword, tmp, disp_hdr, oop);
3585 __ cmp(tmp, box);
3586 } else {
3587 Label retry_load;
3588 if ((VM_Version::features() & VM_Version::CPU_STXR_PREFETCH))
3589 __ prfm(Address(oop), PSTL1STRM);
3590 __ bind(retry_load);
3591 __ ldxr(tmp, oop);
3592 __ cmp(box, tmp);
3593 __ br(Assembler::NE, cas_failed);
3594 // use stlxr to ensure update is immediately visible
3595 __ stlxr(tmp, disp_hdr, oop);
3596 __ cbzw(tmp, cont);
3597 __ b(retry_load);
3598 }
3599
3600 // __ cmpxchgptr(/*compare_value=*/box,
3601 // /*exchange_value=*/disp_hdr,
3602 // /*where=*/oop,
3603 // /*result=*/tmp,
3604 // cont,
3605 // /*cas_failed*/NULL);
3606 assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
3607
3608 __ bind(cas_failed);
3609
3610 // Handle existing monitor.
3611 __ b(cont);
3612
3613 __ bind(object_has_monitor);
3614 __ add(tmp, tmp, -markOopDesc::monitor_value); // monitor
3615 __ ldr(rscratch1, Address(tmp, ObjectMonitor::owner_offset_in_bytes()));
3616 __ ldr(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset_in_bytes()));
3617 __ eor(rscratch1, rscratch1, rthread); // Will be 0 if we are the owner.
3618 __ orr(rscratch1, rscratch1, disp_hdr); // Will be 0 if there are 0 recursions
3619 __ cmp(rscratch1, zr);
3620 __ br(Assembler::NE, cont);
3621
3622 __ ldr(rscratch1, Address(tmp, ObjectMonitor::EntryList_offset_in_bytes()));
3623 __ ldr(disp_hdr, Address(tmp, ObjectMonitor::cxq_offset_in_bytes()));
3624 __ orr(rscratch1, rscratch1, disp_hdr); // Will be 0 if both are 0.
3625 __ cmp(rscratch1, zr);
3626 __ cbnz(rscratch1, cont);
3627 // need a release store here
3628 __ lea(tmp, Address(tmp, ObjectMonitor::owner_offset_in_bytes()));
3629 __ stlr(rscratch1, tmp); // rscratch1 is zero
3630
3631 __ bind(cont);
3632 // flag == EQ indicates success
|
3503 }
3504 __ bind(retry_load);
3505 __ ldaxr(rscratch1, tmp);
3506 __ cmp(disp_hdr, rscratch1);
3507 __ br(Assembler::NE, fail);
3508 // use stlxr to ensure update is immediately visible
3509 __ stlxr(rscratch1, rthread, tmp);
3510 __ cbnzw(rscratch1, retry_load);
3511 __ bind(fail);
3512 }
3513
3514 // Label next;
3515 // __ cmpxchgptr(/*oldv=*/disp_hdr,
3516 // /*newv=*/rthread,
3517 // /*addr=*/tmp,
3518 // /*tmp=*/rscratch1,
3519 // /*succeed*/next,
3520 // /*fail*/NULL);
3521 // __ bind(next);
3522
3523 // Store a non-null value into the box to avoid looking like a re-entrant
3524 // lock. The fast-path monitor unlock code checks for
3525 // markOopDesc::monitor_value so use markOopDesc::unused_mark which has the
3526 // relevant bit set, and also matches ObjectSynchronizer::slow_enter.
3527 __ mov(tmp, (address)markOopDesc::unused_mark());
3528 __ str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
3529
3530 // PPC port checks the following invariants
3531 // #ifdef ASSERT
3532 // bne(flag, cont);
3533 // We have acquired the monitor, check some invariants.
3534 // addw(/*monitor=*/tmp, tmp, -ObjectMonitor::owner_offset_in_bytes());
3535 // Invariant 1: _recursions should be 0.
3536 // assert(ObjectMonitor::recursions_size_in_bytes() == 8, "unexpected size");
3537 // assert_mem8_is_zero(ObjectMonitor::recursions_offset_in_bytes(), tmp,
3538 // "monitor->_recursions should be 0", -1);
3539 // Invariant 2: OwnerIsThread shouldn't be 0.
3540 // assert(ObjectMonitor::OwnerIsThread_size_in_bytes() == 4, "unexpected size");
3541 //assert_mem4_isnot_zero(ObjectMonitor::OwnerIsThread_offset_in_bytes(), tmp,
3542 // "monitor->OwnerIsThread shouldn't be 0", -1);
3543 // #endif
3544
3545 __ bind(cont);
3546 // flag == EQ indicates success
3547 // flag == NE indicates failure
3548
3549 %}
3550
3551 // TODO
3552 // reimplement this with custom cmpxchgptr code
3553 // which avoids some of the unnecessary branching
3554 enc_class aarch64_enc_fast_unlock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{
3555 MacroAssembler _masm(&cbuf);
3556 Register oop = as_Register($object$$reg);
3557 Register box = as_Register($box$$reg);
3558 Register disp_hdr = as_Register($tmp$$reg);
3559 Register tmp = as_Register($tmp2$$reg);
3560 Label cont;
3561 Label object_has_monitor;
3562
3563 assert_different_registers(oop, box, tmp, disp_hdr);
3564
3565 if (UseBiasedLocking && !UseOptoBiasInlining) {
3566 __ biased_locking_exit(oop, tmp, cont);
3567 }
3568
3569 // Find the lock address and load the displaced header from the stack.
3570 __ ldr(disp_hdr, Address(box, BasicLock::displaced_header_offset_in_bytes()));
3571
3572 // If the displaced header is 0, we have a recursive unlock.
3573 __ cmp(disp_hdr, zr);
3574 __ br(Assembler::EQ, cont);
3575
3576
3577 // Handle existing monitor.
3578 __ ldr(tmp, Address(oop, oopDesc::mark_offset_in_bytes()));
3579 __ tbnz(disp_hdr, exact_log2(markOopDesc::monitor_value), object_has_monitor);
3580
3581 // Check if it is still a light weight lock, this is is true if we
3582 // see the stack address of the basicLock in the markOop of the
3583 // object.
3584
3585 if (UseLSE) {
3586 __ mov(tmp, box);
3587 __ casl(Assembler::xword, tmp, disp_hdr, oop);
3588 __ cmp(tmp, box);
3589 __ b(cont);
3590 } else {
3591 Label retry_load;
3592 if ((VM_Version::features() & VM_Version::CPU_STXR_PREFETCH))
3593 __ prfm(Address(oop), PSTL1STRM);
3594 __ bind(retry_load);
3595 __ ldxr(tmp, oop);
3596 __ cmp(box, tmp);
3597 __ br(Assembler::NE, cont);
3598 // use stlxr to ensure update is immediately visible
3599 __ stlxr(tmp, disp_hdr, oop);
3600 __ cbzw(tmp, cont);
3601 __ b(retry_load);
3602 }
3603
3604 // __ cmpxchgptr(/*compare_value=*/box,
3605 // /*exchange_value=*/disp_hdr,
3606 // /*where=*/oop,
3607 // /*result=*/tmp,
3608 // cont,
3609 // /*cas_failed*/NULL);
3610 assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
3611
3612 // Handle existing monitor.
3613 __ bind(object_has_monitor);
3614 __ add(tmp, tmp, -markOopDesc::monitor_value); // monitor
3615 __ ldr(rscratch1, Address(tmp, ObjectMonitor::owner_offset_in_bytes()));
3616 __ ldr(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset_in_bytes()));
3617 __ eor(rscratch1, rscratch1, rthread); // Will be 0 if we are the owner.
3618 __ orr(rscratch1, rscratch1, disp_hdr); // Will be 0 if there are 0 recursions
3619 __ cmp(rscratch1, zr);
3620 __ br(Assembler::NE, cont);
3621
3622 __ ldr(rscratch1, Address(tmp, ObjectMonitor::EntryList_offset_in_bytes()));
3623 __ ldr(disp_hdr, Address(tmp, ObjectMonitor::cxq_offset_in_bytes()));
3624 __ orr(rscratch1, rscratch1, disp_hdr); // Will be 0 if both are 0.
3625 __ cmp(rscratch1, zr);
3626 __ cbnz(rscratch1, cont);
3627 // need a release store here
3628 __ lea(tmp, Address(tmp, ObjectMonitor::owner_offset_in_bytes()));
3629 __ stlr(rscratch1, tmp); // rscratch1 is zero
3630
3631 __ bind(cont);
3632 // flag == EQ indicates success
|