src/cpu/x86/vm/x86_64.ad

*** 2597,2831 ****
      __ movdbl(Address(rsp, 0), $src$$XMMRegister);
      __ fld_d(Address(rsp, 0));
    %}
  
  
-   // obj: object to lock
-   // box: box address (header location) -- killed
-   // tmp: rax -- killed
-   // scr: rbx -- killed
-   //
-   // What follows is a direct transliteration of fast_lock() and fast_unlock()
-   // from i486.ad.  See that file for comments.
-   // TODO: where possible switch from movq (r, 0) to movl(r,0) and
-   // use the shorter encoding.  (Movl clears the high-order 32-bits).
- 
- 
-   enc_class Fast_Lock(rRegP obj, rRegP box, rax_RegI tmp, rRegP scr)
-   %{
-     Register objReg = as_Register((int)$obj$$reg);
-     Register boxReg = as_Register((int)$box$$reg);
-     Register tmpReg = as_Register($tmp$$reg);
-     Register scrReg = as_Register($scr$$reg);
-     MacroAssembler masm(&cbuf);
- 
-     // Verify uniqueness of register assignments -- necessary but not sufficient
-     assert (objReg != boxReg && objReg != tmpReg &&
-             objReg != scrReg && tmpReg != scrReg, "invariant") ;
- 
-     if (_counters != NULL) {
-       masm.atomic_incl(ExternalAddress((address) _counters->total_entry_count_addr()));
-     }
-     if (EmitSync & 1) {
-         // Without cast to int32_t a movptr will destroy r10 which is typically obj
-         masm.movptr (Address(boxReg, 0), (int32_t)intptr_t(markOopDesc::unused_mark())) ;
-         masm.cmpptr(rsp, (int32_t)NULL_WORD) ;
-     } else
-     if (EmitSync & 2) {
-         Label DONE_LABEL;
-         if (UseBiasedLocking) {
-            // Note: tmpReg maps to the swap_reg argument and scrReg to the tmp_reg argument.
-           masm.biased_locking_enter(boxReg, objReg, tmpReg, scrReg, false, DONE_LABEL, NULL, _counters);
-         }
-         // QQQ was movl...
-         masm.movptr(tmpReg, 0x1);
-         masm.orptr(tmpReg, Address(objReg, 0));
-         masm.movptr(Address(boxReg, 0), tmpReg);
-         if (os::is_MP()) {
-           masm.lock();
-         }
-         masm.cmpxchgptr(boxReg, Address(objReg, 0)); // Updates tmpReg
-         masm.jcc(Assembler::equal, DONE_LABEL);
- 
-         // Recursive locking
-         masm.subptr(tmpReg, rsp);
-         masm.andptr(tmpReg, 7 - os::vm_page_size());
-         masm.movptr(Address(boxReg, 0), tmpReg);
- 
-         masm.bind(DONE_LABEL);
-         masm.nop(); // avoid branch to branch
-     } else {
-         Label DONE_LABEL, IsInflated, Egress;
- 
-         masm.movptr(tmpReg, Address(objReg, 0)) ;
-         masm.testl (tmpReg, 0x02) ;         // inflated vs stack-locked|neutral|biased
-         masm.jcc   (Assembler::notZero, IsInflated) ;
- 
-         // it's stack-locked, biased or neutral
-         // TODO: optimize markword triage order to reduce the number of
-         // conditional branches in the most common cases.
-         // Beware -- there's a subtle invariant that fetch of the markword
-         // at [FETCH], below, will never observe a biased encoding (*101b).
-         // If this invariant is not held we'll suffer exclusion (safety) failure.
- 
-         if (UseBiasedLocking && !UseOptoBiasInlining) {
-           masm.biased_locking_enter(boxReg, objReg, tmpReg, scrReg, true, DONE_LABEL, NULL, _counters);
-           masm.movptr(tmpReg, Address(objReg, 0)) ;        // [FETCH]
-         }
- 
-         // was q will it destroy high?
-         masm.orl   (tmpReg, 1) ;
-         masm.movptr(Address(boxReg, 0), tmpReg) ;
-         if (os::is_MP()) { masm.lock(); }
-         masm.cmpxchgptr(boxReg, Address(objReg, 0)); // Updates tmpReg
-         if (_counters != NULL) {
-            masm.cond_inc32(Assembler::equal,
-                            ExternalAddress((address) _counters->fast_path_entry_count_addr()));
-         }
-         masm.jcc   (Assembler::equal, DONE_LABEL);
- 
-         // Recursive locking
-         masm.subptr(tmpReg, rsp);
-         masm.andptr(tmpReg, 7 - os::vm_page_size());
-         masm.movptr(Address(boxReg, 0), tmpReg);
-         if (_counters != NULL) {
-            masm.cond_inc32(Assembler::equal,
-                            ExternalAddress((address) _counters->fast_path_entry_count_addr()));
-         }
-         masm.jmp   (DONE_LABEL) ;
- 
-         masm.bind  (IsInflated) ;
-         // It's inflated
- 
-         // TODO: someday avoid the ST-before-CAS penalty by
-         // relocating (deferring) the following ST.
-         // We should also think about trying a CAS without having
-         // fetched _owner.  If the CAS is successful we may
-         // avoid an RTO->RTS upgrade on the $line.
-         // Without cast to int32_t a movptr will destroy r10 which is typically obj
-         masm.movptr(Address(boxReg, 0), (int32_t)intptr_t(markOopDesc::unused_mark())) ;
- 
-         masm.mov    (boxReg, tmpReg) ;
-         masm.movptr (tmpReg, Address(tmpReg, ObjectMonitor::owner_offset_in_bytes()-2)) ;
-         masm.testptr(tmpReg, tmpReg) ;
-         masm.jcc    (Assembler::notZero, DONE_LABEL) ;
- 
-         // It's inflated and appears unlocked
-         if (os::is_MP()) { masm.lock(); }
-         masm.cmpxchgptr(r15_thread, Address(boxReg, ObjectMonitor::owner_offset_in_bytes()-2)) ;
-         // Intentional fall-through into DONE_LABEL ...
- 
-         masm.bind  (DONE_LABEL) ;
-         masm.nop   () ;                 // avoid jmp to jmp
-     }
-   %}
- 
-   // obj: object to unlock
-   // box: box address (displaced header location), killed
-   // RBX: killed tmp; cannot be obj nor box
-   enc_class Fast_Unlock(rRegP obj, rax_RegP box, rRegP tmp)
-   %{
- 
-     Register objReg = as_Register($obj$$reg);
-     Register boxReg = as_Register($box$$reg);
-     Register tmpReg = as_Register($tmp$$reg);
-     MacroAssembler masm(&cbuf);
- 
-     if (EmitSync & 4) {
-        masm.cmpptr(rsp, 0) ;
-     } else
-     if (EmitSync & 8) {
-        Label DONE_LABEL;
-        if (UseBiasedLocking) {
-          masm.biased_locking_exit(objReg, tmpReg, DONE_LABEL);
-        }
- 
-        // Check whether the displaced header is 0
-        //(=> recursive unlock)
-        masm.movptr(tmpReg, Address(boxReg, 0));
-        masm.testptr(tmpReg, tmpReg);
-        masm.jcc(Assembler::zero, DONE_LABEL);
- 
-        // If not recursive lock, reset the header to displaced header
-        if (os::is_MP()) {
-          masm.lock();
-        }
-        masm.cmpxchgptr(tmpReg, Address(objReg, 0)); // Uses RAX which is box
-        masm.bind(DONE_LABEL);
-        masm.nop(); // avoid branch to branch
-     } else {
-        Label DONE_LABEL, Stacked, CheckSucc ;
- 
-        if (UseBiasedLocking && !UseOptoBiasInlining) {
-          masm.biased_locking_exit(objReg, tmpReg, DONE_LABEL);
-        }
- 
-        masm.movptr(tmpReg, Address(objReg, 0)) ;
-        masm.cmpptr(Address(boxReg, 0), (int32_t)NULL_WORD) ;
-        masm.jcc   (Assembler::zero, DONE_LABEL) ;
-        masm.testl (tmpReg, 0x02) ;
-        masm.jcc   (Assembler::zero, Stacked) ;
- 
-        // It's inflated
-        masm.movptr(boxReg, Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2)) ;
-        masm.xorptr(boxReg, r15_thread) ;
-        masm.orptr (boxReg, Address (tmpReg, ObjectMonitor::recursions_offset_in_bytes()-2)) ;
-        masm.jcc   (Assembler::notZero, DONE_LABEL) ;
-        masm.movptr(boxReg, Address (tmpReg, ObjectMonitor::cxq_offset_in_bytes()-2)) ;
-        masm.orptr (boxReg, Address (tmpReg, ObjectMonitor::EntryList_offset_in_bytes()-2)) ;
-        masm.jcc   (Assembler::notZero, CheckSucc) ;
-        masm.movptr(Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2), (int32_t)NULL_WORD) ;
-        masm.jmp   (DONE_LABEL) ;
- 
-        if ((EmitSync & 65536) == 0) {
-          Label LSuccess, LGoSlowPath ;
-          masm.bind  (CheckSucc) ;
-          masm.cmpptr(Address (tmpReg, ObjectMonitor::succ_offset_in_bytes()-2), (int32_t)NULL_WORD) ;
-          masm.jcc   (Assembler::zero, LGoSlowPath) ;
- 
-          // I'd much rather use lock:andl m->_owner, 0 as it's faster than the
-          // the explicit ST;MEMBAR combination, but masm doesn't currently support
-          // "ANDQ M,IMM".  Don't use MFENCE here.  lock:add to TOS, xchg, etc
-          // are all faster when the write buffer is populated.
-          masm.movptr (Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2), (int32_t)NULL_WORD) ;
-          if (os::is_MP()) {
-             masm.lock () ; masm.addl (Address(rsp, 0), 0) ;
-          }
-          masm.cmpptr(Address (tmpReg, ObjectMonitor::succ_offset_in_bytes()-2), (int32_t)NULL_WORD) ;
-          masm.jcc   (Assembler::notZero, LSuccess) ;
- 
-          masm.movptr (boxReg, (int32_t)NULL_WORD) ;                   // box is really EAX
-          if (os::is_MP()) { masm.lock(); }
-          masm.cmpxchgptr(r15_thread, Address(tmpReg, ObjectMonitor::owner_offset_in_bytes()-2));
-          masm.jcc   (Assembler::notEqual, LSuccess) ;
-          // Intentional fall-through into slow-path
- 
-          masm.bind  (LGoSlowPath) ;
-          masm.orl   (boxReg, 1) ;                      // set ICC.ZF=0 to indicate failure
-          masm.jmp   (DONE_LABEL) ;
- 
-          masm.bind  (LSuccess) ;
-          masm.testl (boxReg, 0) ;                      // set ICC.ZF=1 to indicate success
-          masm.jmp   (DONE_LABEL) ;
-        }
- 
-        masm.bind  (Stacked) ;
-        masm.movptr(tmpReg, Address (boxReg, 0)) ;      // re-fetch
-        if (os::is_MP()) { masm.lock(); }
-        masm.cmpxchgptr(tmpReg, Address(objReg, 0)); // Uses RAX which is box
- 
-        if (EmitSync & 65536) {
-           masm.bind (CheckSucc) ;
-        }
-        masm.bind(DONE_LABEL);
-        if (EmitSync & 32768) {
-           masm.nop();                      // avoid branch to branch
-        }
-     }
-   %}
- 
- 
    enc_class enc_rethrow()
    %{
      cbuf.set_insts_mark();
      emit_opcode(cbuf, 0xE9); // jmp entry
      emit_d32_reloc(cbuf,
--- 2597,2606 ----
*** 11451,11481 ****
  %}
  
  // ============================================================================
  // inlined locking and unlocking
  
! instruct cmpFastLock(rFlagsReg cr,
!                      rRegP object, rbx_RegP box, rax_RegI tmp, rRegP scr)
! %{
    match(Set cr (FastLock object box));
    effect(TEMP tmp, TEMP scr, USE_KILL box);
- 
    ins_cost(300);
    format %{ "fastlock $object,$box\t! kills $box,$tmp,$scr" %}
!   ins_encode(Fast_Lock(object, box, tmp, scr));
    ins_pipe(pipe_slow);
  %}
  
! instruct cmpFastUnlock(rFlagsReg cr,
!                        rRegP object, rax_RegP box, rRegP tmp)
! %{
    match(Set cr (FastUnlock object box));
    effect(TEMP tmp, USE_KILL box);
- 
    ins_cost(300);
    format %{ "fastunlock $object,$box\t! kills $box,$tmp" %}
!   ins_encode(Fast_Unlock(object, box, tmp));
    ins_pipe(pipe_slow);
  %}
  
  
  // ============================================================================
--- 11226,11254 ----
  %}
  
  // ============================================================================
  // inlined locking and unlocking
  
! instruct cmpFastLock(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI tmp, rRegP scr) %{
    match(Set cr (FastLock object box));
    effect(TEMP tmp, TEMP scr, USE_KILL box);
    ins_cost(300);
    format %{ "fastlock $object,$box\t! kills $box,$tmp,$scr" %}
!   ins_encode %{
!     __ fast_lock($object$$Register, $box$$Register, $tmp$$Register, $scr$$Register, _counters);
!   %}
    ins_pipe(pipe_slow);
  %}
  
! instruct cmpFastUnlock(rFlagsReg cr, rRegP object, rax_RegP box, rRegP tmp) %{
    match(Set cr (FastUnlock object box));
    effect(TEMP tmp, USE_KILL box);
    ins_cost(300);
    format %{ "fastunlock $object,$box\t! kills $box,$tmp" %}
!   ins_encode %{
!     __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register);
!   %}
    ins_pipe(pipe_slow);
  %}
  
  
  // ============================================================================