1567 rtm_retry_lock_on_abort(retry_on_abort_count_Reg, abort_status_Reg, L_rtm_retry);
1568 }
1569 }
1570
1571 // Use RTM for inflating locks
1572 // inputs: objReg (object to lock)
1573 // boxReg (on-stack box address (displaced header location) - KILLED)
1574 // tmpReg (ObjectMonitor address + markWord::monitor_value)
1575 void MacroAssembler::rtm_inflated_locking(Register objReg, Register boxReg, Register tmpReg,
1576 Register scrReg, Register retry_on_busy_count_Reg,
1577 Register retry_on_abort_count_Reg,
1578 RTMLockingCounters* rtm_counters,
1579 Metadata* method_data, bool profile_rtm,
1580 Label& DONE_LABEL) {
1581 assert(UseRTMLocking, "why call this otherwise?");
1582 assert(tmpReg == rax, "");
1583 assert(scrReg == rdx, "");
1584 Label L_rtm_retry, L_decrement_retry, L_on_abort, L_local_done;
1585 int owner_offset = OM_OFFSET_NO_MONITOR_VALUE_TAG(owner);
1586
1587 if (!HandshakeAfterDeflateIdleMonitors) {
1588 // Increment the ObjectMonitor's ref_count for safety or force the
1589 // enter slow path via DONE_LABEL.
1590 // In rtm_inflated_locking(), initially tmpReg contains the object's
1591 // mark word which, in this case, is the (ObjectMonitor* | monitor_value).
1592 // Also this code uses scrReg as its temporary register.
1593 inc_om_ref_count(objReg, tmpReg /* om_reg */, scrReg /* tmp_reg */, DONE_LABEL);
1594 }
1595
1596 // Without cast to int32_t this style of movptr will destroy r10 which is typically obj.
1597 movptr(Address(boxReg, 0), (int32_t)intptr_t(markWord::unused_mark().value()));
1598 movptr(boxReg, tmpReg); // Save ObjectMonitor address
1599
1600 if (RTMRetryCount > 0) {
1601 movl(retry_on_busy_count_Reg, RTMRetryCount); // Retry on lock busy
1602 movl(retry_on_abort_count_Reg, RTMRetryCount); // Retry on abort
1603 bind(L_rtm_retry);
1604 }
1605 if (PrintPreciseRTMLockingStatistics || profile_rtm) {
1606 Label L_noincrement;
1607 if (RTMTotalCountIncrRate > 1) {
1608 // tmpReg, scrReg and flags are killed
1609 branch_on_random_using_rdtsc(tmpReg, scrReg, RTMTotalCountIncrRate, L_noincrement);
1610 }
1611 assert(rtm_counters != NULL, "should not be NULL when profiling RTM");
1612 atomic_incptr(ExternalAddress((address)rtm_counters->total_count_addr()), scrReg);
1613 bind(L_noincrement);
1614 }
1615 xbegin(L_on_abort);
1616 movptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes()));
1617 movptr(tmpReg, Address(tmpReg, owner_offset));
1881 movptr(Address(scrReg, 0), 3); // box->_displaced_header = 3
1882 // If we weren't able to swing _owner from NULL to the BasicLock
1883 // then take the slow path.
1884 jccb (Assembler::notZero, DONE_LABEL);
1885 // update _owner from BasicLock to thread
1886 get_thread (scrReg); // beware: clobbers ICCs
1887 movptr(Address(boxReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)), scrReg);
1888 xorptr(boxReg, boxReg); // set icc.ZFlag = 1 to indicate success
1889
1890 // If the CAS fails we can either retry or pass control to the slow path.
1891 // We use the latter tactic.
1892 // Pass the CAS result in the icc.ZFlag into DONE_LABEL
1893 // If the CAS was successful ...
1894 // Self has acquired the lock
1895 // Invariant: m->_recursions should already be 0, so we don't need to explicitly set it.
1896 // Intentional fall-through into DONE_LABEL ...
1897 #else // _LP64
1898 // It's inflated and we use scrReg for ObjectMonitor* in this section.
1899 movq(scrReg, tmpReg);
1900
1901 if (!HandshakeAfterDeflateIdleMonitors) {
1902 // Increment the ObjectMonitor's ref_count for safety or force the
1903 // enter slow path via DONE_LABEL.
1904 // In fast_lock(), scrReg contains the object's mark word which,
1905 // in this case, is the (ObjectMonitor* | monitor_value). Also this
1906 // code uses tmpReg as its temporary register.
1907 inc_om_ref_count(objReg, scrReg /* om_reg */, tmpReg /* tmp_reg */, DONE_LABEL);
1908 }
1909
1910 xorq(tmpReg, tmpReg);
1911 lock();
1912 cmpxchgptr(r15_thread, Address(scrReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)));
1913 // Unconditionally set box->_displaced_header = markWord::unused_mark().
1914 // Without cast to int32_t this style of movptr will destroy r10 which is typically obj.
1915 movptr(Address(boxReg, 0), (int32_t)intptr_t(markWord::unused_mark().value()));
1916 // Intentional fall-through into DONE_LABEL ...
1917 // Propagate ICC.ZF from CAS above into DONE_LABEL.
1918
1919 if (!HandshakeAfterDeflateIdleMonitors) {
1920 pushf(); // Preserve flags.
1921 // Decrement the ObjectMonitor's ref_count.
1922 lock();
1923 decrementl(Address(scrReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(ref_count)));
1924 popf(); // Restore flags so we have the proper ICC.ZF value.
1925 }
1926 #endif // _LP64
1927 #if INCLUDE_RTM_OPT
1928 } // use_rtm()
1929 #endif
1930 // DONE_LABEL is a hot target - we'd really like to place it at the
1931 // start of cache line by padding with NOPs.
1932 // See the AMD and Intel software optimization manuals for the
1933 // most efficient "long" NOP encodings.
1934 // Unfortunately none of our alignment mechanisms suffice.
1935 bind(DONE_LABEL);
|
1567 rtm_retry_lock_on_abort(retry_on_abort_count_Reg, abort_status_Reg, L_rtm_retry);
1568 }
1569 }
1570
1571 // Use RTM for inflating locks
1572 // inputs: objReg (object to lock)
1573 // boxReg (on-stack box address (displaced header location) - KILLED)
1574 // tmpReg (ObjectMonitor address + markWord::monitor_value)
1575 void MacroAssembler::rtm_inflated_locking(Register objReg, Register boxReg, Register tmpReg,
1576 Register scrReg, Register retry_on_busy_count_Reg,
1577 Register retry_on_abort_count_Reg,
1578 RTMLockingCounters* rtm_counters,
1579 Metadata* method_data, bool profile_rtm,
1580 Label& DONE_LABEL) {
1581 assert(UseRTMLocking, "why call this otherwise?");
1582 assert(tmpReg == rax, "");
1583 assert(scrReg == rdx, "");
1584 Label L_rtm_retry, L_decrement_retry, L_on_abort, L_local_done;
1585 int owner_offset = OM_OFFSET_NO_MONITOR_VALUE_TAG(owner);
1586
1587 // Without cast to int32_t this style of movptr will destroy r10 which is typically obj.
1588 movptr(Address(boxReg, 0), (int32_t)intptr_t(markWord::unused_mark().value()));
1589
1590 if (!HandshakeAfterDeflateIdleMonitors) {
1591 // Increment the ObjectMonitor's ref_count for safety or force the
1592 // enter slow path via DONE_LABEL.
1593 // In rtm_inflated_locking(), initially tmpReg contains the object's
1594 // mark word which, in this case, is the (ObjectMonitor* | monitor_value).
1595 // Also this code uses scrReg as its temporary register.
1596 inc_om_ref_count(objReg, tmpReg /* om_reg */, scrReg /* tmp_reg */, DONE_LABEL);
1597 }
1598
1599 movptr(boxReg, tmpReg); // Save ObjectMonitor address
1600
1601 if (RTMRetryCount > 0) {
1602 movl(retry_on_busy_count_Reg, RTMRetryCount); // Retry on lock busy
1603 movl(retry_on_abort_count_Reg, RTMRetryCount); // Retry on abort
1604 bind(L_rtm_retry);
1605 }
1606 if (PrintPreciseRTMLockingStatistics || profile_rtm) {
1607 Label L_noincrement;
1608 if (RTMTotalCountIncrRate > 1) {
1609 // tmpReg, scrReg and flags are killed
1610 branch_on_random_using_rdtsc(tmpReg, scrReg, RTMTotalCountIncrRate, L_noincrement);
1611 }
1612 assert(rtm_counters != NULL, "should not be NULL when profiling RTM");
1613 atomic_incptr(ExternalAddress((address)rtm_counters->total_count_addr()), scrReg);
1614 bind(L_noincrement);
1615 }
1616 xbegin(L_on_abort);
1617 movptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes()));
1618 movptr(tmpReg, Address(tmpReg, owner_offset));
1882 movptr(Address(scrReg, 0), 3); // box->_displaced_header = 3
1883 // If we weren't able to swing _owner from NULL to the BasicLock
1884 // then take the slow path.
1885 jccb (Assembler::notZero, DONE_LABEL);
1886 // update _owner from BasicLock to thread
1887 get_thread (scrReg); // beware: clobbers ICCs
1888 movptr(Address(boxReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)), scrReg);
1889 xorptr(boxReg, boxReg); // set icc.ZFlag = 1 to indicate success
1890
1891 // If the CAS fails we can either retry or pass control to the slow path.
1892 // We use the latter tactic.
1893 // Pass the CAS result in the icc.ZFlag into DONE_LABEL
1894 // If the CAS was successful ...
1895 // Self has acquired the lock
1896 // Invariant: m->_recursions should already be 0, so we don't need to explicitly set it.
1897 // Intentional fall-through into DONE_LABEL ...
1898 #else // _LP64
1899 // It's inflated and we use scrReg for ObjectMonitor* in this section.
1900 movq(scrReg, tmpReg);
1901
1902 // Unconditionally set box->_displaced_header = markWord::unused_mark().
1903 // Without cast to int32_t this style of movptr will destroy r10 which is typically obj.
1904 movptr(Address(boxReg, 0), (int32_t)intptr_t(markWord::unused_mark().value()));
1905
1906 if (!HandshakeAfterDeflateIdleMonitors) {
1907 // Increment the ObjectMonitor's ref_count for safety or force the
1908 // enter slow path via DONE_LABEL.
1909 // In fast_lock(), scrReg contains the object's mark word which,
1910 // in this case, is the (ObjectMonitor* | monitor_value). Also this
1911 // code uses tmpReg as its temporary register.
1912 inc_om_ref_count(objReg, scrReg /* om_reg */, tmpReg /* tmp_reg */, DONE_LABEL);
1913 }
1914
1915 xorq(tmpReg, tmpReg);
1916 lock();
1917 cmpxchgptr(r15_thread, Address(scrReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)));
1918 // Intentional fall-through into DONE_LABEL ...
1919 // Propagate ICC.ZF from CAS above into DONE_LABEL.
1920
1921 if (!HandshakeAfterDeflateIdleMonitors) {
1922 pushf(); // Preserve flags.
1923 // Decrement the ObjectMonitor's ref_count.
1924 lock();
1925 decrementl(Address(scrReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(ref_count)));
1926 popf(); // Restore flags so we have the proper ICC.ZF value.
1927 }
1928 #endif // _LP64
1929 #if INCLUDE_RTM_OPT
1930 } // use_rtm()
1931 #endif
1932 // DONE_LABEL is a hot target - we'd really like to place it at the
1933 // start of cache line by padding with NOPs.
1934 // See the AMD and Intel software optimization manuals for the
1935 // most efficient "long" NOP encodings.
1936 // Unfortunately none of our alignment mechanisms suffice.
1937 bind(DONE_LABEL);
|