< prev index next >

src/hotspot/cpu/x86/macroAssembler_x86.cpp

Print this page
rev 57595 : v2.09a with 8235795, 8235931 and 8236035 extracted; rebased to jdk-14+28; merge with 8236035.patch.cr1; merge with 8235795.patch.cr1; merge with 8236035.patch.cr2; merge with 8235795.patch.cr2; merge with 8235795.patch.cr3.

*** 1294,1303 **** --- 1294,1355 ---- jcc(Assembler::equal, done); } #ifdef COMPILER2 + // Increment the ObjectMonitor's ref_count for safety or force a branch + // to 'done' with ICC.ZF=0 to indicate failure/take the slow path. + void MacroAssembler::inc_om_ref_count(Register obj_reg, Register om_reg, Register tmp_reg, Label& done) { + atomic_incl(Address(om_reg, OM_OFFSET_NO_MONITOR_VALUE_TAG(ref_count))); + + Label LGoSlowPath; + if (AsyncDeflateIdleMonitors) { + // Race here if monitor is not owned! The above ref_count bump + // will cause subsequent async deflation to skip it. However, + // previous or concurrent async deflation is a race. + + // First check: if the owner field == DEFLATER_MARKER: + movptr(tmp_reg, Address(om_reg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner))); + // DEFLATER_MARKER == reinterpret_cast<void*>(-1) so the compiler + // doesn't like to use the define here: + cmpptr(tmp_reg, -1); + // If marked for async deflation, then take the slow path. This is a + // simpler check than what ObjectMonitorHandle::save_om_ptr() does + // so ObjectMonitor::install_displaced_markword_in_object() doesn't + // have to be implemented in macro assembler. + jccb(Assembler::equal, LGoSlowPath); + + // Second check: if ref_count field <= 0: + movptr(tmp_reg, Address(om_reg, OM_OFFSET_NO_MONITOR_VALUE_TAG(ref_count))); + cmpptr(tmp_reg, 0); + // If async deflation is in the process of bailing out, but has not + // yet restored the ref_count field, then we take the slow path. We + // want a stable ref_count value for the fast path. + jccb(Assembler::lessEqual, LGoSlowPath); + + // Final check: if object field == obj_reg: + cmpptr(obj_reg, Address(om_reg, OM_OFFSET_NO_MONITOR_VALUE_TAG(object))); + // If the ObjectMonitor has been deflated and recycled, then take + // the slow path. + jccb(Assembler::notEqual, LGoSlowPath); + } + + Label LRetToCaller; + // We leave the ref_count incremented to protect the caller's code + // paths against async deflation. + jmpb(LRetToCaller); + + bind(LGoSlowPath); + lock(); + decrementl(Address(om_reg, OM_OFFSET_NO_MONITOR_VALUE_TAG(ref_count))); + // Jump to 'done' with ICC.ZF=0 to indicate failure/take the slow path. + orl(tmp_reg, 1); + jmp(done); + + bind(LRetToCaller); + } + #if INCLUDE_RTM_OPT // Update rtm_counters based on abort status // input: abort_status // rtm_counters (RTMLockingCounters*)
*** 1527,1541 **** Metadata* method_data, bool profile_rtm, Label& DONE_LABEL) { assert(UseRTMLocking, "why call this otherwise?"); assert(tmpReg == rax, ""); assert(scrReg == rdx, ""); ! Label L_rtm_retry, L_decrement_retry, L_on_abort; int owner_offset = OM_OFFSET_NO_MONITOR_VALUE_TAG(owner); // Without cast to int32_t this style of movptr will destroy r10 which is typically obj. movptr(Address(boxReg, 0), (int32_t)intptr_t(markWord::unused_mark().value())); movptr(boxReg, tmpReg); // Save ObjectMonitor address if (RTMRetryCount > 0) { movl(retry_on_busy_count_Reg, RTMRetryCount); // Retry on lock busy movl(retry_on_abort_count_Reg, RTMRetryCount); // Retry on abort --- 1579,1603 ---- Metadata* method_data, bool profile_rtm, Label& DONE_LABEL) { assert(UseRTMLocking, "why call this otherwise?"); assert(tmpReg == rax, ""); assert(scrReg == rdx, ""); ! Label L_rtm_retry, L_decrement_retry, L_on_abort, L_local_done; int owner_offset = OM_OFFSET_NO_MONITOR_VALUE_TAG(owner); // Without cast to int32_t this style of movptr will destroy r10 which is typically obj. movptr(Address(boxReg, 0), (int32_t)intptr_t(markWord::unused_mark().value())); + + if (!HandshakeAfterDeflateIdleMonitors) { + // Increment the ObjectMonitor's ref_count for safety or force the + // enter slow path via DONE_LABEL. + // In rtm_inflated_locking(), initially tmpReg contains the object's + // mark word which, in this case, is the (ObjectMonitor* | monitor_value). + // Also this code uses scrReg as its temporary register. + inc_om_ref_count(objReg, tmpReg /* om_reg */, scrReg /* tmp_reg */, DONE_LABEL); + } + movptr(boxReg, tmpReg); // Save ObjectMonitor address if (RTMRetryCount > 0) { movl(retry_on_busy_count_Reg, RTMRetryCount); // Retry on lock busy movl(retry_on_abort_count_Reg, RTMRetryCount); // Retry on abort
*** 1553,1563 **** } xbegin(L_on_abort); movptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes())); movptr(tmpReg, Address(tmpReg, owner_offset)); testptr(tmpReg, tmpReg); ! jcc(Assembler::zero, DONE_LABEL); if (UseRTMXendForLockBusy) { xend(); jmp(L_decrement_retry); } else { --- 1615,1625 ---- } xbegin(L_on_abort); movptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes())); movptr(tmpReg, Address(tmpReg, owner_offset)); testptr(tmpReg, tmpReg); ! jcc(Assembler::zero, L_local_done); if (UseRTMXendForLockBusy) { xend(); jmp(L_decrement_retry); } else {
*** 1588,1605 **** lock(); cmpxchgptr(threadReg, Address(boxReg, owner_offset)); // Updates tmpReg if (RTMRetryCount > 0) { // success done else retry ! jccb(Assembler::equal, DONE_LABEL) ; bind(L_decrement_retry); // Spin and retry if lock is busy. rtm_retry_lock_on_busy(retry_on_busy_count_Reg, boxReg, tmpReg, scrReg, L_rtm_retry); } else { bind(L_decrement_retry); } } #endif // INCLUDE_RTM_OPT // fast_lock and fast_unlock used by C2 --- 1650,1680 ---- lock(); cmpxchgptr(threadReg, Address(boxReg, owner_offset)); // Updates tmpReg if (RTMRetryCount > 0) { // success done else retry ! jccb(Assembler::equal, L_local_done); bind(L_decrement_retry); // Spin and retry if lock is busy. rtm_retry_lock_on_busy(retry_on_busy_count_Reg, boxReg, tmpReg, scrReg, L_rtm_retry); } else { bind(L_decrement_retry); } + + // rtm_inflated_locking() exit paths come here except for a failed + // inc_om_ref_count() which goes directly to DONE_LABEL. + bind(L_local_done); + if (!HandshakeAfterDeflateIdleMonitors) { + pushf(); // Preserve flags. + // Decrement the ObjectMonitor's ref_count. + lock(); + decrementl(Address(boxReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(ref_count))); + popf(); // Restore flags so we have the proper ICC.ZF value. + } + + jmp(DONE_LABEL) ; } #endif // INCLUDE_RTM_OPT // fast_lock and fast_unlock used by C2
*** 1821,1838 **** // Invariant: m->_recursions should already be 0, so we don't need to explicitly set it. // Intentional fall-through into DONE_LABEL ... #else // _LP64 // It's inflated and we use scrReg for ObjectMonitor* in this section. movq(scrReg, tmpReg); ! xorq(tmpReg, tmpReg); ! lock(); ! cmpxchgptr(r15_thread, Address(scrReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner))); // Unconditionally set box->_displaced_header = markWord::unused_mark(). // Without cast to int32_t this style of movptr will destroy r10 which is typically obj. movptr(Address(boxReg, 0), (int32_t)intptr_t(markWord::unused_mark().value())); // Intentional fall-through into DONE_LABEL ... // Propagate ICC.ZF from CAS above into DONE_LABEL. #endif // _LP64 #if INCLUDE_RTM_OPT } // use_rtm() #endif // DONE_LABEL is a hot target - we'd really like to place it at the --- 1896,1932 ---- // Invariant: m->_recursions should already be 0, so we don't need to explicitly set it. // Intentional fall-through into DONE_LABEL ... #else // _LP64 // It's inflated and we use scrReg for ObjectMonitor* in this section. movq(scrReg, tmpReg); ! // Unconditionally set box->_displaced_header = markWord::unused_mark(). // Without cast to int32_t this style of movptr will destroy r10 which is typically obj. movptr(Address(boxReg, 0), (int32_t)intptr_t(markWord::unused_mark().value())); + + if (!HandshakeAfterDeflateIdleMonitors) { + // Increment the ObjectMonitor's ref_count for safety or force the + // enter slow path via DONE_LABEL. + // In fast_lock(), scrReg contains the object's mark word which, + // in this case, is the (ObjectMonitor* | monitor_value). Also this + // code uses tmpReg as its temporary register. + inc_om_ref_count(objReg, scrReg /* om_reg */, tmpReg /* tmp_reg */, DONE_LABEL); + } + + xorq(tmpReg, tmpReg); + lock(); + cmpxchgptr(r15_thread, Address(scrReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner))); // Intentional fall-through into DONE_LABEL ... // Propagate ICC.ZF from CAS above into DONE_LABEL. + + if (!HandshakeAfterDeflateIdleMonitors) { + pushf(); // Preserve flags. + // Decrement the ObjectMonitor's ref_count. + lock(); + decrementl(Address(scrReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(ref_count))); + popf(); // Restore flags so we have the proper ICC.ZF value. + } #endif // _LP64 #if INCLUDE_RTM_OPT } // use_rtm() #endif // DONE_LABEL is a hot target - we'd really like to place it at the
*** 1908,1929 **** cmpptr(Address(boxReg, 0), (int32_t)NULL_WORD); // Examine the displaced header jcc (Assembler::zero, DONE_LABEL); // 0 indicates recursive stack-lock movptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes())); // Examine the object's markword testptr(tmpReg, markWord::monitor_value); // Inflated? ! jccb (Assembler::zero, Stacked); // It's inflated. #if INCLUDE_RTM_OPT if (use_rtm) { Label L_regular_inflated_unlock; int owner_offset = OM_OFFSET_NO_MONITOR_VALUE_TAG(owner); movptr(boxReg, Address(tmpReg, owner_offset)); testptr(boxReg, boxReg); jccb(Assembler::notZero, L_regular_inflated_unlock); xend(); ! jmpb(DONE_LABEL); bind(L_regular_inflated_unlock); } #endif // Despite our balanced locking property we still check that m->_owner == Self --- 2002,2023 ---- cmpptr(Address(boxReg, 0), (int32_t)NULL_WORD); // Examine the displaced header jcc (Assembler::zero, DONE_LABEL); // 0 indicates recursive stack-lock movptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes())); // Examine the object's markword testptr(tmpReg, markWord::monitor_value); // Inflated? ! jcc (Assembler::zero, Stacked); // It's inflated. #if INCLUDE_RTM_OPT if (use_rtm) { Label L_regular_inflated_unlock; int owner_offset = OM_OFFSET_NO_MONITOR_VALUE_TAG(owner); movptr(boxReg, Address(tmpReg, owner_offset)); testptr(boxReg, boxReg); jccb(Assembler::notZero, L_regular_inflated_unlock); xend(); ! jmp(DONE_LABEL); bind(L_regular_inflated_unlock); } #endif // Despite our balanced locking property we still check that m->_owner == Self
*** 1981,2002 **** // most efficient "long" NOP encodings. // Unfortunately none of our alignment mechanisms suffice. bind (CheckSucc); #else // _LP64 // It's inflated xorptr(boxReg, boxReg); orptr(boxReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(recursions))); ! jccb (Assembler::notZero, DONE_LABEL); movptr(boxReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(cxq))); orptr(boxReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(EntryList))); jccb (Assembler::notZero, CheckSucc); // Without cast to int32_t this style of movptr will destroy r10 which is typically obj. movptr(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)), (int32_t)NULL_WORD); ! jmpb (DONE_LABEL); - // Try to avoid passing control into the slow_path ... - Label LSuccess, LGoSlowPath ; bind (CheckSucc); // The following optional optimization can be elided if necessary // Effectively: if (succ == null) goto slow path // The code reduces the window for a race, however, --- 2075,2106 ---- // most efficient "long" NOP encodings. // Unfortunately none of our alignment mechanisms suffice. bind (CheckSucc); #else // _LP64 // It's inflated + + if (!HandshakeAfterDeflateIdleMonitors) { + // Increment the ObjectMonitor's ref_count for safety or force the + // exit slow path via DONE_LABEL. + // In fast_unlock(), tmpReg contains the object's mark word which, + // in this case, is the (ObjectMonitor* | monitor_value). Also this + // code uses boxReg as its temporary register. + inc_om_ref_count(objReg, tmpReg /* om_reg */, boxReg /* tmp_reg */, DONE_LABEL); + } + + // Try to avoid passing control into the slow path ... + Label LSuccess, LGoSlowPath; xorptr(boxReg, boxReg); orptr(boxReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(recursions))); ! jccb(Assembler::notZero, LGoSlowPath); movptr(boxReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(cxq))); orptr(boxReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(EntryList))); jccb (Assembler::notZero, CheckSucc); // Without cast to int32_t this style of movptr will destroy r10 which is typically obj. movptr(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)), (int32_t)NULL_WORD); ! jmpb(LSuccess); bind (CheckSucc); // The following optional optimization can be elided if necessary // Effectively: if (succ == null) goto slow path // The code reduces the window for a race, however,
*** 2041,2054 **** --- 2145,2166 ---- // lock so we're done (and exit was a success). jccb (Assembler::notEqual, LSuccess); // Intentional fall-through into slow path bind (LGoSlowPath); + if (!HandshakeAfterDeflateIdleMonitors) { + lock(); + decrementl(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(ref_count))); + } orl (boxReg, 1); // set ICC.ZF=0 to indicate failure jmpb (DONE_LABEL); bind (LSuccess); + if (!HandshakeAfterDeflateIdleMonitors) { + lock(); + decrementl(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(ref_count))); + } testl (boxReg, 0); // set ICC.ZF=1 to indicate success jmpb (DONE_LABEL); bind (Stacked); movptr(tmpReg, Address (boxReg, 0)); // re-fetch
< prev index next >