< prev index next >
src/hotspot/cpu/x86/macroAssembler_x86.cpp
Print this page
rev 58110 : v2.09a with 8235795, 8235931 and 8236035 extracted; rebased to jdk-14+28; merge with 8236035.patch.cr1; merge with 8235795.patch.cr1; merge with 8236035.patch.cr2; merge with 8235795.patch.cr2; merge with 8235795.patch.cr3.
rev 58111 : See CR9-to-CR10-changes; merge with jdk-15+11.
*** 1283,1344 ****
jcc(Assembler::equal, done);
}
#ifdef COMPILER2
- // Increment the ObjectMonitor's ref_count for safety or force a branch
- // to 'done' with ICC.ZF=0 to indicate failure/take the slow path.
- void MacroAssembler::inc_om_ref_count(Register obj_reg, Register om_reg, Register tmp_reg, Label& done) {
- atomic_incl(Address(om_reg, OM_OFFSET_NO_MONITOR_VALUE_TAG(ref_count)));
-
- Label LGoSlowPath;
- if (AsyncDeflateIdleMonitors) {
- // Race here if monitor is not owned! The above ref_count bump
- // will cause subsequent async deflation to skip it. However,
- // previous or concurrent async deflation is a race.
-
- // First check: if the owner field == DEFLATER_MARKER:
- movptr(tmp_reg, Address(om_reg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)));
- // DEFLATER_MARKER == reinterpret_cast<void*>(-1) so the compiler
- // doesn't like to use the define here:
- cmpptr(tmp_reg, -1);
- // If marked for async deflation, then take the slow path. This is a
- // simpler check than what ObjectMonitorHandle::save_om_ptr() does
- // so ObjectMonitor::install_displaced_markword_in_object() doesn't
- // have to be implemented in macro assembler.
- jccb(Assembler::equal, LGoSlowPath);
-
- // Second check: if ref_count field <= 0:
- movptr(tmp_reg, Address(om_reg, OM_OFFSET_NO_MONITOR_VALUE_TAG(ref_count)));
- cmpptr(tmp_reg, 0);
- // If async deflation is in the process of bailing out, but has not
- // yet restored the ref_count field, then we take the slow path. We
- // want a stable ref_count value for the fast path.
- jccb(Assembler::lessEqual, LGoSlowPath);
-
- // Final check: if object field == obj_reg:
- cmpptr(obj_reg, Address(om_reg, OM_OFFSET_NO_MONITOR_VALUE_TAG(object)));
- // If the ObjectMonitor has been deflated and recycled, then take
- // the slow path.
- jccb(Assembler::notEqual, LGoSlowPath);
- }
-
- Label LRetToCaller;
- // We leave the ref_count incremented to protect the caller's code
- // paths against async deflation.
- jmpb(LRetToCaller);
-
- bind(LGoSlowPath);
- lock();
- decrementl(Address(om_reg, OM_OFFSET_NO_MONITOR_VALUE_TAG(ref_count)));
- // Jump to 'done' with ICC.ZF=0 to indicate failure/take the slow path.
- orl(tmp_reg, 1);
- jmp(done);
-
- bind(LRetToCaller);
- }
-
#if INCLUDE_RTM_OPT
// Update rtm_counters based on abort status
// input: abort_status
// rtm_counters (RTMLockingCounters*)
--- 1283,1292 ----
*** 1568,1592 ****
Metadata* method_data, bool profile_rtm,
Label& DONE_LABEL) {
assert(UseRTMLocking, "why call this otherwise?");
assert(tmpReg == rax, "");
assert(scrReg == rdx, "");
! Label L_rtm_retry, L_decrement_retry, L_on_abort, L_local_done;
int owner_offset = OM_OFFSET_NO_MONITOR_VALUE_TAG(owner);
// Without cast to int32_t this style of movptr will destroy r10 which is typically obj.
movptr(Address(boxReg, 0), (int32_t)intptr_t(markWord::unused_mark().value()));
-
- if (!HandshakeAfterDeflateIdleMonitors) {
- // Increment the ObjectMonitor's ref_count for safety or force the
- // enter slow path via DONE_LABEL.
- // In rtm_inflated_locking(), initially tmpReg contains the object's
- // mark word which, in this case, is the (ObjectMonitor* | monitor_value).
- // Also this code uses scrReg as its temporary register.
- inc_om_ref_count(objReg, tmpReg /* om_reg */, scrReg /* tmp_reg */, DONE_LABEL);
- }
-
movptr(boxReg, tmpReg); // Save ObjectMonitor address
if (RTMRetryCount > 0) {
movl(retry_on_busy_count_Reg, RTMRetryCount); // Retry on lock busy
movl(retry_on_abort_count_Reg, RTMRetryCount); // Retry on abort
--- 1516,1530 ----
Metadata* method_data, bool profile_rtm,
Label& DONE_LABEL) {
assert(UseRTMLocking, "why call this otherwise?");
assert(tmpReg == rax, "");
assert(scrReg == rdx, "");
! Label L_rtm_retry, L_decrement_retry, L_on_abort;
int owner_offset = OM_OFFSET_NO_MONITOR_VALUE_TAG(owner);
// Without cast to int32_t this style of movptr will destroy r10 which is typically obj.
movptr(Address(boxReg, 0), (int32_t)intptr_t(markWord::unused_mark().value()));
movptr(boxReg, tmpReg); // Save ObjectMonitor address
if (RTMRetryCount > 0) {
movl(retry_on_busy_count_Reg, RTMRetryCount); // Retry on lock busy
movl(retry_on_abort_count_Reg, RTMRetryCount); // Retry on abort
*** 1604,1614 ****
}
xbegin(L_on_abort);
movptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes()));
movptr(tmpReg, Address(tmpReg, owner_offset));
testptr(tmpReg, tmpReg);
! jcc(Assembler::zero, L_local_done);
if (UseRTMXendForLockBusy) {
xend();
jmp(L_decrement_retry);
}
else {
--- 1542,1552 ----
}
xbegin(L_on_abort);
movptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes()));
movptr(tmpReg, Address(tmpReg, owner_offset));
testptr(tmpReg, tmpReg);
! jcc(Assembler::zero, DONE_LABEL);
if (UseRTMXendForLockBusy) {
xend();
jmp(L_decrement_retry);
}
else {
*** 1639,1669 ****
lock();
cmpxchgptr(threadReg, Address(boxReg, owner_offset)); // Updates tmpReg
if (RTMRetryCount > 0) {
// success done else retry
! jccb(Assembler::equal, L_local_done);
bind(L_decrement_retry);
// Spin and retry if lock is busy.
rtm_retry_lock_on_busy(retry_on_busy_count_Reg, boxReg, tmpReg, scrReg, L_rtm_retry);
}
else {
bind(L_decrement_retry);
}
-
- // rtm_inflated_locking() exit paths come here except for a failed
- // inc_om_ref_count() which goes directly to DONE_LABEL.
- bind(L_local_done);
- if (!HandshakeAfterDeflateIdleMonitors) {
- pushf(); // Preserve flags.
- // Decrement the ObjectMonitor's ref_count.
- lock();
- decrementl(Address(boxReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(ref_count)));
- popf(); // Restore flags so we have the proper ICC.ZF value.
- }
-
- jmp(DONE_LABEL) ;
}
#endif // INCLUDE_RTM_OPT
// fast_lock and fast_unlock used by C2
--- 1577,1594 ----
lock();
cmpxchgptr(threadReg, Address(boxReg, owner_offset)); // Updates tmpReg
if (RTMRetryCount > 0) {
// success done else retry
! jccb(Assembler::equal, DONE_LABEL) ;
bind(L_decrement_retry);
// Spin and retry if lock is busy.
rtm_retry_lock_on_busy(retry_on_busy_count_Reg, boxReg, tmpReg, scrReg, L_rtm_retry);
}
else {
bind(L_decrement_retry);
}
}
#endif // INCLUDE_RTM_OPT
// fast_lock and fast_unlock used by C2
*** 1885,1921 ****
// Invariant: m->_recursions should already be 0, so we don't need to explicitly set it.
// Intentional fall-through into DONE_LABEL ...
#else // _LP64
// It's inflated and we use scrReg for ObjectMonitor* in this section.
movq(scrReg, tmpReg);
-
- // Unconditionally set box->_displaced_header = markWord::unused_mark().
- // Without cast to int32_t this style of movptr will destroy r10 which is typically obj.
- movptr(Address(boxReg, 0), (int32_t)intptr_t(markWord::unused_mark().value()));
-
- if (!HandshakeAfterDeflateIdleMonitors) {
- // Increment the ObjectMonitor's ref_count for safety or force the
- // enter slow path via DONE_LABEL.
- // In fast_lock(), scrReg contains the object's mark word which,
- // in this case, is the (ObjectMonitor* | monitor_value). Also this
- // code uses tmpReg as its temporary register.
- inc_om_ref_count(objReg, scrReg /* om_reg */, tmpReg /* tmp_reg */, DONE_LABEL);
- }
-
xorq(tmpReg, tmpReg);
lock();
cmpxchgptr(r15_thread, Address(scrReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)));
// Intentional fall-through into DONE_LABEL ...
// Propagate ICC.ZF from CAS above into DONE_LABEL.
-
- if (!HandshakeAfterDeflateIdleMonitors) {
- pushf(); // Preserve flags.
- // Decrement the ObjectMonitor's ref_count.
- lock();
- decrementl(Address(scrReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(ref_count)));
- popf(); // Restore flags so we have the proper ICC.ZF value.
- }
#endif // _LP64
#if INCLUDE_RTM_OPT
} // use_rtm()
#endif
// DONE_LABEL is a hot target - we'd really like to place it at the
--- 1810,1827 ----
// Invariant: m->_recursions should already be 0, so we don't need to explicitly set it.
// Intentional fall-through into DONE_LABEL ...
#else // _LP64
// It's inflated and we use scrReg for ObjectMonitor* in this section.
movq(scrReg, tmpReg);
xorq(tmpReg, tmpReg);
lock();
cmpxchgptr(r15_thread, Address(scrReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)));
+ // Unconditionally set box->_displaced_header = markWord::unused_mark().
+ // Without cast to int32_t this style of movptr will destroy r10 which is typically obj.
+ movptr(Address(boxReg, 0), (int32_t)intptr_t(markWord::unused_mark().value()));
// Intentional fall-through into DONE_LABEL ...
// Propagate ICC.ZF from CAS above into DONE_LABEL.
#endif // _LP64
#if INCLUDE_RTM_OPT
} // use_rtm()
#endif
// DONE_LABEL is a hot target - we'd really like to place it at the
*** 1991,2012 ****
cmpptr(Address(boxReg, 0), (int32_t)NULL_WORD); // Examine the displaced header
jcc (Assembler::zero, DONE_LABEL); // 0 indicates recursive stack-lock
movptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes())); // Examine the object's markword
testptr(tmpReg, markWord::monitor_value); // Inflated?
! jcc (Assembler::zero, Stacked);
// It's inflated.
#if INCLUDE_RTM_OPT
if (use_rtm) {
Label L_regular_inflated_unlock;
int owner_offset = OM_OFFSET_NO_MONITOR_VALUE_TAG(owner);
movptr(boxReg, Address(tmpReg, owner_offset));
testptr(boxReg, boxReg);
jccb(Assembler::notZero, L_regular_inflated_unlock);
xend();
! jmp(DONE_LABEL);
bind(L_regular_inflated_unlock);
}
#endif
// Despite our balanced locking property we still check that m->_owner == Self
--- 1897,1918 ----
cmpptr(Address(boxReg, 0), (int32_t)NULL_WORD); // Examine the displaced header
jcc (Assembler::zero, DONE_LABEL); // 0 indicates recursive stack-lock
movptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes())); // Examine the object's markword
testptr(tmpReg, markWord::monitor_value); // Inflated?
! jccb (Assembler::zero, Stacked);
// It's inflated.
#if INCLUDE_RTM_OPT
if (use_rtm) {
Label L_regular_inflated_unlock;
int owner_offset = OM_OFFSET_NO_MONITOR_VALUE_TAG(owner);
movptr(boxReg, Address(tmpReg, owner_offset));
testptr(boxReg, boxReg);
jccb(Assembler::notZero, L_regular_inflated_unlock);
xend();
! jmpb(DONE_LABEL);
bind(L_regular_inflated_unlock);
}
#endif
// Despite our balanced locking property we still check that m->_owner == Self
*** 2064,2095 ****
// most efficient "long" NOP encodings.
// Unfortunately none of our alignment mechanisms suffice.
bind (CheckSucc);
#else // _LP64
// It's inflated
-
- if (!HandshakeAfterDeflateIdleMonitors) {
- // Increment the ObjectMonitor's ref_count for safety or force the
- // exit slow path via DONE_LABEL.
- // In fast_unlock(), tmpReg contains the object's mark word which,
- // in this case, is the (ObjectMonitor* | monitor_value). Also this
- // code uses boxReg as its temporary register.
- inc_om_ref_count(objReg, tmpReg /* om_reg */, boxReg /* tmp_reg */, DONE_LABEL);
- }
-
- // Try to avoid passing control into the slow path ...
- Label LSuccess, LGoSlowPath;
xorptr(boxReg, boxReg);
orptr(boxReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(recursions)));
! jccb(Assembler::notZero, LGoSlowPath);
movptr(boxReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(cxq)));
orptr(boxReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(EntryList)));
jccb (Assembler::notZero, CheckSucc);
// Without cast to int32_t this style of movptr will destroy r10 which is typically obj.
movptr(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)), (int32_t)NULL_WORD);
! jmpb(LSuccess);
bind (CheckSucc);
// The following optional optimization can be elided if necessary
// Effectively: if (succ == null) goto slow path
// The code reduces the window for a race, however,
--- 1970,1991 ----
// most efficient "long" NOP encodings.
// Unfortunately none of our alignment mechanisms suffice.
bind (CheckSucc);
#else // _LP64
// It's inflated
xorptr(boxReg, boxReg);
orptr(boxReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(recursions)));
! jccb (Assembler::notZero, DONE_LABEL);
movptr(boxReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(cxq)));
orptr(boxReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(EntryList)));
jccb (Assembler::notZero, CheckSucc);
// Without cast to int32_t this style of movptr will destroy r10 which is typically obj.
movptr(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)), (int32_t)NULL_WORD);
! jmpb (DONE_LABEL);
+ // Try to avoid passing control into the slow_path ...
+ Label LSuccess, LGoSlowPath ;
bind (CheckSucc);
// The following optional optimization can be elided if necessary
// Effectively: if (succ == null) goto slow path
// The code reduces the window for a race, however,
*** 2134,2155 ****
// lock so we're done (and exit was a success).
jccb (Assembler::notEqual, LSuccess);
// Intentional fall-through into slow path
bind (LGoSlowPath);
- if (!HandshakeAfterDeflateIdleMonitors) {
- lock();
- decrementl(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(ref_count)));
- }
orl (boxReg, 1); // set ICC.ZF=0 to indicate failure
jmpb (DONE_LABEL);
bind (LSuccess);
- if (!HandshakeAfterDeflateIdleMonitors) {
- lock();
- decrementl(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(ref_count)));
- }
testl (boxReg, 0); // set ICC.ZF=1 to indicate success
jmpb (DONE_LABEL);
bind (Stacked);
movptr(tmpReg, Address (boxReg, 0)); // re-fetch
--- 2030,2043 ----
< prev index next >