src/cpu/x86/vm/macroAssembler_x86.cpp
Index
Unified diffs
Context diffs
Sdiffs
Wdiffs
Patch
New
Old
Previous File
Next File
8031320_9 Cdiff src/cpu/x86/vm/macroAssembler_x86.cpp
src/cpu/x86/vm/macroAssembler_x86.cpp
Print this page
*** 299,309 ****
void MacroAssembler::mov_metadata(Address dst, Metadata* obj) {
mov_literal32(dst, (int32_t)obj, metadata_Relocation::spec_for_immediate());
}
! void MacroAssembler::movptr(Register dst, AddressLiteral src) {
if (src.is_lval()) {
mov_literal32(dst, (intptr_t)src.target(), src.rspec());
} else {
movl(dst, as_Address(src));
}
--- 299,309 ----
void MacroAssembler::mov_metadata(Address dst, Metadata* obj) {
mov_literal32(dst, (int32_t)obj, metadata_Relocation::spec_for_immediate());
}
! void MacroAssembler::movptr(Register dst, AddressLiteral src, Register scratch) {
if (src.is_lval()) {
mov_literal32(dst, (intptr_t)src.target(), src.rspec());
} else {
movl(dst, as_Address(src));
}
*** 611,620 ****
--- 611,629 ----
if (value == 0) { ; return; }
if (value == 1 && UseIncDec) { decq(dst) ; return; }
/* else */ { subq(dst, value) ; return; }
}
+ void MacroAssembler::incrementq(AddressLiteral dst) {
+ if (reachable(dst)) {
+ incrementq(as_Address(dst));
+ } else {
+ lea(rscratch1, dst);
+ incrementq(Address(rscratch1, 0));
+ }
+ }
+
void MacroAssembler::incrementq(Register reg, int value) {
if (value == min_jint) { addq(reg, value); return; }
if (value < 0) { decrementq(reg, -value); return; }
if (value == 0) { ; return; }
if (value == 1 && UseIncDec) { incq(reg) ; return; }
*** 679,697 ****
void MacroAssembler::mov_metadata(Address dst, Metadata* obj) {
mov_literal64(rscratch1, (intptr_t)obj, metadata_Relocation::spec_for_immediate());
movq(dst, rscratch1);
}
! void MacroAssembler::movptr(Register dst, AddressLiteral src) {
if (src.is_lval()) {
mov_literal64(dst, (intptr_t)src.target(), src.rspec());
} else {
if (reachable(src)) {
movq(dst, as_Address(src));
} else {
! lea(rscratch1, src);
! movq(dst, Address(rscratch1,0));
}
}
}
void MacroAssembler::movptr(ArrayAddress dst, Register src) {
--- 688,706 ----
void MacroAssembler::mov_metadata(Address dst, Metadata* obj) {
mov_literal64(rscratch1, (intptr_t)obj, metadata_Relocation::spec_for_immediate());
movq(dst, rscratch1);
}
! void MacroAssembler::movptr(Register dst, AddressLiteral src, Register scratch) {
if (src.is_lval()) {
mov_literal64(dst, (intptr_t)src.target(), src.rspec());
} else {
if (reachable(src)) {
movq(dst, as_Address(src));
} else {
! lea(scratch, src);
! movq(dst, Address(scratch,0));
}
}
}
void MacroAssembler::movptr(ArrayAddress dst, Register src) {
*** 986,1009 ****
void MacroAssembler::andptr(Register dst, int32_t imm32) {
LP64_ONLY(andq(dst, imm32)) NOT_LP64(andl(dst, imm32));
}
! void MacroAssembler::atomic_incl(AddressLiteral counter_addr) {
! pushf();
! if (reachable(counter_addr)) {
if (os::is_MP())
lock();
! incrementl(as_Address(counter_addr));
} else {
! lea(rscratch1, counter_addr);
if (os::is_MP())
lock();
! incrementl(Address(rscratch1, 0));
}
- popf();
}
// Writes to stack successive pages until offset reached to check for
// stack overflow + shadow pages. This clobbers tmp.
void MacroAssembler::bang_stack_size(Register size, Register tmp) {
movptr(tmp, rsp);
--- 995,1035 ----
void MacroAssembler::andptr(Register dst, int32_t imm32) {
LP64_ONLY(andq(dst, imm32)) NOT_LP64(andl(dst, imm32));
}
! void MacroAssembler::atomic_incl(Address counter_addr) {
if (os::is_MP())
lock();
! incrementl(counter_addr);
! }
!
! void MacroAssembler::atomic_incl(AddressLiteral counter_addr, Register scr) {
! if (reachable(counter_addr)) {
! atomic_incl(as_Address(counter_addr));
} else {
! lea(scr, counter_addr);
! atomic_incl(Address(scr, 0));
! }
! }
!
! #ifdef _LP64
! void MacroAssembler::atomic_incq(Address counter_addr) {
if (os::is_MP())
lock();
! incrementq(counter_addr);
! }
!
! void MacroAssembler::atomic_incq(AddressLiteral counter_addr, Register scr) {
! if (reachable(counter_addr)) {
! atomic_incq(as_Address(counter_addr));
! } else {
! lea(scr, counter_addr);
! atomic_incq(Address(scr, 0));
}
}
+ #endif
// Writes to stack successive pages until offset reached to check for
// stack overflow + shadow pages. This clobbers tmp.
void MacroAssembler::bang_stack_size(Register size, Register tmp) {
movptr(tmp, rsp);
*** 1272,1281 ****
--- 1298,1457 ----
cmpptr(temp_reg, markOopDesc::biased_lock_pattern);
jcc(Assembler::equal, done);
}
#ifdef COMPILER2
+
+ #if INCLUDE_RTM_OPT
+
+ // Update rtmcounters based on abort status
+ // input: tmpReg (abort status)
+ // scrReg (RTMLockingCounters*)
+ // flag register as scratch
+ void MacroAssembler::rtmcounters_update(Register tmpReg, Register scrReg) {
+
+ atomic_incptr(Address(scrReg, RTMLockingCounters::abort_count_offset()));
+ if (PrintPreciseRTMLockingStatistics) {
+ for (int i = 0; i < RTMLockingCounters::ABORT_STATUS_LIMIT; i++) {
+ Label check_abort;
+ testl(tmpReg, (1<<i));
+ jccb(Assembler::equal, check_abort);
+ atomic_incptr(Address(scrReg, RTMLockingCounters::abortX_count_offset() + (i * sizeof(uintx))));
+ bind(check_abort);
+ }
+ }
+ }
+
+ // Branch if (orandom & count != 0)
+ // tmpReg, scrReg and flags as scratch
+ void MacroAssembler::branch_on_random_using_rdtsc(Register tmpReg, Register scrReg, int count, Label& brLabel) {
+ assert(tmpReg == rax, "");
+ assert(scrReg == rdx, "");
+ rdtsc(); // modifies EDX:EAX
+ andptr(tmpReg, count-1);
+ jccb(Assembler::notZero, brLabel);
+ }
+
+ // Perform abort ratio calculation, set no_rtm bit if high ratio
+ // input: boxReg (object monitor address) (unused for inflated locks)
+ // input: scrReg (RTMLockingCounters* address)
+ // tmpReg, scrReg and flags as scratch
+ // output: boxReg set to 1 if dontelide, 0 if elide (for stack locks)
+ void MacroAssembler::rtm_abortratio_calculation(Register boxReg, Register tmpReg, Register scrReg,
+ RTMLockingCounters* rtmcounters,
+ Metadata* method_data,
+ bool isStackLock) {
+ Label L_done, L_check_always_rtm1, L_check_always_rtm2;
+
+ if (isStackLock) {
+ movptr(boxReg, 0);
+ }
+ if (RTMLockingCalculationDelay > 0) {
+ // Delay calculation
+ movptr(tmpReg, ExternalAddress((address) RTMLockingCounters::rtm_calculation_flag()), tmpReg);
+ testptr(tmpReg, tmpReg);
+ jccb(Assembler::equal, L_done);
+ }
+ // Abort ratio calculation only if abort_count > RTMAbortThreshold
+ // Aborted transactions = abort_count * 100
+ // All transactions = total_count * RTMTotalCountIncrRate
+ // Set no_rtm bit if (Aborted transactions >= All transactions * RTMAbortRatio)
+
+ movptr(tmpReg, Address(scrReg, RTMLockingCounters::abort_count_offset()));
+ cmpptr(tmpReg, RTMAbortThreshold);
+ jccb(Assembler::below, L_check_always_rtm2);
+ imulptr(tmpReg, tmpReg, 100);
+ movptr(scrReg, Address(scrReg, RTMLockingCounters::total_count_offset()));
+ imulptr(scrReg, scrReg, RTMTotalCountIncrRate);
+ imulptr(scrReg, scrReg, RTMAbortRatio);
+ cmpptr(tmpReg, scrReg);
+ jccb(Assembler::below, L_check_always_rtm1);
+ if (isStackLock) {
+ increment(boxReg); // = 1
+ }
+ if (method_data != NULL) {
+ // set rtm_state to "no rtm" in method oop
+ mov_metadata(tmpReg, method_data);
+ if (os::is_MP()) {
+ lock();
+ }
+ orl(Address(tmpReg, MethodData::rtm_state_offset_in_bytes()), NoRTM);
+ }
+ jmpb(L_done);
+ bind(L_check_always_rtm1);
+ // Reload RTMLockingCounters* address
+ lea(scrReg, ExternalAddress((address)rtmcounters));
+ bind(L_check_always_rtm2);
+ movptr(tmpReg, Address(scrReg, RTMLockingCounters::total_count_offset()));
+ cmpptr(tmpReg, RTMLockingThreshold / RTMTotalCountIncrRate);
+ jccb(Assembler::below, L_done);
+ if (method_data != NULL) {
+ // set rtm_state to "always rtm" in method oop
+ mov_metadata(tmpReg, method_data);
+ if (os::is_MP()) {
+ lock();
+ }
+ orl(Address(tmpReg, MethodData::rtm_state_offset_in_bytes()), UseRTM);
+ }
+ bind(L_done);
+ }
+
+ // Retry on lock abort if abort status is 0x2
+ // inputs: boxReg (monitor address), countReg (retry count),
+ // : tmpReg(abort status)
+ // output: tmpReg set to boxReg, countReg decremented by 1
+ // flags as scratch
+ void MacroAssembler::rtm_retry_lockabort(Register countReg, Register boxReg, Register tmpReg, Label& retryLabel, bool isStackLock) {
+ Label doneRetry;
+
+ assert(tmpReg == rax, "");
+ // The abort reason bits are in eax (see all states in rtmLocking.hpp)
+ // 0x6 = conflict on which we can retry (0x2) | memory conflict (0x4)
+ // if reason is in 0x6 and retry count != 0 then retry
+ andptr(tmpReg, 0x6);
+ jccb(Assembler::zero, doneRetry);
+ testl(countReg, countReg);
+ jccb(Assembler::zero, doneRetry);
+ pause();
+ decrementl(countReg);
+ if (!isStackLock) {
+ movptr(tmpReg, boxReg);
+ }
+ jmp(retryLabel);
+ bind(doneRetry);
+ }
+
+ // Spin and retry if lock is busy,
+ // inputs: boxReg (monitor address), countReg (retry count)
+ // output: tmpReg set to boxReg, countReg decremented by 1
+ // : clear z flags if retry count exceeded
+ // scrReg as scratch
+ void MacroAssembler::rtm_retry_lockbusy(Register countReg, Register boxReg, Register tmpReg, Register scrReg, Label& retryLabel) {
+ Label SpinLoop, SpinExit, doneRetry;
+
+ testl(countReg, countReg);
+ jccb(Assembler::zero, doneRetry);
+ decrementl(countReg);
+ movptr(scrReg, RTMSpinLoopCount);
+
+ bind(SpinLoop);
+ pause();
+ decrementl(scrReg);
+ jccb(Assembler::lessEqual, SpinExit);
+ movptr(tmpReg, Address(boxReg, ObjectMonitor::owner_offset_in_bytes()-2)) ;
+ testptr(tmpReg, tmpReg) ;
+ jccb(Assembler::notZero, SpinLoop) ;
+
+ bind(SpinExit);
+ movptr(tmpReg, boxReg);
+ jmp(retryLabel);
+ bind(doneRetry);
+ incrementl(countReg); // clear z flag
+ }
+
+ #endif // INCLUDE_RTM_OPT
+
// Fast_Lock and Fast_Unlock used by C2
// Because the transitions from emitted code to the runtime
// monitorenter/exit helper stubs are so slow it's critical that
// we inline both the stack-locking fast-path and the inflated fast path.
*** 1348,1368 ****
// obj: object to lock
// box: on-stack box address (displaced header location) - KILLED
// rax,: tmp -- KILLED
// scr: tmp -- KILLED
! void MacroAssembler::fast_lock(Register objReg, Register boxReg, Register tmpReg, Register scrReg, BiasedLockingCounters* counters) {
// Ensure the register assignents are disjoint
! guarantee (objReg != boxReg, "");
! guarantee (objReg != tmpReg, "");
! guarantee (objReg != scrReg, "");
! guarantee (boxReg != tmpReg, "");
! guarantee (boxReg != scrReg, "");
! guarantee (tmpReg == rax, "");
if (counters != NULL) {
! atomic_incl(ExternalAddress((address)counters->total_entry_count_addr()));
}
if (EmitSync & 1) {
// set box->dhw = unused_mark (3)
// Force all sync thru slow-path: slow_enter() and slow_exit()
movptr (Address(boxReg, 0), (int32_t)intptr_t(markOopDesc::unused_mark()));
--- 1524,1553 ----
// obj: object to lock
// box: on-stack box address (displaced header location) - KILLED
// rax,: tmp -- KILLED
// scr: tmp -- KILLED
! void MacroAssembler::fast_lock(Register objReg, Register boxReg, Register tmpReg,
! Register scrReg, Register cx1Reg, Register cx2Reg,
! BiasedLockingCounters* counters,
! RTMLockingCounters* rtmcounters,
! RTMLockingCounters* stackrtmcounters,
! Metadata* method_data,
! bool use_rtm, bool profile_rtm) {
// Ensure the register assignents are disjoint
! assert(tmpReg == rax, "");
!
! if (use_rtm) {
! assert_different_registers(objReg, boxReg, tmpReg, scrReg, cx1Reg, cx2Reg);
! } else {
! assert(cx1Reg == noreg, "");
! assert(cx2Reg == noreg, "");
! assert_different_registers(objReg, boxReg, tmpReg, scrReg);
! }
if (counters != NULL) {
! atomic_incl(ExternalAddress((address)counters->total_entry_count_addr()), scrReg);
}
if (EmitSync & 1) {
// set box->dhw = unused_mark (3)
// Force all sync thru slow-path: slow_enter() and slow_exit()
movptr (Address(boxReg, 0), (int32_t)intptr_t(markOopDesc::unused_mark()));
*** 1417,1428 ****
// If this invariant is not held we risk exclusion (safety) failure.
if (UseBiasedLocking && !UseOptoBiasInlining) {
biased_locking_enter(boxReg, objReg, tmpReg, scrReg, true, DONE_LABEL, NULL, counters);
}
movptr(tmpReg, Address(objReg, 0)); // [FETCH]
! testl (tmpReg, markOopDesc::monitor_value); // inflated vs stack-locked|neutral|biased
jccb (Assembler::notZero, IsInflated);
// Attempt stack-locking ...
orptr (tmpReg, 0x1);
movptr(Address(boxReg, 0), tmpReg); // Anticipate successful CAS
--- 1602,1689 ----
// If this invariant is not held we risk exclusion (safety) failure.
if (UseBiasedLocking && !UseOptoBiasInlining) {
biased_locking_enter(boxReg, objReg, tmpReg, scrReg, true, DONE_LABEL, NULL, counters);
}
+ #if INCLUDE_RTM_OPT
+ if (UseRTMForStackLocks && use_rtm) {
+ assert(!UseBiasedLocking, "Biased locking is not supported with RTM locking");
+ Label L_rtm_retry, L_decrement_retry, L_on_abort;
+
+ if (RTMRetryCount > 0) {
+ movl(cx2Reg, RTMRetryCount); // Retry on abort
+ bind(L_rtm_retry);
+ }
+ if (!UseRTMXendForLockBusy) {
+ movptr(tmpReg, Address(objReg, 0));
+ testptr(tmpReg, markOopDesc::monitor_value); // inflated vs stack-locked|neutral|biased
+ jcc(Assembler::notZero, IsInflated);
+ }
+ if (PrintPreciseRTMLockingStatistics || profile_rtm) {
+ Label L_noincrement;
+ if (RTMTotalCountIncrRate > 1) {
+ // tmpReg, scrReg and flags as scratch
+ branch_on_random_using_rdtsc(tmpReg, scrReg, (int)RTMTotalCountIncrRate, L_noincrement);
+ }
+ assert(stackrtmcounters != NULL, "should not be NULL when profiling RTM");
+ atomic_incptr(ExternalAddress((address)stackrtmcounters->total_count_addr()), scrReg);
+ bind(L_noincrement);
+ }
+ xbegin(L_on_abort);
+ movptr(tmpReg, Address(objReg, 0)); // fetch markword
+ andptr(tmpReg, markOopDesc::biased_lock_mask_in_place); // look at 3 lock bits
+ cmpptr(tmpReg, markOopDesc::unlocked_value); // bits = 001 unlocked
+ jcc(Assembler::equal, DONE_LABEL); // all done if unlocked
+ if (UseRTMXendForLockBusy) {
+ xend();
+ movptr(tmpReg, Address(objReg, 0));
+ testptr(tmpReg, markOopDesc::monitor_value); // inflated vs stack-locked|neutral|biased
+ jcc(Assembler::notZero, IsInflated);
+ movptr(tmpReg,0x1); // Set the transaction status in rax (tmpReg)
+ jmp(L_decrement_retry);
+ }
+ else {
+ xabort(0);
+ }
+ bind(L_on_abort);
+ if (PrintPreciseRTMLockingStatistics || profile_rtm) {
+ assert(stackrtmcounters != NULL, "should not be NULL when profiling RTM");
+ // update rtm counters based on rax value at abort
+ // reads tmpReg(rax), updates flags
+ lea(scrReg, ExternalAddress((address)stackrtmcounters));
+ rtmcounters_update(tmpReg, scrReg);
+ }
+ if (profile_rtm) {
+ if (RTMRetryCount > 0) {
+ // Save abort status
+ push(tmpReg);
+ }
+ // Perform abort ratio calculation, set dontelide bit and rtm_state
+ // input: scrReg (stackrtmcounters address)
+ // output: cx1Reg (=1 if dont elide, =0 if elide)
+ // tmpReg, scrReg, flags as scratch
+ assert(stackrtmcounters != NULL, "should not be NULL when profiling RTM");
+ rtm_abortratio_calculation(cx1Reg, tmpReg, scrReg, stackrtmcounters, method_data, true);
+
+ // restore abort status
+ if (RTMRetryCount > 0) {
+ pop(tmpReg);
+ }
+ }
+ bind(L_decrement_retry);
+ if (RTMRetryCount > 0) {
+ // retry on lock abort if abort status is one of 0xD
+ // inputs: cx2Reg (retry count),
+ // : tmpReg(abort status)
+ // output: cx2Reg decremented by 1
+ rtm_retry_lockabort(cx2Reg, noreg, tmpReg, L_rtm_retry, true);
+ }
+ }
+ #endif // INCLUDE_RTM_OPT
+
movptr(tmpReg, Address(objReg, 0)); // [FETCH]
! testptr(tmpReg, markOopDesc::monitor_value); // inflated vs stack-locked|neutral|biased
jccb (Assembler::notZero, IsInflated);
// Attempt stack-locking ...
orptr (tmpReg, 0x1);
movptr(Address(boxReg, 0), tmpReg); // Anticipate successful CAS
*** 1432,1454 ****
cmpxchgptr(boxReg, Address(objReg, 0)); // Updates tmpReg
if (counters != NULL) {
cond_inc32(Assembler::equal,
ExternalAddress((address)counters->fast_path_entry_count_addr()));
}
! jccb(Assembler::equal, DONE_LABEL);
// Recursive locking
subptr(tmpReg, rsp);
andptr(tmpReg, (int32_t) (NOT_LP64(0xFFFFF003) LP64_ONLY(7 - os::vm_page_size())) );
movptr(Address(boxReg, 0), tmpReg);
if (counters != NULL) {
cond_inc32(Assembler::equal,
ExternalAddress((address)counters->fast_path_entry_count_addr()));
}
! jmpb(DONE_LABEL);
bind(IsInflated);
#ifndef _LP64
// The object is inflated.
//
// TODO-FIXME: eliminate the ugly use of manifest constants:
// Use markOopDesc::monitor_value instead of "2".
--- 1693,1819 ----
cmpxchgptr(boxReg, Address(objReg, 0)); // Updates tmpReg
if (counters != NULL) {
cond_inc32(Assembler::equal,
ExternalAddress((address)counters->fast_path_entry_count_addr()));
}
! jcc(Assembler::equal, DONE_LABEL);
// Recursive locking
subptr(tmpReg, rsp);
andptr(tmpReg, (int32_t) (NOT_LP64(0xFFFFF003) LP64_ONLY(7 - os::vm_page_size())) );
movptr(Address(boxReg, 0), tmpReg);
if (counters != NULL) {
cond_inc32(Assembler::equal,
ExternalAddress((address)counters->fast_path_entry_count_addr()));
}
! jmp(DONE_LABEL);
bind(IsInflated);
+ // The object is inflated.
+
+ #if INCLUDE_RTM_OPT
+ // Use the same RTM locking code in 32- and 64-bit VM.
+ if (use_rtm) {
+ Label L_rtm_retry, L_decrement_retry, L_on_abort;
+
+ // Without cast to int32_t a movptr will destroy r10 which is typically obj
+ movptr(Address(boxReg, 0), (int32_t)intptr_t(markOopDesc::unused_mark()));
+ movptr(boxReg, tmpReg);
+
+ if (RTMRetryCount > 0) {
+ movl(cx1Reg, RTMRetryCount); // Retry on lock busy
+ movl(cx2Reg, RTMRetryCount); // Retry on abort
+ bind(L_rtm_retry);
+ }
+ if (PrintPreciseRTMLockingStatistics || profile_rtm) {
+ Label L_noincrement;
+ if (RTMTotalCountIncrRate > 1) {
+ // tmpReg, scrReg and flags as scratch
+ branch_on_random_using_rdtsc(tmpReg, scrReg, (int)RTMTotalCountIncrRate, L_noincrement);
+ }
+ assert(rtmcounters != NULL, "should not be NULL when profiling RTM");
+ atomic_incptr(ExternalAddress((address)rtmcounters->total_count_addr()), scrReg);
+ bind(L_noincrement);
+ }
+ xbegin(L_on_abort);
+ movptr(tmpReg, Address(objReg,0));
+ movptr(tmpReg, Address(tmpReg, ObjectMonitor::owner_offset_in_bytes()-2));
+ testptr(tmpReg, tmpReg) ;
+ jcc(Assembler::zero, DONE_LABEL);
+ if (UseRTMXendForLockBusy) {
+ xend();
+ jmp(L_decrement_retry);
+ }
+ else {
+ xabort(0);
+ }
+ bind(L_on_abort);
+ if (PrintPreciseRTMLockingStatistics || profile_rtm) {
+ assert(rtmcounters != NULL, "should not be NULL when profiling RTM");
+ // update rtm counters based on rax value at abort
+ // reads tmpReg(rax), updates flags
+ lea(scrReg, ExternalAddress((address)rtmcounters));
+ rtmcounters_update(tmpReg, scrReg);
+ }
+ if (profile_rtm) {
+ // Save abort status
+ if (RTMRetryCount > 0)
+ push(tmpReg);
+
+ // Perform abort ratio calculation, set dontelide bit and rtm_state
+ // input: boxReg (object monitor address)
+ // input: scrReg (rtmcounters address)
+ // tmpReg, scrReg, flags as scratch
+ assert(rtmcounters != NULL, "should not be NULL when profiling RTM");
+ rtm_abortratio_calculation(boxReg, tmpReg, scrReg, rtmcounters, method_data, false);
+
+ // restore abort status
+ if (RTMRetryCount > 0)
+ pop(tmpReg);
+ }
+ if (RTMRetryCount > 0) {
+ // retry on lock abort if abort status is one of 0xD
+ // inputs: boxReg (monitor address), cx2Reg (retry count),
+ // : tmpReg(abort status)
+ // output: tmpReg set to boxReg, cx2Reg decremented by 1
+ rtm_retry_lockabort(cx2Reg, boxReg, tmpReg, L_rtm_retry, false);
+ }
+ movptr(tmpReg, boxReg);
+
+ movptr(tmpReg, Address(tmpReg, ObjectMonitor::owner_offset_in_bytes()-2)) ;
+ testptr(tmpReg, tmpReg) ;
+ jccb(Assembler::notZero, L_decrement_retry) ;
+
+ // Appears unlocked - try to swing _owner from null to non-null.
+ // Use either "Self" (in threadReg) or rsp as thread identity in _owner.
+ // Invariant: tmpReg == 0. tmpReg is EAX which is the implicit cmpxchg comparand.
+ #ifdef _LP64
+ Register threadReg = r15_thread;
+ #else
+ get_thread(scrReg);
+ Register threadReg = scrReg;
+ #endif
+ if (os::is_MP()) {
+ lock();
+ }
+ cmpxchgptr(threadReg, Address(boxReg, ObjectMonitor::owner_offset_in_bytes()-2)); // Updates tmpReg
+
+ if (RTMRetryCount > 0) {
+ // success done else retry
+ jccb(Assembler::equal, DONE_LABEL) ;
+ // inputs: boxReg (monitor address), cx1Reg (retry count)
+ // output: tmpReg set to boxReg, cx1Reg decremented by 1
+ // : clear z flags if retry count exceeded, scrReg scratch
+ bind(L_decrement_retry);
+ rtm_retry_lockbusy(cx1Reg, boxReg, tmpReg, scrReg, L_rtm_retry);
+ }
+ else {
+ bind(L_decrement_retry);
+ }
+ } else { // !use_rtm()
+ #endif // INCLUDE_RTM_OPT
+
#ifndef _LP64
// The object is inflated.
//
// TODO-FIXME: eliminate the ugly use of manifest constants:
// Use markOopDesc::monitor_value instead of "2".
*** 1574,1597 ****
// avoid an RTO->RTS upgrade on the $line.
// Without cast to int32_t a movptr will destroy r10 which is typically obj
movptr(Address(boxReg, 0), (int32_t)intptr_t(markOopDesc::unused_mark()));
! mov (boxReg, tmpReg);
movptr (tmpReg, Address(boxReg, ObjectMonitor::owner_offset_in_bytes()-2));
testptr(tmpReg, tmpReg);
jccb (Assembler::notZero, DONE_LABEL);
// It's inflated and appears unlocked
if (os::is_MP()) {
lock();
}
cmpxchgptr(r15_thread, Address(boxReg, ObjectMonitor::owner_offset_in_bytes()-2));
// Intentional fall-through into DONE_LABEL ...
#endif
-
// DONE_LABEL is a hot target - we'd really like to place it at the
// start of cache line by padding with NOPs.
// See the AMD and Intel software optimization manuals for the
// most efficient "long" NOP encodings.
// Unfortunately none of our alignment mechanisms suffice.
--- 1939,1964 ----
// avoid an RTO->RTS upgrade on the $line.
// Without cast to int32_t a movptr will destroy r10 which is typically obj
movptr(Address(boxReg, 0), (int32_t)intptr_t(markOopDesc::unused_mark()));
! movptr (boxReg, tmpReg);
movptr (tmpReg, Address(boxReg, ObjectMonitor::owner_offset_in_bytes()-2));
testptr(tmpReg, tmpReg);
jccb (Assembler::notZero, DONE_LABEL);
// It's inflated and appears unlocked
if (os::is_MP()) {
lock();
}
cmpxchgptr(r15_thread, Address(boxReg, ObjectMonitor::owner_offset_in_bytes()-2));
// Intentional fall-through into DONE_LABEL ...
+ #endif // _LP64
+ #if INCLUDE_RTM_OPT
+ } // use_rtm()
#endif
// DONE_LABEL is a hot target - we'd really like to place it at the
// start of cache line by padding with NOPs.
// See the AMD and Intel software optimization manuals for the
// most efficient "long" NOP encodings.
// Unfortunately none of our alignment mechanisms suffice.
*** 1629,1643 ****
// The only other source of unbalanced locking would be JNI. The "Java Native Interface:
// Programmer's Guide and Specification" claims that an object locked by jni_monitorenter
// should not be unlocked by "normal" java-level locking and vice-versa. The specification
// doesn't specify what will occur if a program engages in such mixed-mode locking, however.
! void MacroAssembler::fast_unlock(Register objReg, Register boxReg, Register tmpReg) {
! guarantee (objReg != boxReg, "");
! guarantee (objReg != tmpReg, "");
! guarantee (boxReg != tmpReg, "");
! guarantee (boxReg == rax, "");
if (EmitSync & 4) {
// Disable - inhibit all inlining. Force control through the slow-path
cmpptr (rsp, 0);
} else
--- 1996,2008 ----
// The only other source of unbalanced locking would be JNI. The "Java Native Interface:
// Programmer's Guide and Specification" claims that an object locked by jni_monitorenter
// should not be unlocked by "normal" java-level locking and vice-versa. The specification
// doesn't specify what will occur if a program engages in such mixed-mode locking, however.
! void MacroAssembler::fast_unlock(Register objReg, Register boxReg, Register tmpReg, bool use_rtm) {
! assert(boxReg == rax, "");
! assert_different_registers(objReg, boxReg, tmpReg);
if (EmitSync & 4) {
// Disable - inhibit all inlining. Force control through the slow-path
cmpptr (rsp, 0);
} else
*** 1665,1682 ****
// and appear before the (box->dhw == 0) recursive stack-lock test.
if (UseBiasedLocking && !UseOptoBiasInlining) {
biased_locking_exit(objReg, tmpReg, DONE_LABEL);
}
cmpptr(Address(boxReg, 0), (int32_t)NULL_WORD); // Examine the displaced header
movptr(tmpReg, Address(objReg, 0)); // Examine the object's markword
! jccb (Assembler::zero, DONE_LABEL); // 0 indicates recursive stack-lock
!
! testptr(tmpReg, 0x02); // Inflated?
! jccb (Assembler::zero, Stacked);
// It's inflated.
// Despite our balanced locking property we still check that m->_owner == Self
// as java routines or native JNI code called by this thread might
// have released the lock.
// Refer to the comments in synchronizer.cpp for how we might encode extra
// state in _succ so we can avoid fetching EntryList|cxq.
--- 2030,2072 ----
// and appear before the (box->dhw == 0) recursive stack-lock test.
if (UseBiasedLocking && !UseOptoBiasInlining) {
biased_locking_exit(objReg, tmpReg, DONE_LABEL);
}
+ #if INCLUDE_RTM_OPT
+ if (UseRTMForStackLocks && use_rtm) {
+ assert(!UseBiasedLocking, "Biased locking is not supported with RTM locking");
+ Label L_regular_unlock;
+ movptr(tmpReg, Address(objReg, 0)); // fetch markword
+ andptr(tmpReg, markOopDesc::biased_lock_mask_in_place); // look at 3 lock bits
+ cmpptr(tmpReg, markOopDesc::unlocked_value); // bits = 001 unlocked
+ jccb(Assembler::notEqual, L_regular_unlock); // if !HLE RegularLock
+ xend(); // otherwise end...
+ jmp(DONE_LABEL); // ... and we're done
+ bind(L_regular_unlock);
+ }
+ #endif
+
cmpptr(Address(boxReg, 0), (int32_t)NULL_WORD); // Examine the displaced header
+ jcc (Assembler::zero, DONE_LABEL); // 0 indicates recursive stack-lock
movptr(tmpReg, Address(objReg, 0)); // Examine the object's markword
! testptr(tmpReg, markOopDesc::monitor_value); // Inflated?
! jcc (Assembler::zero, Stacked);
// It's inflated.
+ #if INCLUDE_RTM_OPT
+ if (use_rtm) {
+ Label L_regular_inflated_unlock;
+ movptr(boxReg, Address(tmpReg, ObjectMonitor::owner_offset_in_bytes()-2)) ;
+ testptr(boxReg, boxReg) ;
+ jccb(Assembler::notZero, L_regular_inflated_unlock) ;
+ xend();
+ jmp(DONE_LABEL) ;
+ bind(L_regular_inflated_unlock);
+ }
+ #endif
+
// Despite our balanced locking property we still check that m->_owner == Self
// as java routines or native JNI code called by this thread might
// have released the lock.
// Refer to the comments in synchronizer.cpp for how we might encode extra
// state in _succ so we can avoid fetching EntryList|cxq.
*** 2446,2456 ****
--- 2836,2848 ----
void MacroAssembler::cond_inc32(Condition cond, AddressLiteral counter_addr) {
Condition negated_cond = negate_condition(cond);
Label L;
jcc(negated_cond, L);
+ pushf(); // Preserve flags
atomic_incl(counter_addr);
+ popf();
bind(L);
}
int MacroAssembler::corrected_idivl(Register reg) {
// Full implementation of Java idiv and irem; checks for
src/cpu/x86/vm/macroAssembler_x86.cpp
Index
Unified diffs
Context diffs
Sdiffs
Wdiffs
Patch
New
Old
Previous File
Next File