< prev index next >
src/cpu/x86/vm/macroAssembler_x86.cpp
Print this page
rev 12152 : [mq]: verification.patch
@@ -43,10 +43,12 @@
#include "utilities/macros.hpp"
#if INCLUDE_ALL_GCS
#include "gc/g1/g1CollectedHeap.inline.hpp"
#include "gc/g1/g1SATBCardTableModRefBS.hpp"
#include "gc/g1/heapRegion.hpp"
+#include "gc/shenandoah/shenandoahHeap.inline.hpp"
+#include "gc/shenandoah/shenandoahHeapRegion.hpp"
#endif // INCLUDE_ALL_GCS
#include "crc32c.h"
#ifdef COMPILER2
#include "opto/intrinsicnode.hpp"
#endif
@@ -1104,10 +1106,12 @@
assert_different_registers(lock_reg, obj_reg, swap_reg, tmp_reg);
assert(markOopDesc::age_shift == markOopDesc::lock_bits + markOopDesc::biased_lock_bits, "biased locking makes assumptions about bit layout");
Address mark_addr (obj_reg, oopDesc::mark_offset_in_bytes());
NOT_LP64( Address saved_mark_addr(lock_reg, 0); )
+ shenandoah_store_addr_check(obj_reg);
+
if (PrintBiasedLockingStatistics && counters == NULL) {
counters = BiasedLocking::counters();
}
// Biased locking
// See whether the lock is currently biased toward our thread and
@@ -1167,11 +1171,11 @@
// If the low three bits in the xor result aren't clear, that means
// the prototype header is no longer biased and we have to revoke
// the bias on this object.
testptr(header_reg, markOopDesc::biased_lock_mask_in_place);
- jccb(Assembler::notZero, try_revoke_bias);
+ jccb_if_possible(Assembler::notZero, try_revoke_bias);
// Biasing is still enabled for this data type. See whether the
// epoch of the current bias is still valid, meaning that the epoch
// bits of the mark word are equal to the epoch bits of the
// prototype header. (Note that the prototype header's epoch bits
@@ -1179,11 +1183,11 @@
// toward the current thread. Note that we must be absolutely sure
// that the current epoch is invalid in order to do this because
// otherwise the manipulations it performs on the mark word are
// illegal.
testptr(header_reg, markOopDesc::epoch_mask_in_place);
- jccb(Assembler::notZero, try_rebias);
+ jccb_if_possible(Assembler::notZero, try_rebias);
// The epoch of the current bias is still valid but we know nothing
// about the owner; it might be set or it might be clear. Try to
// acquire the bias of the object using an atomic operation. If this
// fails we will go in to the runtime to revoke the object's bias.
@@ -1288,10 +1292,11 @@
// Note: we do not have to check the thread ID for two reasons.
// First, the interpreter checks for IllegalMonitorStateException at
// a higher level. Second, if the bias was revoked while we held the
// lock, the object could not be rebiased toward another thread, so
// the bias bit would be clear.
+ shenandoah_store_addr_check(obj_reg); // Access mark word
movptr(temp_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
andptr(temp_reg, markOopDesc::biased_lock_mask_in_place);
cmpptr(temp_reg, markOopDesc::biased_lock_pattern);
jcc(Assembler::equal, done);
}
@@ -1480,10 +1485,11 @@
if (RTMRetryCount > 0) {
movl(retry_on_abort_count_Reg, RTMRetryCount); // Retry on abort
bind(L_rtm_retry);
}
+ shenandoah_store_addr_check(objReg); // Access mark word
movptr(tmpReg, Address(objReg, 0));
testptr(tmpReg, markOopDesc::monitor_value); // inflated vs stack-locked|neutral|biased
jcc(Assembler::notZero, IsInflated);
if (PrintPreciseRTMLockingStatistics || profile_rtm) {
@@ -1556,10 +1562,11 @@
assert(rtm_counters != NULL, "should not be NULL when profiling RTM");
atomic_incptr(ExternalAddress((address)rtm_counters->total_count_addr()), scrReg);
bind(L_noincrement);
}
xbegin(L_on_abort);
+ shenandoah_store_addr_check(objReg); // Access mark word
movptr(tmpReg, Address(objReg, 0));
movptr(tmpReg, Address(tmpReg, owner_offset));
testptr(tmpReg, tmpReg);
jcc(Assembler::zero, DONE_LABEL);
if (UseRTMXendForLockBusy) {
@@ -1702,10 +1709,12 @@
assert(cx1Reg == noreg, "");
assert(cx2Reg == noreg, "");
assert_different_registers(objReg, boxReg, tmpReg, scrReg);
}
+ shenandoah_store_addr_check(objReg); // Access mark word
+
if (counters != NULL) {
atomic_incl(ExternalAddress((address)counters->total_entry_count_addr()), scrReg);
}
if (EmitSync & 1) {
// set box->dhw = markOopDesc::unused_mark()
@@ -1751,11 +1760,11 @@
}
#endif // INCLUDE_RTM_OPT
movptr(tmpReg, Address(objReg, 0)); // [FETCH]
testptr(tmpReg, markOopDesc::monitor_value); // inflated vs stack-locked|neutral|biased
- jccb(Assembler::notZero, IsInflated);
+ jccb_if_possible(Assembler::notZero, IsInflated);
// Attempt stack-locking ...
orptr (tmpReg, markOopDesc::unlocked_value);
movptr(Address(boxReg, 0), tmpReg); // Anticipate successful CAS
if (os::is_MP()) {
@@ -1828,11 +1837,11 @@
} else {
// Can suffer RTS->RTO upgrades on shared or cold $ lines
// Test-And-CAS instead of CAS
movptr(tmpReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner))); // rax, = m->_owner
testptr(tmpReg, tmpReg); // Locked ?
- jccb (Assembler::notZero, DONE_LABEL);
+ jccb_if_possible(Assembler::notZero, DONE_LABEL);
}
// Appears unlocked - try to swing _owner from null to non-null.
// Ideally, I'd manifest "Self" with get_thread and then attempt
// to CAS the register containing Self into m->Owner.
@@ -1846,11 +1855,11 @@
}
cmpxchgptr(scrReg, Address(boxReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)));
movptr(Address(scrReg, 0), 3); // box->_displaced_header = 3
// If we weren't able to swing _owner from NULL to the BasicLock
// then take the slow path.
- jccb (Assembler::notZero, DONE_LABEL);
+ jccb_if_possible(Assembler::notZero, DONE_LABEL);
// update _owner from BasicLock to thread
get_thread (scrReg); // beware: clobbers ICCs
movptr(Address(boxReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)), scrReg);
xorptr(boxReg, boxReg); // set icc.ZFlag = 1 to indicate success
@@ -1876,11 +1885,11 @@
xorptr (tmpReg, tmpReg);
} else {
// Can suffer RTS->RTO upgrades on shared or cold $ lines
movptr(tmpReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner))); // rax, = m->_owner
testptr(tmpReg, tmpReg); // Locked ?
- jccb (Assembler::notZero, DONE_LABEL);
+ jccb_if_possible(Assembler::notZero, DONE_LABEL);
}
// Appears unlocked - try to swing _owner from null to non-null.
// Use either "Self" (in scr) or rsp as thread identity in _owner.
// Invariant: tmpReg == 0. tmpReg is EAX which is the implicit cmpxchg comparand.
@@ -1964,10 +1973,12 @@
void MacroAssembler::fast_unlock(Register objReg, Register boxReg, Register tmpReg, bool use_rtm) {
assert(boxReg == rax, "");
assert_different_registers(objReg, boxReg, tmpReg);
+ shenandoah_store_addr_check(objReg); // Access mark word
+
if (EmitSync & 4) {
// Disable - inhibit all inlining. Force control through the slow-path
cmpptr (rsp, 0);
} else {
Label DONE_LABEL, Stacked, CheckSucc;
@@ -2005,11 +2016,11 @@
int owner_offset = OM_OFFSET_NO_MONITOR_VALUE_TAG(owner);
movptr(boxReg, Address(tmpReg, owner_offset));
testptr(boxReg, boxReg);
jccb(Assembler::notZero, L_regular_inflated_unlock);
xend();
- jmpb(DONE_LABEL);
+ jmpb_if_possible(DONE_LABEL);
bind(L_regular_inflated_unlock);
}
#endif
// Despite our balanced locking property we still check that m->_owner == Self
@@ -2049,21 +2060,21 @@
if ((EmitSync & 65536) == 0 && (EmitSync & 256)) {
// Attempt to reduce branch density - AMD's branch predictor.
orptr(boxReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(recursions)));
orptr(boxReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(EntryList)));
orptr(boxReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(cxq)));
- jccb (Assembler::notZero, DONE_LABEL);
+ jccb_if_possible(Assembler::notZero, DONE_LABEL);
movptr(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)), NULL_WORD);
- jmpb (DONE_LABEL);
+ jmpb_if_possible(DONE_LABEL);
} else {
orptr(boxReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(recursions)));
- jccb (Assembler::notZero, DONE_LABEL);
+ jccb_if_possible(Assembler::notZero, DONE_LABEL);
movptr(boxReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(EntryList)));
orptr(boxReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(cxq)));
jccb (Assembler::notZero, CheckSucc);
movptr(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)), NULL_WORD);
- jmpb (DONE_LABEL);
+ jmpb_if_possible(DONE_LABEL);
}
// The Following code fragment (EmitSync & 65536) improves the performance of
// contended applications and contended synchronization microbenchmarks.
// Unfortunately the emission of the code - even though not executed - causes regressions
@@ -2129,15 +2140,15 @@
movptr(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)), boxReg);
// Intentional fall-through into LGoSlowPath ...
bind (LGoSlowPath);
orptr(boxReg, 1); // set ICC.ZF=0 to indicate failure
- jmpb (DONE_LABEL);
+ jmpb_if_possible(DONE_LABEL);
bind (LSuccess);
xorptr(boxReg, boxReg); // set ICC.ZF=1 to indicate success
- jmpb (DONE_LABEL);
+ jmpb_if_possible(DONE_LABEL);
}
bind (Stacked);
// It's not inflated and it's not recursively stack-locked and it's not biased.
// It must be stack-locked.
@@ -2171,16 +2182,16 @@
xorptr(boxReg, r15_thread);
} else {
xorptr(boxReg, boxReg);
}
orptr(boxReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(recursions)));
- jccb (Assembler::notZero, DONE_LABEL);
+ jccb_if_possible(Assembler::notZero, DONE_LABEL);
movptr(boxReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(cxq)));
orptr(boxReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(EntryList)));
jccb (Assembler::notZero, CheckSucc);
movptr(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)), (int32_t)NULL_WORD);
- jmpb (DONE_LABEL);
+ jmpb_if_possible(DONE_LABEL);
if ((EmitSync & 65536) == 0) {
// Try to avoid passing control into the slow_path ...
Label LSuccess, LGoSlowPath ;
bind (CheckSucc);
@@ -2233,15 +2244,15 @@
jccb (Assembler::notEqual, LSuccess);
// Intentional fall-through into slow-path
bind (LGoSlowPath);
orl (boxReg, 1); // set ICC.ZF=0 to indicate failure
- jmpb (DONE_LABEL);
+ jmpb_if_possible(DONE_LABEL);
bind (LSuccess);
testl (boxReg, 0); // set ICC.ZF=1 to indicate success
- jmpb (DONE_LABEL);
+ jmpb_if_possible (DONE_LABEL);
}
bind (Stacked);
movptr(tmpReg, Address (boxReg, 0)); // re-fetch
if (os::is_MP()) { lock(); }
@@ -6075,10 +6086,170 @@
call(rax);
// Caller pops the arguments (oop, message) and restores rax, r10
BLOCK_COMMENT("} verify_oop");
}
+void MacroAssembler::in_heap_check(Register raddr, Label& done) {
+ ShenandoahHeap *h = (ShenandoahHeap *)Universe::heap();
+
+ HeapWord* first_region_bottom = h->first_region_bottom();
+ HeapWord* last_region_end = first_region_bottom + (ShenandoahHeapRegion::RegionSizeBytes / HeapWordSize) * h->max_regions();
+
+ cmpptr(raddr, (intptr_t) first_region_bottom);
+ jcc(Assembler::less, done);
+ cmpptr(raddr, (intptr_t) first_region_bottom);
+ jcc(Assembler::greaterEqual, done);
+
+}
+
+void MacroAssembler::shenandoah_cset_check(Register raddr, Register tmp1, Register tmp2, Label& done) {
+ // Test that oop is not in to-space.
+ movptr(tmp1, raddr);
+ shrptr(tmp1, ShenandoahHeapRegion::RegionSizeShift);
+ movptr(tmp2, (intptr_t) ShenandoahHeap::in_cset_fast_test_addr());
+ movbool(tmp2, Address(tmp2, tmp1, Address::times_1));
+ testbool(tmp2);
+ jcc(Assembler::zero, done);
+
+ // Check for cancelled GC.
+ movptr(tmp2, (intptr_t) ShenandoahHeap::cancelled_concgc_addr());
+ movbool(tmp2, Address(tmp2, 0));
+ testbool(tmp2);
+ jcc(Assembler::notZero, done);
+
+}
+
+void MacroAssembler::_shenandoah_store_addr_check(Address addr, const char* msg, const char* file, int line) {
+ _shenandoah_store_addr_check(addr.base(), msg, file, line);
+}
+
+void MacroAssembler::_shenandoah_store_addr_check(Register dst, const char* msg, const char* file, int line) {
+ if (! UseShenandoahGC && ! ShenandoahStoreCheck) return;
+ if (dst == rsp) return; // Stack-based target
+
+ Register raddr = r9;
+ Register tmp1 = r10;
+ Register tmp2 = r11;
+
+ Label done;
+
+ pushf();
+ push(raddr);
+ push(tmp1);
+ push(tmp2);
+
+ movptr(raddr, dst);
+
+ // Check null.
+ testptr(raddr, raddr);
+ jcc(Assembler::zero, done);
+
+ in_heap_check(raddr, done);
+ shenandoah_cset_check(raddr, tmp1, tmp2, done);
+
+ // Fail.
+ pop(tmp2);
+ pop(tmp1);
+ pop(raddr);
+ popf();
+ const char* b = NULL;
+ {
+ ResourceMark rm;
+ stringStream ss;
+ ss.print("shenandoah_store_check: %s in file: %s line: %i", msg, file, line);
+ b = code_string(ss.as_string());
+ }
+ stop(b);
+
+ bind(done);
+
+ pop(tmp2);
+ pop(tmp1);
+ pop(raddr);
+ popf();
+}
+
+void MacroAssembler::_shenandoah_store_check(Register dst, Register value, const char* msg, const char* file, int line) {
+ if (! UseShenandoahGC && ! ShenandoahStoreCheck) return;
+ if (dst == rsp) return; // Stack-based target
+
+ Register raddr = r8;
+ Register rval = r9;
+ Register tmp1 = r10;
+ Register tmp2 = r11;
+
+ // Push tmp regs and flags.
+ pushf();
+ push(raddr);
+ push(rval);
+ push(tmp1);
+ push(tmp2);
+
+ movptr(raddr, dst);
+ movptr(rval, value);
+
+ Label done;
+
+ // If not in-heap target, skip check.
+ in_heap_check(raddr, done);
+
+ // Test that target oop is not in to-space.
+ shenandoah_cset_check(raddr, tmp1, tmp2, done);
+
+ // Do value-check only when concurrent mark is in progress.
+ movptr(tmp1, (intptr_t) ShenandoahHeap::concurrent_mark_in_progress_addr());
+ movbool(tmp1, Address(tmp1, 0));
+ testbool(tmp1);
+ jcc(Assembler::zero, done);
+
+ // Null-check value.
+ testptr(rval, rval);
+ jcc(Assembler::zero, done);
+
+ // Test that value oop is not in to-space.
+ shenandoah_cset_check(rval, tmp1, tmp2, done);
+
+ // Failure.
+ // Pop tmp regs and flags.
+ pop(tmp2);
+ pop(tmp1);
+ pop(rval);
+ pop(raddr);
+ popf();
+ const char* b = NULL;
+ {
+ ResourceMark rm;
+ stringStream ss;
+ ss.print("shenandoah_store_check: %s in file: %s line: %i", msg, file, line);
+ b = code_string(ss.as_string());
+ }
+ stop(b);
+
+ bind(done);
+
+ // Pop tmp regs and flags.
+ pop(tmp2);
+ pop(tmp1);
+ pop(rval);
+ pop(raddr);
+ popf();
+}
+
+void MacroAssembler::_shenandoah_store_check(Address addr, Register value, const char* msg, const char* file, int line) {
+ _shenandoah_store_check(addr.base(), value, msg, file, line);
+}
+
+void MacroAssembler::_shenandoah_lock_check(Register dst, const char* msg, const char* file, int line) {
+#ifdef ASSERT
+ if (! UseShenandoahGC && ! ShenandoahStoreCheck) return;
+
+ push(r8);
+ movptr(r8, Address(dst, BasicObjectLock::obj_offset_in_bytes()));
+ _shenandoah_store_addr_check(r8, msg, file, line);
+ pop(r8);
+#endif
+}
RegisterOrConstant MacroAssembler::delayed_value_impl(intptr_t* delayed_value_addr,
Register tmp,
int offset) {
intptr_t value = *delayed_value_addr;
< prev index next >