< prev index next >
src/cpu/x86/vm/templateTable_x86.cpp
Print this page
@@ -155,10 +155,11 @@
bool precise) {
assert(val == noreg || val == rax, "parameter is just for looks");
switch (barrier) {
#if INCLUDE_ALL_GCS
case BarrierSet::G1SATBCTLogging:
+ case BarrierSet::ShenandoahBarrierSet:
{
// flatten object address if needed
// We do it regardless of precise because we need the registers
if (obj.index() == noreg && obj.disp() == 0) {
if (obj.base() != rdx) {
@@ -187,10 +188,13 @@
Register new_val = val;
if (UseCompressedOops) {
new_val = rbx;
__ movptr(new_val, val);
}
+ // For Shenandoah, make sure we only store refs into to-space.
+ oopDesc::bs()->interpreter_read_barrier(_masm, val);
+
__ store_heap_oop(Address(rdx, 0), val);
__ g1_write_barrier_post(rdx /* store_adr */,
new_val /* new_val */,
rthread /* thread */,
rtmp /* tmp */,
@@ -682,10 +686,15 @@
}
void TemplateTable::index_check_without_pop(Register array, Register index) {
// destroys rbx
// check array
+
+ if (ShenandoahVerifyReadsToFromSpace) {
+ oopDesc::bs()->interpreter_read_barrier(_masm, array);
+ }
+
__ null_check(array, arrayOopDesc::length_offset_in_bytes());
// sign extend index for use by indexed load
__ movl2ptr(index, index);
// check index
__ cmpl(index, Address(array, arrayOopDesc::length_offset_in_bytes()));
@@ -702,10 +711,11 @@
void TemplateTable::iaload() {
transition(itos, itos);
// rax: index
// rdx: array
index_check(rdx, rax); // kills rbx
+ oopDesc::bs()->interpreter_read_barrier_not_null(_masm, rdx);
__ movl(rax, Address(rdx, rax,
Address::times_4,
arrayOopDesc::base_offset_in_bytes(T_INT)));
}
@@ -714,10 +724,11 @@
// rax: index
// rdx: array
index_check(rdx, rax); // kills rbx
NOT_LP64(__ mov(rbx, rax));
// rbx,: index
+ oopDesc::bs()->interpreter_read_barrier_not_null(_masm, rdx);
__ movptr(rax, Address(rdx, rbx, Address::times_8, arrayOopDesc::base_offset_in_bytes(T_LONG) + 0 * wordSize));
NOT_LP64(__ movl(rdx, Address(rdx, rbx, Address::times_8, arrayOopDesc::base_offset_in_bytes(T_LONG) + 1 * wordSize)));
}
@@ -725,48 +736,53 @@
void TemplateTable::faload() {
transition(itos, ftos);
// rax: index
// rdx: array
index_check(rdx, rax); // kills rbx
+ oopDesc::bs()->interpreter_read_barrier_not_null(_masm, rdx);
__ load_float(Address(rdx, rax,
Address::times_4,
arrayOopDesc::base_offset_in_bytes(T_FLOAT)));
}
void TemplateTable::daload() {
transition(itos, dtos);
// rax: index
// rdx: array
index_check(rdx, rax); // kills rbx
+ oopDesc::bs()->interpreter_read_barrier_not_null(_masm, rdx);
__ load_double(Address(rdx, rax,
Address::times_8,
arrayOopDesc::base_offset_in_bytes(T_DOUBLE)));
}
void TemplateTable::aaload() {
transition(itos, atos);
// rax: index
// rdx: array
index_check(rdx, rax); // kills rbx
+ oopDesc::bs()->interpreter_read_barrier_not_null(_masm, rdx);
__ load_heap_oop(rax, Address(rdx, rax,
UseCompressedOops ? Address::times_4 : Address::times_ptr,
arrayOopDesc::base_offset_in_bytes(T_OBJECT)));
}
void TemplateTable::baload() {
transition(itos, itos);
// rax: index
// rdx: array
index_check(rdx, rax); // kills rbx
+ oopDesc::bs()->interpreter_read_barrier_not_null(_masm, rdx);
__ load_signed_byte(rax, Address(rdx, rax, Address::times_1, arrayOopDesc::base_offset_in_bytes(T_BYTE)));
}
void TemplateTable::caload() {
transition(itos, itos);
// rax: index
// rdx: array
index_check(rdx, rax); // kills rbx
+ oopDesc::bs()->interpreter_read_barrier_not_null(_masm, rdx);
__ load_unsigned_short(rax, Address(rdx, rax, Address::times_2, arrayOopDesc::base_offset_in_bytes(T_CHAR)));
}
// iload followed by caload frequent pair
void TemplateTable::fast_icaload() {
@@ -776,10 +792,11 @@
__ movl(rax, iaddress(rbx));
// rax: index
// rdx: array
index_check(rdx, rax); // kills rbx
+ oopDesc::bs()->interpreter_read_barrier_not_null(_masm, rdx);
__ load_unsigned_short(rax,
Address(rdx, rax,
Address::times_2,
arrayOopDesc::base_offset_in_bytes(T_CHAR)));
}
@@ -788,10 +805,11 @@
void TemplateTable::saload() {
transition(itos, itos);
// rax: index
// rdx: array
index_check(rdx, rax); // kills rbx
+ oopDesc::bs()->interpreter_read_barrier_not_null(_masm, rdx);
__ load_signed_short(rax, Address(rdx, rax, Address::times_2, arrayOopDesc::base_offset_in_bytes(T_SHORT)));
}
void TemplateTable::iload(int n) {
transition(vtos, itos);
@@ -982,10 +1000,11 @@
__ pop_i(rbx);
// rax: value
// rbx: index
// rdx: array
index_check(rdx, rbx); // prefer index in rbx
+ oopDesc::bs()->interpreter_write_barrier(_masm, rdx);
__ movl(Address(rdx, rbx,
Address::times_4,
arrayOopDesc::base_offset_in_bytes(T_INT)),
rax);
}
@@ -996,10 +1015,11 @@
// rax,: low(value)
// rcx: array
// rdx: high(value)
index_check(rcx, rbx); // prefer index in rbx,
// rbx,: index
+ oopDesc::bs()->interpreter_write_barrier(_masm, rcx);
__ movptr(Address(rcx, rbx, Address::times_8, arrayOopDesc::base_offset_in_bytes(T_LONG) + 0 * wordSize), rax);
NOT_LP64(__ movl(Address(rcx, rbx, Address::times_8, arrayOopDesc::base_offset_in_bytes(T_LONG) + 1 * wordSize), rdx));
}
@@ -1008,20 +1028,22 @@
__ pop_i(rbx);
// value is in UseSSE >= 1 ? xmm0 : ST(0)
// rbx: index
// rdx: array
index_check(rdx, rbx); // prefer index in rbx
+ oopDesc::bs()->interpreter_write_barrier(_masm, rdx);
__ store_float(Address(rdx, rbx, Address::times_4, arrayOopDesc::base_offset_in_bytes(T_FLOAT)));
}
void TemplateTable::dastore() {
transition(dtos, vtos);
__ pop_i(rbx);
// value is in UseSSE >= 2 ? xmm0 : ST(0)
// rbx: index
// rdx: array
index_check(rdx, rbx); // prefer index in rbx
+ oopDesc::bs()->interpreter_write_barrier(_masm, rdx);
__ store_double(Address(rdx, rbx, Address::times_8, arrayOopDesc::base_offset_in_bytes(T_DOUBLE)));
}
void TemplateTable::aastore() {
Label is_null, ok_is_subtype, done;
@@ -1034,10 +1056,11 @@
Address element_address(rdx, rcx,
UseCompressedOops? Address::times_4 : Address::times_ptr,
arrayOopDesc::base_offset_in_bytes(T_OBJECT));
index_check_without_pop(rdx, rcx); // kills rbx
+ oopDesc::bs()->interpreter_write_barrier(_masm, rdx);
__ testptr(rax, rax);
__ jcc(Assembler::zero, is_null);
// Move subklass into rbx
__ load_klass(rbx, rax);
@@ -1082,10 +1105,11 @@
__ pop_i(rbx);
// rax: value
// rbx: index
// rdx: array
index_check(rdx, rbx); // prefer index in rbx
+ oopDesc::bs()->interpreter_write_barrier(_masm, rdx);
__ movb(Address(rdx, rbx,
Address::times_1,
arrayOopDesc::base_offset_in_bytes(T_BYTE)),
rax);
}
@@ -1095,10 +1119,11 @@
__ pop_i(rbx);
// rax: value
// rbx: index
// rdx: array
index_check(rdx, rbx); // prefer index in rbx
+ oopDesc::bs()->interpreter_write_barrier(_masm, rdx);
__ movw(Address(rdx, rbx,
Address::times_2,
arrayOopDesc::base_offset_in_bytes(T_CHAR)),
rax);
}
@@ -2309,10 +2334,21 @@
void TemplateTable::if_acmp(Condition cc) {
transition(atos, vtos);
// assume branch is more often taken than not (loops use backward branches)
Label not_taken;
__ pop_ptr(rdx);
+ if (UseShenandoahGC) {
+ // For Shenandoah, if the objects are not equal, we try again after
+ // resolving both objects through a read barrier, to make sure we're
+ // not comparing from-space and to-space copies of the same object.
+ Label eq;
+ __ cmpptr(rdx, rax);
+ __ jcc(Assembler::equal, eq);
+ oopDesc::bs()->interpreter_read_barrier(_masm, rax);
+ oopDesc::bs()->interpreter_read_barrier(_masm, rdx);
+ __ bind(eq);
+ }
__ cmpptr(rdx, rax);
__ jcc(j_not(cc), not_taken);
branch(false, false);
__ bind(not_taken);
__ profile_not_taken_branch(rax);
@@ -2749,10 +2785,11 @@
resolve_cache_and_index(byte_no, cache, index, sizeof(u2));
jvmti_post_field_access(cache, index, is_static, false);
load_field_cp_cache_entry(obj, cache, index, off, flags, is_static);
if (!is_static) pop_and_check_object(obj);
+ oopDesc::bs()->interpreter_read_barrier_not_null(_masm, obj);
const Address field(obj, off, Address::times_1, 0*wordSize);
NOT_LP64(const Address hi(obj, off, Address::times_1, 1*wordSize));
Label Done, notByte, notInt, notShort, notChar, notLong, notFloat, notObj, notDouble;
@@ -3018,10 +3055,11 @@
// btos
{
__ pop(btos);
if (!is_static) pop_and_check_object(obj);
+ oopDesc::bs()->interpreter_write_barrier(_masm, obj);
__ movb(field, rax);
if (!is_static && rc == may_rewrite) {
patch_bytecode(Bytecodes::_fast_bputfield, bc, rbx, true, byte_no);
}
__ jmp(Done);
@@ -3033,10 +3071,11 @@
// atos
{
__ pop(atos);
if (!is_static) pop_and_check_object(obj);
+ oopDesc::bs()->interpreter_write_barrier(_masm, obj);
// Store into the field
do_oop_store(_masm, field, rax, _bs->kind(), false);
if (!is_static && rc == may_rewrite) {
patch_bytecode(Bytecodes::_fast_aputfield, bc, rbx, true, byte_no);
}
@@ -3049,10 +3088,11 @@
// itos
{
__ pop(itos);
if (!is_static) pop_and_check_object(obj);
+ oopDesc::bs()->interpreter_write_barrier(_masm, obj);
__ movl(field, rax);
if (!is_static && rc == may_rewrite) {
patch_bytecode(Bytecodes::_fast_iputfield, bc, rbx, true, byte_no);
}
__ jmp(Done);
@@ -3064,10 +3104,11 @@
// ctos
{
__ pop(ctos);
if (!is_static) pop_and_check_object(obj);
+ oopDesc::bs()->interpreter_write_barrier(_masm, obj);
__ movw(field, rax);
if (!is_static && rc == may_rewrite) {
patch_bytecode(Bytecodes::_fast_cputfield, bc, rbx, true, byte_no);
}
__ jmp(Done);
@@ -3079,10 +3120,11 @@
// stos
{
__ pop(stos);
if (!is_static) pop_and_check_object(obj);
+ oopDesc::bs()->interpreter_write_barrier(_masm, obj);
__ movw(field, rax);
if (!is_static && rc == may_rewrite) {
patch_bytecode(Bytecodes::_fast_sputfield, bc, rbx, true, byte_no);
}
__ jmp(Done);
@@ -3095,10 +3137,11 @@
// ltos
#ifdef _LP64
{
__ pop(ltos);
if (!is_static) pop_and_check_object(obj);
+ oopDesc::bs()->interpreter_write_barrier(_masm, obj);
__ movq(field, rax);
if (!is_static && rc == may_rewrite) {
patch_bytecode(Bytecodes::_fast_lputfield, bc, rbx, true, byte_no);
}
__ jmp(Done);
@@ -3141,10 +3184,11 @@
// ftos
{
__ pop(ftos);
if (!is_static) pop_and_check_object(obj);
+ oopDesc::bs()->interpreter_write_barrier(_masm, obj);
__ store_float(field);
if (!is_static && rc == may_rewrite) {
patch_bytecode(Bytecodes::_fast_fputfield, bc, rbx, true, byte_no);
}
__ jmp(Done);
@@ -3158,10 +3202,11 @@
// dtos
{
__ pop(dtos);
if (!is_static) pop_and_check_object(obj);
+ oopDesc::bs()->interpreter_write_barrier(_masm, obj);
__ store_double(field);
if (!is_static && rc == may_rewrite) {
patch_bytecode(Bytecodes::_fast_dputfield, bc, rbx, true, byte_no);
}
}
@@ -3284,34 +3329,41 @@
const Address field(rcx, rbx, Address::times_1);
// access field
switch (bytecode()) {
case Bytecodes::_fast_aputfield:
+ oopDesc::bs()->interpreter_write_barrier(_masm, rcx);
do_oop_store(_masm, field, rax, _bs->kind(), false);
break;
case Bytecodes::_fast_lputfield:
+ oopDesc::bs()->interpreter_write_barrier(_masm, rcx);
#ifdef _LP64
__ movq(field, rax);
#else
__ stop("should not be rewritten");
#endif
break;
case Bytecodes::_fast_iputfield:
+ oopDesc::bs()->interpreter_write_barrier(_masm, rcx);
__ movl(field, rax);
break;
case Bytecodes::_fast_bputfield:
+ oopDesc::bs()->interpreter_write_barrier(_masm, rcx);
__ movb(field, rax);
break;
case Bytecodes::_fast_sputfield:
// fall through
case Bytecodes::_fast_cputfield:
+ oopDesc::bs()->interpreter_write_barrier(_masm, rcx);
__ movw(field, rax);
break;
case Bytecodes::_fast_fputfield:
+ oopDesc::bs()->interpreter_write_barrier(_masm, rcx);
__ store_float(field);
break;
case Bytecodes::_fast_dputfield:
+ oopDesc::bs()->interpreter_write_barrier(_masm, rcx);
__ store_double(field);
break;
default:
ShouldNotReachHere();
}
@@ -3365,10 +3417,11 @@
ConstantPoolCacheEntry::f2_offset())));
// rax: object
__ verify_oop(rax);
__ null_check(rax);
+ oopDesc::bs()->interpreter_read_barrier_not_null(_masm, rax);
Address field(rax, rbx, Address::times_1);
// access field
switch (bytecode()) {
case Bytecodes::_fast_agetfield:
@@ -3426,10 +3479,11 @@
ConstantPoolCacheEntry::f2_offset())));
// make sure exception is reported in correct bcp range (getfield is
// next instruction)
__ increment(rbcp);
__ null_check(rax);
+ oopDesc::bs()->interpreter_read_barrier_not_null(_masm, rax);
const Address field = Address(rax, rbx, Address::times_1, 0*wordSize);
switch (state) {
case itos:
__ movl(rax, field);
break;
@@ -3828,15 +3882,21 @@
__ get_thread(thread);
}
#endif // _LP64
if (UseTLAB) {
+ uint oop_extra_words = Universe::heap()->oop_extra_words();
+ if (oop_extra_words > 0) {
+ __ addq(rdx, oop_extra_words * HeapWordSize);
+ }
+
__ movptr(rax, Address(thread, in_bytes(JavaThread::tlab_top_offset())));
__ lea(rbx, Address(rax, rdx, Address::times_1));
__ cmpptr(rbx, Address(thread, in_bytes(JavaThread::tlab_end_offset())));
__ jcc(Assembler::above, allow_shared_alloc ? allocate_shared : slow_case);
__ movptr(Address(thread, in_bytes(JavaThread::tlab_top_offset())), rbx);
+ Universe::heap()->compile_prepare_oop(_masm);
if (ZeroTLAB) {
// the fields have been already cleared
__ jmp(initialize_header);
} else {
// initialize both the header and fields
@@ -3973,10 +4033,13 @@
rarg1, rarg2, rax);
}
void TemplateTable::arraylength() {
transition(atos, itos);
+ if (ShenandoahVerifyReadsToFromSpace) {
+ oopDesc::bs()->interpreter_read_barrier(_masm, rax);
+ }
__ null_check(rax, arrayOopDesc::length_offset_in_bytes());
__ movl(rax, Address(rax, arrayOopDesc::length_offset_in_bytes()));
}
void TemplateTable::checkcast() {
@@ -4073,11 +4136,15 @@
#endif
__ pop_ptr(rdx); // restore receiver
__ verify_oop(rdx);
__ load_klass(rdx, rdx);
+ if (ShenandoahVerifyReadsToFromSpace) {
+ __ jmp(resolved);
+ } else {
__ jmpb(resolved);
+ }
// Get superklass in rax and subklass in rdx
__ bind(quicked);
__ load_klass(rdx, rax);
__ movptr(rax, Address(rcx, rbx,
@@ -4169,10 +4236,15 @@
transition(atos, vtos);
// check for NULL object
__ null_check(rax);
+ // We need to preemptively evacuate the object, because we later compare
+ // it to objects in the BasicObjectLock list, and we might get false negatives
+ // if another thread evacuates the object in the meantime. See acmp.
+ oopDesc::bs()->interpreter_write_barrier(_masm, rax);
+
const Address monitor_block_top(
rbp, frame::interpreter_frame_monitor_block_top_offset * wordSize);
const Address monitor_block_bot(
rbp, frame::interpreter_frame_initial_sp_offset * wordSize);
const int entry_size = frame::interpreter_frame_monitor_size() * wordSize;
@@ -4191,19 +4263,25 @@
Label entry, loop, exit;
__ movptr(rtop, monitor_block_top); // points to current entry,
// starting with top-most entry
__ lea(rbot, monitor_block_bot); // points to word before bottom
// of monitor block
+ if (UseShenandoahGC && ShenandoahVerifyReadsToFromSpace) {
+ __ jmp(entry);
+ } else {
__ jmpb(entry);
+ }
__ bind(loop);
// check if current entry is used
__ cmpptr(Address(rtop, BasicObjectLock::obj_offset_in_bytes()), (int32_t) NULL_WORD);
// if not used then remember entry in rmon
__ cmovptr(Assembler::equal, rmon, rtop); // cmov => cmovptr
// check if current entry is for same object
- __ cmpptr(rax, Address(rtop, BasicObjectLock::obj_offset_in_bytes()));
+ __ movptr(rscratch1, Address(rtop, BasicObjectLock::obj_offset_in_bytes()));
+ oopDesc::bs()->interpreter_read_barrier(_masm, rscratch1);
+ __ cmpptr(rax, rscratch1);
// if same object then stop searching
__ jccb(Assembler::equal, exit);
// otherwise advance to next entry
__ addptr(rtop, entry_size);
__ bind(entry);
@@ -4266,10 +4344,15 @@
transition(atos, vtos);
// check for NULL object
__ null_check(rax);
+ // We need to preemptively evacuate the object, because we later compare
+ // it to objects in the BasicObjectLock list, and we might get false negatives
+ // if another thread evacuates the object in the meantime. See acmp.
+ oopDesc::bs()->interpreter_write_barrier(_masm, rax);
+
const Address monitor_block_top(
rbp, frame::interpreter_frame_monitor_block_top_offset * wordSize);
const Address monitor_block_bot(
rbp, frame::interpreter_frame_initial_sp_offset * wordSize);
const int entry_size = frame::interpreter_frame_monitor_size() * wordSize;
@@ -4284,15 +4367,21 @@
Label entry, loop;
__ movptr(rtop, monitor_block_top); // points to current entry,
// starting with top-most entry
__ lea(rbot, monitor_block_bot); // points to word before bottom
// of monitor block
+ if (UseShenandoahGC && ShenandoahVerifyReadsToFromSpace) {
+ __ jmp(entry);
+ } else {
__ jmpb(entry);
+ }
__ bind(loop);
// check if current entry is for same object
- __ cmpptr(rax, Address(rtop, BasicObjectLock::obj_offset_in_bytes()));
+ __ movptr(rscratch1, Address(rtop, BasicObjectLock::obj_offset_in_bytes()));
+ oopDesc::bs()->interpreter_read_barrier(_masm, rscratch1);
+ __ cmpptr(rax, rscratch1);
// if same object then stop searching
__ jcc(Assembler::equal, found);
// otherwise advance to next entry
__ addptr(rtop, entry_size);
__ bind(entry);
< prev index next >