< prev index next >
src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp
Print this page
rev 54386 : 8221766: Load-reference barriers for Shenandoah
*** 38,48 ****
#include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
#endif
#define __ masm->
! address ShenandoahBarrierSetAssembler::_shenandoah_wb = NULL;
void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
Register addr, Register count, RegSet saved_regs) {
if (is_oop) {
bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
--- 38,48 ----
#include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
#endif
#define __ masm->
! address ShenandoahBarrierSetAssembler::_shenandoah_lrb = NULL;
void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
Register addr, Register count, RegSet saved_regs) {
if (is_oop) {
bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
*** 196,267 ****
__ pop(saved, sp);
__ bind(done);
}
! void ShenandoahBarrierSetAssembler::read_barrier(MacroAssembler* masm, Register dst) {
! if (ShenandoahReadBarrier) {
! read_barrier_impl(masm, dst);
! }
! }
!
! void ShenandoahBarrierSetAssembler::read_barrier_impl(MacroAssembler* masm, Register dst) {
! assert(UseShenandoahGC && (ShenandoahReadBarrier || ShenandoahStoreValReadBarrier || ShenandoahCASBarrier), "should be enabled");
Label is_null;
__ cbz(dst, is_null);
! read_barrier_not_null_impl(masm, dst);
__ bind(is_null);
}
! void ShenandoahBarrierSetAssembler::read_barrier_not_null(MacroAssembler* masm, Register dst) {
! if (ShenandoahReadBarrier) {
! read_barrier_not_null_impl(masm, dst);
! }
! }
!
!
! void ShenandoahBarrierSetAssembler::read_barrier_not_null_impl(MacroAssembler* masm, Register dst) {
! assert(UseShenandoahGC && (ShenandoahReadBarrier || ShenandoahStoreValReadBarrier || ShenandoahCASBarrier), "should be enabled");
__ ldr(dst, Address(dst, ShenandoahBrooksPointer::byte_offset()));
}
! void ShenandoahBarrierSetAssembler::write_barrier(MacroAssembler* masm, Register dst) {
! if (ShenandoahWriteBarrier) {
! write_barrier_impl(masm, dst);
! }
! }
!
! void ShenandoahBarrierSetAssembler::write_barrier_impl(MacroAssembler* masm, Register dst) {
! assert(UseShenandoahGC && (ShenandoahWriteBarrier || ShenandoahStoreValEnqueueBarrier), "Should be enabled");
! assert(dst != rscratch1, "need rscratch1");
assert(dst != rscratch2, "need rscratch2");
Label done;
Address gc_state(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
! __ ldrb(rscratch1, gc_state);
// Check for heap stability
__ mov(rscratch2, ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::EVACUATION | ShenandoahHeap::TRAVERSAL);
! __ tst(rscratch1, rscratch2);
__ br(Assembler::EQ, done);
! // Heap is unstable, need to perform the read-barrier even if WB is inactive
! __ ldr(dst, Address(dst, ShenandoahBrooksPointer::byte_offset()));
! // Check for evacuation-in-progress and jump to WB slow-path if needed
__ mov(rscratch2, ShenandoahHeap::EVACUATION | ShenandoahHeap::TRAVERSAL);
! __ tst(rscratch1, rscratch2);
__ br(Assembler::EQ, done);
RegSet to_save = RegSet::of(r0);
if (dst != r0) {
__ push(to_save, sp);
__ mov(r0, dst);
}
! __ far_call(RuntimeAddress(CAST_FROM_FN_PTR(address, ShenandoahBarrierSetAssembler::shenandoah_wb())));
if (dst != r0) {
__ mov(dst, r0);
__ pop(to_save, sp);
}
--- 196,251 ----
__ pop(saved, sp);
__ bind(done);
}
! void ShenandoahBarrierSetAssembler::resolve_forward_pointer(MacroAssembler* masm, Register dst) {
! assert(ShenandoahLoadRefBarrier || ShenandoahCASBarrier, "Should be enabled");
Label is_null;
__ cbz(dst, is_null);
! resolve_forward_pointer_not_null(masm, dst);
__ bind(is_null);
}
! void ShenandoahBarrierSetAssembler::resolve_forward_pointer_not_null(MacroAssembler* masm, Register dst) {
! assert(ShenandoahLoadRefBarrier || ShenandoahCASBarrier, "Should be enabled");
__ ldr(dst, Address(dst, ShenandoahBrooksPointer::byte_offset()));
}
! void ShenandoahBarrierSetAssembler::load_reference_barrier_not_null(MacroAssembler* masm, Register dst, Register tmp) {
! assert(ShenandoahLoadRefBarrier, "Should be enabled");
assert(dst != rscratch2, "need rscratch2");
+ if (tmp == noreg) {
+ assert(dst != rscratch1, "need rscratch1");
+ tmp = rscratch1;
+ }
Label done;
Address gc_state(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
! __ ldrb(tmp, gc_state);
// Check for heap stability
__ mov(rscratch2, ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::EVACUATION | ShenandoahHeap::TRAVERSAL);
! __ tst(tmp, rscratch2);
__ br(Assembler::EQ, done);
! // Heap is unstable, need to perform the resolve even if LRB is inactive
! resolve_forward_pointer_not_null(masm, dst);
! // Check for evacuation-in-progress and jump to LRB slow-path if needed
__ mov(rscratch2, ShenandoahHeap::EVACUATION | ShenandoahHeap::TRAVERSAL);
! __ tst(tmp, rscratch2);
__ br(Assembler::EQ, done);
RegSet to_save = RegSet::of(r0);
if (dst != r0) {
__ push(to_save, sp);
__ mov(r0, dst);
}
! __ far_call(RuntimeAddress(CAST_FROM_FN_PTR(address, ShenandoahBarrierSetAssembler::shenandoah_lrb())));
if (dst != r0) {
__ mov(dst, r0);
__ pop(to_save, sp);
}
*** 269,282 ****
__ bind(done);
}
void ShenandoahBarrierSetAssembler::storeval_barrier(MacroAssembler* masm, Register dst, Register tmp) {
if (ShenandoahStoreValEnqueueBarrier) {
- Label is_null;
- __ cbz(dst, is_null);
- write_barrier_impl(masm, dst);
- __ bind(is_null);
// Save possibly live regs.
RegSet live_regs = RegSet::range(r0, r4) - dst;
__ push(live_regs, sp);
__ strd(v0, __ pre(sp, 2 * -wordSize));
--- 253,262 ----
*** 284,331 ****
// Restore possibly live regs.
__ ldrd(v0, __ post(sp, 2 * wordSize));
__ pop(live_regs, sp);
}
! if (ShenandoahStoreValReadBarrier) {
! read_barrier_impl(masm, dst);
}
}
void ShenandoahBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
Register dst, Address src, Register tmp1, Register tmp_thread) {
bool on_oop = type == T_OBJECT || type == T_ARRAY;
- bool in_heap = (decorators & IN_HEAP) != 0;
bool on_weak = (decorators & ON_WEAK_OOP_REF) != 0;
bool on_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0;
bool on_reference = on_weak || on_phantom;
- if (in_heap) {
- read_barrier_not_null(masm, src.base());
- }
-
BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread);
! if (ShenandoahKeepAliveBarrier && on_oop && on_reference) {
__ enter();
satb_write_barrier_pre(masm /* masm */,
noreg /* obj */,
dst /* pre_val */,
rthread /* thread */,
tmp1 /* tmp */,
true /* tosca_live */,
true /* expand_call */);
__ leave();
}
}
void ShenandoahBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
Address dst, Register val, Register tmp1, Register tmp2) {
bool on_oop = type == T_OBJECT || type == T_ARRAY;
- bool in_heap = (decorators & IN_HEAP) != 0;
- if (in_heap) {
- write_barrier(masm, dst.base());
- }
if (!on_oop) {
BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2);
return;
}
--- 264,312 ----
// Restore possibly live regs.
__ ldrd(v0, __ post(sp, 2 * wordSize));
__ pop(live_regs, sp);
}
! }
!
! void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler* masm, Register dst, Register tmp) {
! if (ShenandoahLoadRefBarrier) {
! Label is_null;
! __ cbz(dst, is_null);
! load_reference_barrier_not_null(masm, dst, tmp);
! __ bind(is_null);
}
}
void ShenandoahBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
Register dst, Address src, Register tmp1, Register tmp_thread) {
bool on_oop = type == T_OBJECT || type == T_ARRAY;
bool on_weak = (decorators & ON_WEAK_OOP_REF) != 0;
bool on_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0;
bool on_reference = on_weak || on_phantom;
BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread);
! if (on_oop) {
! load_reference_barrier(masm, dst, tmp1);
!
! if (ShenandoahKeepAliveBarrier && on_reference) {
__ enter();
satb_write_barrier_pre(masm /* masm */,
noreg /* obj */,
dst /* pre_val */,
rthread /* thread */,
tmp1 /* tmp */,
true /* tosca_live */,
true /* expand_call */);
__ leave();
}
+ }
}
void ShenandoahBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
Address dst, Register val, Register tmp1, Register tmp2) {
bool on_oop = type == T_OBJECT || type == T_ARRAY;
if (!on_oop) {
BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2);
return;
}
*** 359,383 ****
BarrierSetAssembler::store_at(masm, decorators, type, Address(r3, 0), val, noreg, noreg);
}
}
- void ShenandoahBarrierSetAssembler::obj_equals(MacroAssembler* masm, Register op1, Register op2) {
- __ cmp(op1, op2);
- if (ShenandoahAcmpBarrier) {
- Label done;
- __ br(Assembler::EQ, done);
- // The object may have been evacuated, but we won't see it without a
- // membar here.
- __ membar(Assembler::LoadStore| Assembler::LoadLoad);
- read_barrier(masm, op1);
- read_barrier(masm, op2);
- __ cmp(op1, op2);
- __ bind(done);
- }
- }
-
void ShenandoahBarrierSetAssembler::tlab_allocate(MacroAssembler* masm, Register obj,
Register var_size_in_bytes,
int con_size_in_bytes,
Register t1,
Register t2,
--- 340,349 ----
*** 408,438 ****
if (var_size_in_bytes == end) {
__ sub(var_size_in_bytes, var_size_in_bytes, obj);
}
}
- void ShenandoahBarrierSetAssembler::resolve(MacroAssembler* masm, DecoratorSet decorators, Register obj) {
- bool oop_not_null = (decorators & IS_NOT_NULL) != 0;
- bool is_write = (decorators & ACCESS_WRITE) != 0;
- if (is_write) {
- if (oop_not_null) {
- write_barrier(masm, obj);
- } else {
- Label done;
- __ cbz(obj, done);
- write_barrier(masm, obj);
- __ bind(done);
- }
- } else {
- if (oop_not_null) {
- read_barrier_not_null(masm, obj);
- } else {
- read_barrier(masm, obj);
- }
- }
- }
-
void ShenandoahBarrierSetAssembler::cmpxchg_oop(MacroAssembler* masm, Register addr, Register expected, Register new_val,
bool acquire, bool release, bool weak, bool is_cae,
Register result) {
Register tmp1 = rscratch1;
Register tmp2 = rscratch2;
--- 374,383 ----
*** 467,478 ****
__ mov(expected, tmp1);
if (is_narrow) {
__ decode_heap_oop(tmp1, tmp1);
__ decode_heap_oop(tmp2, tmp2);
}
! read_barrier_impl(masm, tmp1);
! read_barrier_impl(masm, tmp2);
__ cmp(tmp1, tmp2);
// Retry with expected now being the value we just loaded from addr.
__ br(Assembler::EQ, retry);
if (is_cae && is_narrow) {
// For cmp-and-exchange and narrow oops, we need to restore
--- 412,423 ----
__ mov(expected, tmp1);
if (is_narrow) {
__ decode_heap_oop(tmp1, tmp1);
__ decode_heap_oop(tmp2, tmp2);
}
! resolve_forward_pointer(masm, tmp1);
! resolve_forward_pointer(masm, tmp2);
__ cmp(tmp1, tmp2);
// Retry with expected now being the value we just loaded from addr.
__ br(Assembler::EQ, retry);
if (is_cae && is_narrow) {
// For cmp-and-exchange and narrow oops, we need to restore
*** 513,523 ****
ce->store_parameter(stub->pre_val()->as_register(), 0);
__ far_call(RuntimeAddress(bs->pre_barrier_c1_runtime_code_blob()->code_begin()));
__ b(*stub->continuation());
}
! void ShenandoahBarrierSetAssembler::gen_write_barrier_stub(LIR_Assembler* ce, ShenandoahWriteBarrierStub* stub) {
Register obj = stub->obj()->as_register();
Register res = stub->result()->as_register();
Label done;
--- 458,468 ----
ce->store_parameter(stub->pre_val()->as_register(), 0);
__ far_call(RuntimeAddress(bs->pre_barrier_c1_runtime_code_blob()->code_begin()));
__ b(*stub->continuation());
}
! void ShenandoahBarrierSetAssembler::gen_load_reference_barrier_stub(LIR_Assembler* ce, ShenandoahLoadReferenceBarrierStub* stub) {
Register obj = stub->obj()->as_register();
Register res = stub->result()->as_register();
Label done;
*** 530,540 ****
// Check for null.
if (stub->needs_null_check()) {
__ cbz(res, done);
}
! write_barrier(ce->masm(), res);
__ bind(done);
__ b(*stub->continuation());
}
--- 475,485 ----
// Check for null.
if (stub->needs_null_check()) {
__ cbz(res, done);
}
! load_reference_barrier_not_null(ce->masm(), res, rscratch1);
__ bind(done);
__ b(*stub->continuation());
}
*** 590,619 ****
#undef __
#endif // COMPILER1
! address ShenandoahBarrierSetAssembler::shenandoah_wb() {
! assert(_shenandoah_wb != NULL, "need write barrier stub");
! return _shenandoah_wb;
}
#define __ cgen->assembler()->
! // Shenandoah write barrier.
//
// Input:
// r0: OOP to evacuate. Not null.
//
// Output:
// r0: Pointer to evacuated OOP.
//
// Trash rscratch1, rscratch2. Preserve everything else.
! address ShenandoahBarrierSetAssembler::generate_shenandoah_wb(StubCodeGenerator* cgen) {
__ align(6);
! StubCodeMark mark(cgen, "StubRoutines", "shenandoah_wb");
address start = __ pc();
Label work;
__ mov(rscratch2, ShenandoahHeap::in_cset_fast_test_addr());
__ lsr(rscratch1, r0, ShenandoahHeapRegion::region_size_bytes_shift_jint());
--- 535,564 ----
#undef __
#endif // COMPILER1
! address ShenandoahBarrierSetAssembler::shenandoah_lrb() {
! assert(_shenandoah_lrb != NULL, "need load reference barrier stub");
! return _shenandoah_lrb;
}
#define __ cgen->assembler()->
! // Shenandoah load reference barrier.
//
// Input:
// r0: OOP to evacuate. Not null.
//
// Output:
// r0: Pointer to evacuated OOP.
//
// Trash rscratch1, rscratch2. Preserve everything else.
! address ShenandoahBarrierSetAssembler::generate_shenandoah_lrb(StubCodeGenerator* cgen) {
__ align(6);
! StubCodeMark mark(cgen, "StubRoutines", "shenandoah_lrb");
address start = __ pc();
Label work;
__ mov(rscratch2, ShenandoahHeap::in_cset_fast_test_addr());
__ lsr(rscratch1, r0, ShenandoahHeapRegion::region_size_bytes_shift_jint());
*** 626,636 ****
__ enter(); // required for proper stackwalking of RuntimeStub frame
__ push_call_clobbered_registers();
! __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_JRT));
__ blrt(lr, 1, 0, MacroAssembler::ret_type_integral);
__ mov(rscratch1, obj);
__ pop_call_clobbered_registers();
__ mov(obj, rscratch1);
--- 571,581 ----
__ enter(); // required for proper stackwalking of RuntimeStub frame
__ push_call_clobbered_registers();
! __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_JRT));
__ blrt(lr, 1, 0, MacroAssembler::ret_type_integral);
__ mov(rscratch1, obj);
__ pop_call_clobbered_registers();
__ mov(obj, rscratch1);
*** 641,654 ****
}
#undef __
void ShenandoahBarrierSetAssembler::barrier_stubs_init() {
! if (ShenandoahWriteBarrier || ShenandoahStoreValEnqueueBarrier) {
int stub_code_size = 2048;
ResourceMark rm;
BufferBlob* bb = BufferBlob::create("shenandoah_barrier_stubs", stub_code_size);
CodeBuffer buf(bb);
StubCodeGenerator cgen(&buf);
! _shenandoah_wb = generate_shenandoah_wb(&cgen);
}
}
--- 586,599 ----
}
#undef __
void ShenandoahBarrierSetAssembler::barrier_stubs_init() {
! if (ShenandoahLoadRefBarrier) {
int stub_code_size = 2048;
ResourceMark rm;
BufferBlob* bb = BufferBlob::create("shenandoah_barrier_stubs", stub_code_size);
CodeBuffer buf(bb);
StubCodeGenerator cgen(&buf);
! _shenandoah_lrb = generate_shenandoah_lrb(&cgen);
}
}
< prev index next >