< prev index next >
src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp
Print this page
rev 54386 : 8221766: Load-reference barriers for Shenandoah
@@ -38,11 +38,11 @@
#include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
#endif
#define __ masm->
-address ShenandoahBarrierSetAssembler::_shenandoah_wb = NULL;
+address ShenandoahBarrierSetAssembler::_shenandoah_lrb = NULL;
void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
Register addr, Register count, RegSet saved_regs) {
if (is_oop) {
bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
@@ -196,72 +196,56 @@
__ pop(saved, sp);
__ bind(done);
}
-void ShenandoahBarrierSetAssembler::read_barrier(MacroAssembler* masm, Register dst) {
- if (ShenandoahReadBarrier) {
- read_barrier_impl(masm, dst);
- }
-}
-
-void ShenandoahBarrierSetAssembler::read_barrier_impl(MacroAssembler* masm, Register dst) {
- assert(UseShenandoahGC && (ShenandoahReadBarrier || ShenandoahStoreValReadBarrier || ShenandoahCASBarrier), "should be enabled");
+void ShenandoahBarrierSetAssembler::resolve_forward_pointer(MacroAssembler* masm, Register dst) {
+ assert(ShenandoahLoadRefBarrier || ShenandoahCASBarrier, "Should be enabled");
Label is_null;
__ cbz(dst, is_null);
- read_barrier_not_null_impl(masm, dst);
+ resolve_forward_pointer_not_null(masm, dst);
__ bind(is_null);
}
-void ShenandoahBarrierSetAssembler::read_barrier_not_null(MacroAssembler* masm, Register dst) {
- if (ShenandoahReadBarrier) {
- read_barrier_not_null_impl(masm, dst);
- }
-}
-
-
-void ShenandoahBarrierSetAssembler::read_barrier_not_null_impl(MacroAssembler* masm, Register dst) {
- assert(UseShenandoahGC && (ShenandoahReadBarrier || ShenandoahStoreValReadBarrier || ShenandoahCASBarrier), "should be enabled");
+void ShenandoahBarrierSetAssembler::resolve_forward_pointer_not_null(MacroAssembler* masm, Register dst) {
+ assert(ShenandoahLoadRefBarrier || ShenandoahCASBarrier, "Should be enabled");
__ ldr(dst, Address(dst, ShenandoahBrooksPointer::byte_offset()));
}
-void ShenandoahBarrierSetAssembler::write_barrier(MacroAssembler* masm, Register dst) {
- if (ShenandoahWriteBarrier) {
- write_barrier_impl(masm, dst);
- }
-}
-
-void ShenandoahBarrierSetAssembler::write_barrier_impl(MacroAssembler* masm, Register dst) {
- assert(UseShenandoahGC && (ShenandoahWriteBarrier || ShenandoahStoreValEnqueueBarrier), "Should be enabled");
- assert(dst != rscratch1, "need rscratch1");
+void ShenandoahBarrierSetAssembler::load_reference_barrier_not_null(MacroAssembler* masm, Register dst, Register tmp) {
+ assert(ShenandoahLoadRefBarrier, "Should be enabled");
assert(dst != rscratch2, "need rscratch2");
+ if (tmp == noreg) {
+ assert(dst != rscratch1, "need rscratch1");
+ tmp = rscratch1;
+ }
Label done;
Address gc_state(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
- __ ldrb(rscratch1, gc_state);
+ __ ldrb(tmp, gc_state);
// Check for heap stability
__ mov(rscratch2, ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::EVACUATION | ShenandoahHeap::TRAVERSAL);
- __ tst(rscratch1, rscratch2);
+ __ tst(tmp, rscratch2);
__ br(Assembler::EQ, done);
- // Heap is unstable, need to perform the read-barrier even if WB is inactive
- __ ldr(dst, Address(dst, ShenandoahBrooksPointer::byte_offset()));
+ // Heap is unstable, need to perform the resolve even if LRB is inactive
+ resolve_forward_pointer_not_null(masm, dst);
- // Check for evacuation-in-progress and jump to WB slow-path if needed
+ // Check for evacuation-in-progress and jump to LRB slow-path if needed
__ mov(rscratch2, ShenandoahHeap::EVACUATION | ShenandoahHeap::TRAVERSAL);
- __ tst(rscratch1, rscratch2);
+ __ tst(tmp, rscratch2);
__ br(Assembler::EQ, done);
RegSet to_save = RegSet::of(r0);
if (dst != r0) {
__ push(to_save, sp);
__ mov(r0, dst);
}
- __ far_call(RuntimeAddress(CAST_FROM_FN_PTR(address, ShenandoahBarrierSetAssembler::shenandoah_wb())));
+ __ far_call(RuntimeAddress(CAST_FROM_FN_PTR(address, ShenandoahBarrierSetAssembler::shenandoah_lrb())));
if (dst != r0) {
__ mov(dst, r0);
__ pop(to_save, sp);
}
@@ -269,14 +253,10 @@
__ bind(done);
}
void ShenandoahBarrierSetAssembler::storeval_barrier(MacroAssembler* masm, Register dst, Register tmp) {
if (ShenandoahStoreValEnqueueBarrier) {
- Label is_null;
- __ cbz(dst, is_null);
- write_barrier_impl(masm, dst);
- __ bind(is_null);
// Save possibly live regs.
RegSet live_regs = RegSet::range(r0, r4) - dst;
__ push(live_regs, sp);
__ strd(v0, __ pre(sp, 2 * -wordSize));
@@ -284,48 +264,49 @@
// Restore possibly live regs.
__ ldrd(v0, __ post(sp, 2 * wordSize));
__ pop(live_regs, sp);
}
- if (ShenandoahStoreValReadBarrier) {
- read_barrier_impl(masm, dst);
+}
+
+void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler* masm, Register dst, Register tmp) {
+ if (ShenandoahLoadRefBarrier) {
+ Label is_null;
+ __ cbz(dst, is_null);
+ load_reference_barrier_not_null(masm, dst, tmp);
+ __ bind(is_null);
}
}
void ShenandoahBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
Register dst, Address src, Register tmp1, Register tmp_thread) {
bool on_oop = type == T_OBJECT || type == T_ARRAY;
- bool in_heap = (decorators & IN_HEAP) != 0;
bool on_weak = (decorators & ON_WEAK_OOP_REF) != 0;
bool on_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0;
bool on_reference = on_weak || on_phantom;
- if (in_heap) {
- read_barrier_not_null(masm, src.base());
- }
-
BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread);
- if (ShenandoahKeepAliveBarrier && on_oop && on_reference) {
+ if (on_oop) {
+ load_reference_barrier(masm, dst, tmp1);
+
+ if (ShenandoahKeepAliveBarrier && on_reference) {
__ enter();
satb_write_barrier_pre(masm /* masm */,
noreg /* obj */,
dst /* pre_val */,
rthread /* thread */,
tmp1 /* tmp */,
true /* tosca_live */,
true /* expand_call */);
__ leave();
}
+ }
}
void ShenandoahBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
Address dst, Register val, Register tmp1, Register tmp2) {
bool on_oop = type == T_OBJECT || type == T_ARRAY;
- bool in_heap = (decorators & IN_HEAP) != 0;
- if (in_heap) {
- write_barrier(masm, dst.base());
- }
if (!on_oop) {
BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2);
return;
}
@@ -359,25 +340,10 @@
BarrierSetAssembler::store_at(masm, decorators, type, Address(r3, 0), val, noreg, noreg);
}
}
-void ShenandoahBarrierSetAssembler::obj_equals(MacroAssembler* masm, Register op1, Register op2) {
- __ cmp(op1, op2);
- if (ShenandoahAcmpBarrier) {
- Label done;
- __ br(Assembler::EQ, done);
- // The object may have been evacuated, but we won't see it without a
- // membar here.
- __ membar(Assembler::LoadStore| Assembler::LoadLoad);
- read_barrier(masm, op1);
- read_barrier(masm, op2);
- __ cmp(op1, op2);
- __ bind(done);
- }
-}
-
void ShenandoahBarrierSetAssembler::tlab_allocate(MacroAssembler* masm, Register obj,
Register var_size_in_bytes,
int con_size_in_bytes,
Register t1,
Register t2,
@@ -408,31 +374,10 @@
if (var_size_in_bytes == end) {
__ sub(var_size_in_bytes, var_size_in_bytes, obj);
}
}
-void ShenandoahBarrierSetAssembler::resolve(MacroAssembler* masm, DecoratorSet decorators, Register obj) {
- bool oop_not_null = (decorators & IS_NOT_NULL) != 0;
- bool is_write = (decorators & ACCESS_WRITE) != 0;
- if (is_write) {
- if (oop_not_null) {
- write_barrier(masm, obj);
- } else {
- Label done;
- __ cbz(obj, done);
- write_barrier(masm, obj);
- __ bind(done);
- }
- } else {
- if (oop_not_null) {
- read_barrier_not_null(masm, obj);
- } else {
- read_barrier(masm, obj);
- }
- }
-}
-
void ShenandoahBarrierSetAssembler::cmpxchg_oop(MacroAssembler* masm, Register addr, Register expected, Register new_val,
bool acquire, bool release, bool weak, bool is_cae,
Register result) {
Register tmp1 = rscratch1;
Register tmp2 = rscratch2;
@@ -467,12 +412,12 @@
__ mov(expected, tmp1);
if (is_narrow) {
__ decode_heap_oop(tmp1, tmp1);
__ decode_heap_oop(tmp2, tmp2);
}
- read_barrier_impl(masm, tmp1);
- read_barrier_impl(masm, tmp2);
+ resolve_forward_pointer(masm, tmp1);
+ resolve_forward_pointer(masm, tmp2);
__ cmp(tmp1, tmp2);
// Retry with expected now being the value we just loaded from addr.
__ br(Assembler::EQ, retry);
if (is_cae && is_narrow) {
// For cmp-and-exchange and narrow oops, we need to restore
@@ -513,11 +458,11 @@
ce->store_parameter(stub->pre_val()->as_register(), 0);
__ far_call(RuntimeAddress(bs->pre_barrier_c1_runtime_code_blob()->code_begin()));
__ b(*stub->continuation());
}
-void ShenandoahBarrierSetAssembler::gen_write_barrier_stub(LIR_Assembler* ce, ShenandoahWriteBarrierStub* stub) {
+void ShenandoahBarrierSetAssembler::gen_load_reference_barrier_stub(LIR_Assembler* ce, ShenandoahLoadReferenceBarrierStub* stub) {
Register obj = stub->obj()->as_register();
Register res = stub->result()->as_register();
Label done;
@@ -530,11 +475,11 @@
// Check for null.
if (stub->needs_null_check()) {
__ cbz(res, done);
}
- write_barrier(ce->masm(), res);
+ load_reference_barrier_not_null(ce->masm(), res, rscratch1);
__ bind(done);
__ b(*stub->continuation());
}
@@ -590,30 +535,30 @@
#undef __
#endif // COMPILER1
-address ShenandoahBarrierSetAssembler::shenandoah_wb() {
- assert(_shenandoah_wb != NULL, "need write barrier stub");
- return _shenandoah_wb;
+address ShenandoahBarrierSetAssembler::shenandoah_lrb() {
+ assert(_shenandoah_lrb != NULL, "need load reference barrier stub");
+ return _shenandoah_lrb;
}
#define __ cgen->assembler()->
-// Shenandoah write barrier.
+// Shenandoah load reference barrier.
//
// Input:
// r0: OOP to evacuate. Not null.
//
// Output:
// r0: Pointer to evacuated OOP.
//
// Trash rscratch1, rscratch2. Preserve everything else.
-address ShenandoahBarrierSetAssembler::generate_shenandoah_wb(StubCodeGenerator* cgen) {
+address ShenandoahBarrierSetAssembler::generate_shenandoah_lrb(StubCodeGenerator* cgen) {
__ align(6);
- StubCodeMark mark(cgen, "StubRoutines", "shenandoah_wb");
+ StubCodeMark mark(cgen, "StubRoutines", "shenandoah_lrb");
address start = __ pc();
Label work;
__ mov(rscratch2, ShenandoahHeap::in_cset_fast_test_addr());
__ lsr(rscratch1, r0, ShenandoahHeapRegion::region_size_bytes_shift_jint());
@@ -626,11 +571,11 @@
__ enter(); // required for proper stackwalking of RuntimeStub frame
__ push_call_clobbered_registers();
- __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_JRT));
+ __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_JRT));
__ blrt(lr, 1, 0, MacroAssembler::ret_type_integral);
__ mov(rscratch1, obj);
__ pop_call_clobbered_registers();
__ mov(obj, rscratch1);
@@ -641,14 +586,14 @@
}
#undef __
void ShenandoahBarrierSetAssembler::barrier_stubs_init() {
- if (ShenandoahWriteBarrier || ShenandoahStoreValEnqueueBarrier) {
+ if (ShenandoahLoadRefBarrier) {
int stub_code_size = 2048;
ResourceMark rm;
BufferBlob* bb = BufferBlob::create("shenandoah_barrier_stubs", stub_code_size);
CodeBuffer buf(bb);
StubCodeGenerator cgen(&buf);
- _shenandoah_wb = generate_shenandoah_wb(&cgen);
+ _shenandoah_lrb = generate_shenandoah_lrb(&cgen);
}
}
< prev index next >