# HG changeset patch # Parent 1aa6ed467484db1a4c8ebbd13ef02b6e534164ef 8232992: Shenandoah: Implement self-fixing interpreter LRB diff -r 1aa6ed467484 src/cpu/aarch64/vm/c1_Runtime1_aarch64.cpp --- a/src/cpu/aarch64/vm/c1_Runtime1_aarch64.cpp Wed Nov 25 13:06:46 2020 -0500 +++ b/src/cpu/aarch64/vm/c1_Runtime1_aarch64.cpp Thu Nov 26 14:49:16 2020 -0500 @@ -1309,9 +1309,9 @@ f.load_argument(0, r0); f.load_argument(1, r1); if (UseCompressedOops) { - __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_fixup_narrow)); + __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_narrow)); } else { - __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_fixup)); + __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier)); } __ blr(lr); __ mov(rscratch1, r0); diff -r 1aa6ed467484 src/cpu/aarch64/vm/macroAssembler_aarch64.cpp --- a/src/cpu/aarch64/vm/macroAssembler_aarch64.cpp Wed Nov 25 13:06:46 2020 -0500 +++ b/src/cpu/aarch64/vm/macroAssembler_aarch64.cpp Thu Nov 26 14:49:16 2020 -0500 @@ -3473,34 +3473,36 @@ void MacroAssembler::load_heap_oop(Register dst, Address src) { +#if INCLUDE_ALL_GCS + if (UseShenandoahGC) { + ShenandoahBarrierSetAssembler::bsasm()->load_heap_oop(this, dst, src); + return; + } +#endif + if (UseCompressedOops) { ldrw(dst, src); decode_heap_oop(dst); } else { ldr(dst, src); } - +} + +void MacroAssembler::load_heap_oop_not_null(Register dst, Address src) +{ #if INCLUDE_ALL_GCS if (UseShenandoahGC) { - ShenandoahBarrierSetAssembler::bsasm()->load_reference_barrier(this, dst); + ShenandoahBarrierSetAssembler::bsasm()->load_heap_oop(this, dst, src); + return; } #endif -} - -void MacroAssembler::load_heap_oop_not_null(Register dst, Address src) -{ + if (UseCompressedOops) { ldrw(dst, src); decode_heap_oop_not_null(dst); } else { ldr(dst, src); } - -#if INCLUDE_ALL_GCS - if (UseShenandoahGC) { - ShenandoahBarrierSetAssembler::bsasm()->load_reference_barrier(this, dst); - } -#endif } void MacroAssembler::store_heap_oop(Address dst, Register src) { diff -r 1aa6ed467484 src/cpu/aarch64/vm/shenandoahBarrierSetAssembler_aarch64.cpp --- a/src/cpu/aarch64/vm/shenandoahBarrierSetAssembler_aarch64.cpp Wed Nov 25 13:06:46 2020 -0500 +++ b/src/cpu/aarch64/vm/shenandoahBarrierSetAssembler_aarch64.cpp Thu Nov 26 14:49:16 2020 -0500 @@ -119,36 +119,60 @@ } } -void ShenandoahBarrierSetAssembler::load_reference_barrier_not_null(MacroAssembler* masm, Register dst) { - assert(ShenandoahLoadRefBarrier, "Should be enabled"); +void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler* masm, Register dst, Address load_addr) { + if (!ShenandoahLoadRefBarrier) { + return; + } + assert(dst != rscratch2, "need rscratch2"); + assert_different_registers(load_addr.base(), load_addr.index(), rscratch1, rscratch2); - Label done; + bool is_narrow = UseCompressedOops; + + Label heap_stable, not_cset; __ enter(); Address gc_state(rthread, in_bytes(JavaThread::gc_state_offset())); __ ldrb(rscratch2, gc_state); // Check for heap stability - __ tbz(rscratch2, ShenandoahHeap::HAS_FORWARDED_BITPOS, done); + __ tbz(rscratch2, ShenandoahHeap::HAS_FORWARDED_BITPOS, heap_stable); - RegSet to_save = RegSet::of(r0); - if (dst != r0) { - __ push(to_save, sp); - __ mov(r0, dst); + // use r1 for load address + Register result_dst = dst; + if (dst == r1) { + __ mov(rscratch1, dst); + dst = rscratch1; } + // Save r0 and r1, unless it is an output register + RegSet to_save = RegSet::of(r0, r1) - result_dst; + __ push(to_save, sp); + __ lea(r1, load_addr); + __ mov(r0, dst); + + // Test for in-cset + __ mov(rscratch2, ShenandoahHeap::in_cset_fast_test_addr()); + __ lsr(rscratch1, r0, ShenandoahHeapRegion::region_size_bytes_shift_jint()); + __ ldrb(rscratch2, Address(rscratch2, rscratch1)); + __ tbz(rscratch2, 0, not_cset); + __ push_call_clobbered_registers(); - __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_interpreter), r0); + if (is_narrow) { + __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_narrow)); + } else { + __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier)); + } + __ blr(lr); __ mov(rscratch1, r0); __ pop_call_clobbered_registers(); __ mov(r0, rscratch1); - if (dst != r0) { - __ mov(dst, r0); - __ pop(to_save, sp); - } + __ bind(not_cset); - __ bind(done); + __ mov(result_dst, r0); + __ pop(to_save, sp); + + __ bind(heap_stable); __ leave(); } @@ -167,12 +191,27 @@ } } -void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler* masm, Register dst) { - if (ShenandoahLoadRefBarrier) { - Label is_null; - __ cbz(dst, is_null); - load_reference_barrier_not_null(masm, dst); - __ bind(is_null); +void ShenandoahBarrierSetAssembler::load_heap_oop(MacroAssembler* masm, Register dst, Address src) { + Register result_dst = dst; + + // Preserve src location for LRB + if (dst == src.base() || dst == src.index()) { + dst = rscratch1; + } + assert_different_registers(dst, src.base(), src.index()); + + if (UseCompressedOops) { + __ ldrw(dst, src); + __ decode_heap_oop(dst); + } else { + __ ldr(dst, src); + } + + load_reference_barrier(masm, dst, src); + + if (dst != result_dst) { + __ mov(result_dst, dst); + dst = result_dst; } } diff -r 1aa6ed467484 src/cpu/aarch64/vm/shenandoahBarrierSetAssembler_aarch64.hpp --- a/src/cpu/aarch64/vm/shenandoahBarrierSetAssembler_aarch64.hpp Wed Nov 25 13:06:46 2020 -0500 +++ b/src/cpu/aarch64/vm/shenandoahBarrierSetAssembler_aarch64.hpp Thu Nov 26 14:49:16 2020 -0500 @@ -39,8 +39,6 @@ void resolve_forward_pointer(MacroAssembler* masm, Register dst, Register tmp = noreg); void resolve_forward_pointer_not_null(MacroAssembler* masm, Register dst, Register tmp = noreg); - void load_reference_barrier_not_null(MacroAssembler* masm, Register dst); - public: static ShenandoahBarrierSetAssembler* bsasm(); @@ -50,7 +48,9 @@ void gen_load_reference_barrier_stub(LIR_Assembler* ce, ShenandoahLoadReferenceBarrierStub* stub); #endif - void load_reference_barrier(MacroAssembler* masm, Register dst); + void load_reference_barrier(MacroAssembler* masm, Register dst, Address src); + + void load_heap_oop(MacroAssembler* masm, Register dst, Address src); virtual void arraycopy_prologue(MacroAssembler* masm, bool dest_uninitialized, Register src, Register dst, Register count); diff -r 1aa6ed467484 src/cpu/x86/vm/c1_Runtime1_x86.cpp --- a/src/cpu/x86/vm/c1_Runtime1_x86.cpp Wed Nov 25 13:06:46 2020 -0500 +++ b/src/cpu/x86/vm/c1_Runtime1_x86.cpp Thu Nov 26 14:49:16 2020 -0500 @@ -1789,14 +1789,14 @@ f.load_argument(0, c_rarg0); f.load_argument(1, c_rarg1); if (UseCompressedOops) { - __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_fixup_narrow), c_rarg0, c_rarg1); + __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_narrow), c_rarg0, c_rarg1); } else { - __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_fixup), c_rarg0, c_rarg1); + __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier), c_rarg0, c_rarg1); } #else f.load_argument(0, rax); f.load_argument(1, rbx); - __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_fixup), rax, rbx); + __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier), rax, rbx); #endif restore_live_registers_except_rax(sasm, true); diff -r 1aa6ed467484 src/cpu/x86/vm/macroAssembler_x86.cpp --- a/src/cpu/x86/vm/macroAssembler_x86.cpp Wed Nov 25 13:06:46 2020 -0500 +++ b/src/cpu/x86/vm/macroAssembler_x86.cpp Thu Nov 26 14:49:16 2020 -0500 @@ -5786,6 +5786,14 @@ } void MacroAssembler::load_heap_oop(Register dst, Address src) { +#if INCLUDE_ALL_GCS + Register result_dst = dst; + if (UseShenandoahGC) { + ShenandoahBarrierSetAssembler::bsasm()->load_heap_oop(this, dst, src); + return; + } +#endif + #ifdef _LP64 // FIXME: Must change all places where we try to load the klass. if (UseCompressedOops) { @@ -5794,16 +5802,18 @@ } else #endif movptr(dst, src); - -#if INCLUDE_ALL_GCS - if (UseShenandoahGC) { - ShenandoahBarrierSetAssembler::bsasm()->load_reference_barrier(this, dst); - } -#endif } // Doesn't do verfication, generates fixed size code void MacroAssembler::load_heap_oop_not_null(Register dst, Address src) { +#if INCLUDE_ALL_GCS + Register result_dst = dst; + if (UseShenandoahGC) { + ShenandoahBarrierSetAssembler::bsasm()->load_heap_oop(this, dst, src); + return; + } +#endif + #ifdef _LP64 if (UseCompressedOops) { movl(dst, src); @@ -5811,12 +5821,6 @@ } else #endif movptr(dst, src); - -#if INCLUDE_ALL_GCS - if (UseShenandoahGC) { - ShenandoahBarrierSetAssembler::bsasm()->load_reference_barrier(this, dst); - } -#endif } void MacroAssembler::store_heap_oop(Address dst, Register src) { diff -r 1aa6ed467484 src/cpu/x86/vm/shenandoahBarrierSetAssembler_x86.cpp --- a/src/cpu/x86/vm/shenandoahBarrierSetAssembler_x86.cpp Wed Nov 25 13:06:46 2020 -0500 +++ b/src/cpu/x86/vm/shenandoahBarrierSetAssembler_x86.cpp Thu Nov 26 14:49:16 2020 -0500 @@ -40,6 +40,30 @@ #define __ masm-> +static void save_xmm_registers(MacroAssembler* masm) { + __ subptr(rsp, 64); + __ movdbl(Address(rsp, 0), xmm0); + __ movdbl(Address(rsp, 8), xmm1); + __ movdbl(Address(rsp, 16), xmm2); + __ movdbl(Address(rsp, 24), xmm3); + __ movdbl(Address(rsp, 32), xmm4); + __ movdbl(Address(rsp, 40), xmm5); + __ movdbl(Address(rsp, 48), xmm6); + __ movdbl(Address(rsp, 56), xmm7); +} + +static void restore_xmm_registers(MacroAssembler* masm) { + __ movdbl(xmm0, Address(rsp, 0)); + __ movdbl(xmm1, Address(rsp, 8)); + __ movdbl(xmm2, Address(rsp, 16)); + __ movdbl(xmm3, Address(rsp, 24)); + __ movdbl(xmm4, Address(rsp, 32)); + __ movdbl(xmm5, Address(rsp, 40)); + __ movdbl(xmm6, Address(rsp, 48)); + __ movdbl(xmm7, Address(rsp, 56)); + __ addptr(rsp, 64); +} + void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, bool dest_uninitialized, Register src, Register dst, Register count) { @@ -102,11 +126,18 @@ } } -void ShenandoahBarrierSetAssembler::load_reference_barrier_not_null(MacroAssembler* masm, Register dst) { - assert(ShenandoahLoadRefBarrier, "Should be enabled"); +void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler* masm, Register dst, Address src) { + if (!ShenandoahLoadRefBarrier) { + return; + } - Label done; + bool is_narrow = UseCompressedOops; + Label heap_stable, not_cset; + + __ block_comment("load_reference_barrier { "); + + // Check if GC is active #ifdef _LP64 Register thread = r15_thread; #else @@ -117,67 +148,113 @@ __ push(thread); __ get_thread(thread); #endif - assert_different_registers(dst, thread); Address gc_state(thread, in_bytes(JavaThread::gc_state_offset())); __ testb(gc_state, ShenandoahHeap::HAS_FORWARDED); - __ jcc(Assembler::zero, done); + __ jcc(Assembler::zero, heap_stable); - { - __ save_vector_registers(); + Register tmp1 = noreg, tmp2 = noreg; - __ subptr(rsp, LP64_ONLY(16) NOT_LP64(8) * wordSize); + // Test for object in cset + // Allocate temporary registers + for (int i = 0; i < 8; i++) { + Register r = as_Register(i); + if (r != rsp && r != rbp && r != dst && r != src.base() && r != src.index()) { + if (tmp1 == noreg) { + tmp1 = r; + } else { + tmp2 = r; + break; + } + } + } + assert(tmp1 != noreg, "tmp1 allocated"); + assert(tmp2 != noreg, "tmp2 allocated"); + assert_different_registers(tmp1, tmp2, src.base(), src.index()); + assert_different_registers(tmp1, tmp2, dst); - __ movptr(Address(rsp, 0 * wordSize), rax); - __ movptr(Address(rsp, 1 * wordSize), rcx); - __ movptr(Address(rsp, 2 * wordSize), rdx); - __ movptr(Address(rsp, 3 * wordSize), rbx); - // skip rsp - __ movptr(Address(rsp, 5 * wordSize), rbp); - __ movptr(Address(rsp, 6 * wordSize), rsi); - __ movptr(Address(rsp, 7 * wordSize), rdi); + __ push(tmp1); + __ push(tmp2); + + // Optimized cset-test + __ movptr(tmp1, dst); + __ shrptr(tmp1, ShenandoahHeapRegion::region_size_bytes_shift_jint()); + __ movptr(tmp2, (intptr_t) ShenandoahHeap::in_cset_fast_test_addr()); + __ movbool(tmp1, Address(tmp1, tmp2, Address::times_1)); + __ testbool(tmp1); + __ jcc(Assembler::zero, not_cset); + + uint num_saved_regs = 4 + (dst != rax ? 1 : 0) LP64_ONLY(+4); + __ subptr(rsp, num_saved_regs * wordSize); + uint slot = num_saved_regs; + if (dst != rax) { + __ movptr(Address(rsp, (--slot) * wordSize), rax); + } + __ movptr(Address(rsp, (--slot) * wordSize), rcx); + __ movptr(Address(rsp, (--slot) * wordSize), rdx); + __ movptr(Address(rsp, (--slot) * wordSize), rdi); + __ movptr(Address(rsp, (--slot) * wordSize), rsi); #ifdef _LP64 - __ movptr(Address(rsp, 8 * wordSize), r8); - __ movptr(Address(rsp, 9 * wordSize), r9); - __ movptr(Address(rsp, 10 * wordSize), r10); - __ movptr(Address(rsp, 11 * wordSize), r11); - __ movptr(Address(rsp, 12 * wordSize), r12); - __ movptr(Address(rsp, 13 * wordSize), r13); - __ movptr(Address(rsp, 14 * wordSize), r14); - __ movptr(Address(rsp, 15 * wordSize), r15); + __ movptr(Address(rsp, (--slot) * wordSize), r8); + __ movptr(Address(rsp, (--slot) * wordSize), r9); + __ movptr(Address(rsp, (--slot) * wordSize), r10); + __ movptr(Address(rsp, (--slot) * wordSize), r11); + // r12-r15 are callee saved in all calling conventions #endif + assert(slot == 0, "must use all slots"); + + // Shuffle registers such that dst is in c_rarg0 and addr in c_rarg1. +#ifdef _LP64 + Register arg0 = c_rarg0, arg1 = c_rarg1; +#else + Register arg0 = rdi, arg1 = rsi; +#endif + if (dst == arg1) { + __ lea(arg0, src); + __ xchgptr(arg1, arg0); + } else { + __ lea(arg1, src); + __ movptr(arg0, dst); } - __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_interpreter), dst); - { + + save_xmm_registers(masm); + if (is_narrow) { + __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_narrow), arg0, arg1); + } else { + __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier), arg0, arg1); + } + restore_xmm_registers(masm); + #ifdef _LP64 - __ movptr(r15, Address(rsp, 15 * wordSize)); - __ movptr(r14, Address(rsp, 14 * wordSize)); - __ movptr(r13, Address(rsp, 13 * wordSize)); - __ movptr(r12, Address(rsp, 12 * wordSize)); - __ movptr(r11, Address(rsp, 11 * wordSize)); - __ movptr(r10, Address(rsp, 10 * wordSize)); - __ movptr(r9, Address(rsp, 9 * wordSize)); - __ movptr(r8, Address(rsp, 8 * wordSize)); + __ movptr(r11, Address(rsp, (slot++) * wordSize)); + __ movptr(r10, Address(rsp, (slot++) * wordSize)); + __ movptr(r9, Address(rsp, (slot++) * wordSize)); + __ movptr(r8, Address(rsp, (slot++) * wordSize)); #endif - __ movptr(rdi, Address(rsp, 7 * wordSize)); - __ movptr(rsi, Address(rsp, 6 * wordSize)); - __ movptr(rbp, Address(rsp, 5 * wordSize)); - // skip rsp - __ movptr(rbx, Address(rsp, 3 * wordSize)); - __ movptr(rdx, Address(rsp, 2 * wordSize)); - __ movptr(rcx, Address(rsp, 1 * wordSize)); - if (dst != rax) { - __ movptr(dst, rax); - __ movptr(rax, Address(rsp, 0 * wordSize)); - } - __ addptr(rsp, LP64_ONLY(16) NOT_LP64(8) * wordSize); + __ movptr(rsi, Address(rsp, (slot++) * wordSize)); + __ movptr(rdi, Address(rsp, (slot++) * wordSize)); + __ movptr(rdx, Address(rsp, (slot++) * wordSize)); + __ movptr(rcx, Address(rsp, (slot++) * wordSize)); - __ restore_vector_registers(); + if (dst != rax) { + __ movptr(dst, rax); + __ movptr(rax, Address(rsp, (slot++) * wordSize)); } - __ bind(done); + + assert(slot == num_saved_regs, "must use all slots"); + __ addptr(rsp, num_saved_regs * wordSize); + + __ bind(not_cset); + + __ pop(tmp2); + __ pop(tmp1); + + __ bind(heap_stable); + + __ block_comment("} load_reference_barrier"); #ifndef _LP64 - __ pop(thread); + __ pop(thread); #endif } @@ -222,13 +299,30 @@ } } -void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler* masm, Register dst) { - if (ShenandoahLoadRefBarrier) { - Label done; - __ testptr(dst, dst); - __ jcc(Assembler::zero, done); - load_reference_barrier_not_null(masm, dst); - __ bind(done); +void ShenandoahBarrierSetAssembler::load_heap_oop(MacroAssembler* masm, Register dst, Address src) { + Register result_dst = dst; + // Preserve src location for LRB + if (dst == src.base() || dst == src.index()) { + dst = rdi; + __ push(dst); + assert_different_registers(dst, src.base(), src.index()); + } + +#ifdef _LP64 + // FIXME: Must change all places where we try to load the klass. + if (UseCompressedOops) { + __ movl(dst, src); + __ decode_heap_oop(dst); + } else +#endif + __ movptr(dst, src); + + load_reference_barrier(masm, dst, src); + + // Move loaded oop to final destination + if (dst != result_dst) { + __ movptr(result_dst, dst); + __ pop(dst); } } diff -r 1aa6ed467484 src/cpu/x86/vm/shenandoahBarrierSetAssembler_x86.hpp --- a/src/cpu/x86/vm/shenandoahBarrierSetAssembler_x86.hpp Wed Nov 25 13:06:46 2020 -0500 +++ b/src/cpu/x86/vm/shenandoahBarrierSetAssembler_x86.hpp Thu Nov 26 14:49:16 2020 -0500 @@ -36,8 +36,6 @@ class ShenandoahBarrierSetAssembler : public CHeapObj { private: - void load_reference_barrier_not_null(MacroAssembler* masm, Register dst); - void storeval_barrier_impl(MacroAssembler* masm, Register dst, Register tmp); public: @@ -48,7 +46,9 @@ void gen_load_reference_barrier_stub(LIR_Assembler* ce, ShenandoahLoadReferenceBarrierStub* stub); #endif - void load_reference_barrier(MacroAssembler* masm, Register dst); + void load_reference_barrier(MacroAssembler* masm, Register dst, Address src); + + void load_heap_oop(MacroAssembler* masm, Register dst, Address src); virtual void arraycopy_prologue(MacroAssembler* masm, bool dest_uninitialized, Register src, Register dst, Register count); diff -r 1aa6ed467484 src/share/vm/gc_implementation/shenandoah/c2/shenandoahBarrierSetC2.cpp --- a/src/share/vm/gc_implementation/shenandoah/c2/shenandoahBarrierSetC2.cpp Wed Nov 25 13:06:46 2020 -0500 +++ b/src/share/vm/gc_implementation/shenandoah/c2/shenandoahBarrierSetC2.cpp Thu Nov 26 14:49:16 2020 -0500 @@ -40,8 +40,7 @@ address entry_point = call->as_CallLeaf()->entry_point(); return (entry_point == CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier)) || - (entry_point == CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_fixup)) || - (entry_point == CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_fixup_narrow)); + (entry_point == CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_narrow)); } bool ShenandoahBarrierSetC2::is_shenandoah_state_load(Node* n) { diff -r 1aa6ed467484 src/share/vm/gc_implementation/shenandoah/c2/shenandoahSupport.cpp --- a/src/share/vm/gc_implementation/shenandoah/c2/shenandoahSupport.cpp Wed Nov 25 13:06:46 2020 -0500 +++ b/src/share/vm/gc_implementation/shenandoah/c2/shenandoahSupport.cpp Thu Nov 26 14:49:16 2020 -0500 @@ -1011,8 +1011,8 @@ phase->register_new_node(mm, ctrl); address target = LP64_ONLY(UseCompressedOops) NOT_LP64(false) ? - CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_fixup_narrow) : - CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_fixup); + CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_narrow) : + CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier); Node* call = new (phase->C) CallLeafNode(ShenandoahBarrierSetC2::shenandoah_load_reference_barrier_Type(), target, diff -r 1aa6ed467484 src/share/vm/gc_implementation/shenandoah/shenandoahRuntime.cpp --- a/src/share/vm/gc_implementation/shenandoah/shenandoahRuntime.cpp Wed Nov 25 13:06:46 2020 -0500 +++ b/src/share/vm/gc_implementation/shenandoah/shenandoahRuntime.cpp Thu Nov 26 14:49:16 2020 -0500 @@ -47,15 +47,11 @@ thread->satb_mark_queue().enqueue_known_active(orig); JRT_END -JRT_LEAF(oopDesc*, ShenandoahRuntime::load_reference_barrier(oopDesc* src)) - return ShenandoahBarrierSet::barrier_set()->load_reference_barrier_mutator(src, (oop*)NULL); -JRT_END - -JRT_LEAF(oopDesc*, ShenandoahRuntime::load_reference_barrier_fixup(oopDesc* src, oop* load_addr)) +JRT_LEAF(oopDesc*, ShenandoahRuntime::load_reference_barrier(oopDesc* src, oop* load_addr)) return ShenandoahBarrierSet::barrier_set()->load_reference_barrier_mutator(src, load_addr); JRT_END -JRT_LEAF(oopDesc*, ShenandoahRuntime::load_reference_barrier_fixup_narrow(oopDesc* src, narrowOop* load_addr)) +JRT_LEAF(oopDesc*, ShenandoahRuntime::load_reference_barrier_narrow(oopDesc* src, narrowOop* load_addr)) return ShenandoahBarrierSet::barrier_set()->load_reference_barrier_mutator(src, load_addr); JRT_END diff -r 1aa6ed467484 src/share/vm/gc_implementation/shenandoah/shenandoahRuntime.hpp --- a/src/share/vm/gc_implementation/shenandoah/shenandoahRuntime.hpp Wed Nov 25 13:06:46 2020 -0500 +++ b/src/share/vm/gc_implementation/shenandoah/shenandoahRuntime.hpp Thu Nov 26 14:49:16 2020 -0500 @@ -38,9 +38,8 @@ static void write_ref_field_pre_entry(oopDesc* orig, JavaThread* thread); - static oopDesc* load_reference_barrier(oopDesc* src); - static oopDesc* load_reference_barrier_fixup(oopDesc* src, oop* load_addr); - static oopDesc* load_reference_barrier_fixup_narrow(oopDesc* src, narrowOop* load_addr); + static oopDesc* load_reference_barrier(oopDesc* src, oop* load_addr); + static oopDesc* load_reference_barrier_narrow(oopDesc* src, narrowOop* load_addr); static oopDesc* load_reference_barrier_interpreter(oopDesc* src); static void shenandoah_clone_barrier(oopDesc* src);