# HG changeset patch # User rkennke # Date 1563545095 -7200 # Fri Jul 19 16:04:55 2019 +0200 # Node ID ba00f2c5b996c95bc611aa6b86fc313396df5d19 # Parent 6f60cfd502c3c03f5b3ef560e8f0c1c2c4407918 8228369: Shenandoah: Refactor LRB C1 stubs diff --git a/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp --- a/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp @@ -472,6 +472,27 @@ } } +// Generate cset check. If obj is not in cset, branch to done label, otherwise fall through +// obj: Register holding the oop, preserved +// tmp1, tmp2: temp registers, trashed +void ShenandoahBarrierSetAssembler::gen_cset_check(MacroAssembler* masm, Register obj, Register tmp1, Register tmp2, Label& done) { + __ mov(tmp2, ShenandoahHeap::in_cset_fast_test_addr()); + __ lsr(tmp1, obj, ShenandoahHeapRegion::region_size_bytes_shift_jint()); + __ ldrb(tmp2, Address(tmp2, tmp1)); + __ tbz(tmp2, 0, done); +} + +// Generate check if object is resolved. Branch to not_resolved label, if not. Otherwise return resolved +// object in obj register. +// obj: object, resolved object on normal return +// tmp1, tmp2: temp registers, trashed +void ShenandoahBarrierSetAssembler::gen_resolved_check(MacroAssembler* masm, Register obj, Register tmp1, Register tmp2, Label& resolved) { + __ mov(tmp2, obj); + resolve_forward_pointer_not_null(masm, obj, tmp1); + __ cmp(tmp2, obj); + __ br(Assembler::EQ, resolved); +} + #undef __ #ifdef COMPILER1 @@ -501,23 +522,33 @@ } void ShenandoahBarrierSetAssembler::gen_load_reference_barrier_stub(LIR_Assembler* ce, ShenandoahLoadReferenceBarrierStub* stub) { + ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1(); + __ bind(*stub->entry()); Register obj = stub->obj()->as_register(); Register res = stub->result()->as_register(); + Register tmp1 = stub->tmp1()->as_register(); + Register tmp2 = stub->tmp2()->as_register(); - Label done; + Label slow_path; - __ bind(*stub->entry()); + assert(res == r0, "result must arrive in r0"); if (res != obj) { __ mov(res, obj); } + // Check for null. - __ cbz(res, done); + __ cbz(res, *stub->continuation()); - load_reference_barrier_not_null(ce->masm(), res, rscratch1); + gen_cset_check(ce->masm(), res, tmp1, tmp2, *stub->continuation()); + gen_resolved_check(ce->masm(), res, tmp1, tmp2, slow_path); + __ b(*stub->continuation()); - __ bind(done); + __ bind(slow_path); + ce->store_parameter(res, 0); + __ far_call(RuntimeAddress(bs->load_reference_barrier_rt_code_blob()->code_begin())); + __ b(*stub->continuation()); } @@ -571,6 +602,21 @@ __ epilogue(); } +void ShenandoahBarrierSetAssembler::generate_c1_load_reference_barrier_runtime_stub(StubAssembler* sasm) { + __ prologue("shenandoah_load_reference_barrier", false); + // arg0 : object to be resolved + + __ push_call_clobbered_registers(); + __ load_parameter(0, r0); + __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier)); + __ blrt(lr, 1, 0, MacroAssembler::ret_type_integral); + __ mov(rscratch1, r0); + __ pop_call_clobbered_registers(); + __ mov(r0, rscratch1); + + __ epilogue(); +} + #undef __ #endif // COMPILER1 @@ -597,19 +643,12 @@ StubCodeMark mark(cgen, "StubRoutines", "shenandoah_lrb"); address start = __ pc(); - Label work, done; - __ mov(rscratch2, ShenandoahHeap::in_cset_fast_test_addr()); - __ lsr(rscratch1, r0, ShenandoahHeapRegion::region_size_bytes_shift_jint()); - __ ldrb(rscratch2, Address(rscratch2, rscratch1)); - __ tbnz(rscratch2, 0, work); + Label done, call_rt; + gen_cset_check(cgen->assembler(), r0, rscratch1, rscratch2, done); + gen_resolved_check(cgen->assembler(), r0, rscratch1, rscratch2, call_rt); __ ret(lr); - __ bind(work); - __ mov(rscratch2, r0); - resolve_forward_pointer_not_null(cgen->assembler(), r0, rscratch1); - __ cmp(rscratch2, r0); - __ br(Assembler::NE, done); - + __ bind(call_rt); __ enter(); // required for proper stackwalking of RuntimeStub frame __ push_call_clobbered_registers(); @@ -621,6 +660,7 @@ __ mov(r0, rscratch1); __ leave(); // required for proper stackwalking of RuntimeStub frame + __ bind(done); __ ret(lr); diff --git a/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.hpp b/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.hpp --- a/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.hpp +++ b/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.hpp @@ -62,6 +62,17 @@ address generate_shenandoah_lrb(StubCodeGenerator* cgen); + // Generate cset check. If obj is not in cset, branch to done label, otherwise fall through + // obj: Register holding the oop, preserved + // tmp1, tmp2: temp registers, trashed + void gen_cset_check(MacroAssembler* masm, Register obj, Register tmp1, Register tmp2, Label& done); + + // Generate check if object is resolved. Branch to not_resolved label, if not. Otherwise return resolved + // object in obj register. + // obj: object, resolved object on normal return + // tmp1, tmp2: temp registers, trashed + void gen_resolved_check(MacroAssembler* masm, Register obj, Register tmp1, Register tmp2, Label& resolved); + public: static address shenandoah_lrb(); @@ -71,6 +82,7 @@ void gen_pre_barrier_stub(LIR_Assembler* ce, ShenandoahPreBarrierStub* stub); void gen_load_reference_barrier_stub(LIR_Assembler* ce, ShenandoahLoadReferenceBarrierStub* stub); void generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm); + void generate_c1_load_reference_barrier_runtime_stub(StubAssembler* sasm); #endif virtual void arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop, diff --git a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp --- a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp +++ b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp @@ -736,106 +736,42 @@ } } -void ShenandoahBarrierSetAssembler::save_vector_registers(MacroAssembler* masm) { - int num_xmm_regs = LP64_ONLY(16) NOT_LP64(8); - if (UseAVX > 2) { - num_xmm_regs = LP64_ONLY(32) NOT_LP64(8); - } - - if (UseSSE == 1) { - __ subptr(rsp, sizeof(jdouble)*8); - for (int n = 0; n < 8; n++) { - __ movflt(Address(rsp, n*sizeof(jdouble)), as_XMMRegister(n)); - } - } else if (UseSSE >= 2) { - if (UseAVX > 2) { - __ push(rbx); - __ movl(rbx, 0xffff); - __ kmovwl(k1, rbx); - __ pop(rbx); - } -#ifdef COMPILER2 - if (MaxVectorSize > 16) { - if(UseAVX > 2) { - // Save upper half of ZMM registers - __ subptr(rsp, 32*num_xmm_regs); - for (int n = 0; n < num_xmm_regs; n++) { - __ vextractf64x4_high(Address(rsp, n*32), as_XMMRegister(n)); - } - } - assert(UseAVX > 0, "256 bit vectors are supported only with AVX"); - // Save upper half of YMM registers - __ subptr(rsp, 16*num_xmm_regs); - for (int n = 0; n < num_xmm_regs; n++) { - __ vextractf128_high(Address(rsp, n*16), as_XMMRegister(n)); - } - } -#endif - // Save whole 128bit (16 bytes) XMM registers - __ subptr(rsp, 16*num_xmm_regs); -#ifdef _LP64 - if (VM_Version::supports_evex()) { - for (int n = 0; n < num_xmm_regs; n++) { - __ vextractf32x4(Address(rsp, n*16), as_XMMRegister(n), 0); - } - } else { - for (int n = 0; n < num_xmm_regs; n++) { - __ movdqu(Address(rsp, n*16), as_XMMRegister(n)); - } - } -#else - for (int n = 0; n < num_xmm_regs; n++) { - __ movdqu(Address(rsp, n*16), as_XMMRegister(n)); - } -#endif - } +// Generate cset check. If obj is not in cset, branch to done label, otherwise fall through +// obj: Register holding the oop, preserved +// tmp1, tmp2: temp registers, trashed +void ShenandoahBarrierSetAssembler::gen_cset_check(MacroAssembler* masm, Register obj, Register tmp1, Register tmp2, Label& done) { + // Check for object being in the collection set. + // TODO: Can we use only 1 register here? + // The source object arrives here in rax. + // live: rax + // live: tmp1 + __ mov(tmp1, obj); + __ shrptr(tmp1, ShenandoahHeapRegion::region_size_bytes_shift_jint()); + // live: tmp2 + __ movptr(tmp2, (intptr_t) ShenandoahHeap::in_cset_fast_test_addr()); + __ movbool(tmp2, Address(tmp2, tmp1, Address::times_1)); + // unlive: tmp1 + __ testbool(tmp2); + // unlive: tmp2 + __ jcc(Assembler::zero, done); } -void ShenandoahBarrierSetAssembler::restore_vector_registers(MacroAssembler* masm) { - int num_xmm_regs = LP64_ONLY(16) NOT_LP64(8); - if (UseAVX > 2) { - num_xmm_regs = LP64_ONLY(32) NOT_LP64(8); - } - if (UseSSE == 1) { - for (int n = 0; n < 8; n++) { - __ movflt(as_XMMRegister(n), Address(rsp, n*sizeof(jdouble))); - } - __ addptr(rsp, sizeof(jdouble)*8); - } else if (UseSSE >= 2) { - // Restore whole 128bit (16 bytes) XMM registers -#ifdef _LP64 - if (VM_Version::supports_evex()) { - for (int n = 0; n < num_xmm_regs; n++) { - __ vinsertf32x4(as_XMMRegister(n), as_XMMRegister(n), Address(rsp, n*16), 0); - } - } else { - for (int n = 0; n < num_xmm_regs; n++) { - __ movdqu(as_XMMRegister(n), Address(rsp, n*16)); - } - } -#else - for (int n = 0; n < num_xmm_regs; n++) { - __ movdqu(as_XMMRegister(n), Address(rsp, n*16)); - } -#endif - __ addptr(rsp, 16*num_xmm_regs); - -#ifdef COMPILER2 - if (MaxVectorSize > 16) { - // Restore upper half of YMM registers. - for (int n = 0; n < num_xmm_regs; n++) { - __ vinsertf128_high(as_XMMRegister(n), Address(rsp, n*16)); - } - __ addptr(rsp, 16*num_xmm_regs); - if (UseAVX > 2) { - for (int n = 0; n < num_xmm_regs; n++) { - __ vinsertf64x4_high(as_XMMRegister(n), Address(rsp, n*32)); - } - __ addptr(rsp, 32*num_xmm_regs); - } - } -#endif - } +// Generate check if object is resolved. Branch to resolved label, if not. Otherwise return resolved +// object in obj register. +// obj: object, resolved object on normal return +// tmp: temp register, trashed +void ShenandoahBarrierSetAssembler::gen_resolved_check(MacroAssembler* masm, Register obj, Register tmp, Label& not_resolved) { + __ movptr(tmp, Address(obj, oopDesc::mark_offset_in_bytes())); + // Test if both lowest bits are set. We trick it by negating the bits + // then test for both bits clear. + __ notptr(tmp); + __ testb(tmp, markOopDesc::marked_value); + __ jccb(Assembler::notZero, not_resolved); + // Clear both lower bits. It's still inverted, so set them, and then invert back. + __ orptr(tmp, markOopDesc::marked_value); + __ notptr(tmp); + // At this point, tmp2 contains the decoded forwarding pointer. + __ mov(obj, tmp); } #undef __ @@ -869,11 +805,17 @@ } void ShenandoahBarrierSetAssembler::gen_load_reference_barrier_stub(LIR_Assembler* ce, ShenandoahLoadReferenceBarrierStub* stub) { + ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1(); __ bind(*stub->entry()); - Label done; Register obj = stub->obj()->as_register(); Register res = stub->result()->as_register(); + Register tmp1 = stub->tmp1()->as_register(); + Register tmp2 = stub->tmp2()->as_register(); + + Label slow_path; + + assert(res == rax, "result must arrive in rax"); if (res != obj) { __ mov(res, obj); @@ -881,11 +823,17 @@ // Check for null. __ testptr(res, res); - __ jcc(Assembler::zero, done); + __ jcc(Assembler::zero, *stub->continuation()); + + gen_cset_check(ce->masm(), res, tmp1, tmp2, *stub->continuation()); + gen_resolved_check(ce->masm(), rax, tmp1, slow_path); - load_reference_barrier_not_null(ce->masm(), res); + __ jmp(*stub->continuation()); - __ bind(done); + __ bind(slow_path); + ce->store_parameter(res, 0); + __ call(RuntimeAddress(bs->load_reference_barrier_rt_code_blob()->code_begin())); + __ jmp(*stub->continuation()); } @@ -949,6 +897,18 @@ __ epilogue(); } +void ShenandoahBarrierSetAssembler::generate_c1_load_reference_barrier_runtime_stub(StubAssembler* sasm) { + __ prologue("shenandoah_load_reference_barrier", false); + // arg0 : object to be resolved + + __ save_live_registers_no_oop_map(true); + __ load_parameter(0, c_rarg0); + __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier), c_rarg0); + __ restore_live_registers_except_rax(true); + + __ epilogue(); +} + #undef __ #endif // COMPILER1 @@ -965,11 +925,11 @@ StubCodeMark mark(cgen, "StubRoutines", "shenandoah_lrb"); address start = __ pc(); - Label resolve_oop, slow_path; + Label resolve_oop, slow_path, done; // We use RDI, which also serves as argument register for slow call. - // RAX always holds the src object ptr, except after the slow call and - // the cmpxchg, then it holds the result. R8/RBX is used as temporary register. + // RAX always holds the src object ptr, except after the slow call, + // then it holds the result. R8/RBX is used as temporary register. Register tmp1 = rdi; Register tmp2 = LP64_ONLY(r8) NOT_LP64(rbx); @@ -977,39 +937,13 @@ __ push(tmp1); __ push(tmp2); - // Check for object being in the collection set. - // TODO: Can we use only 1 register here? - // The source object arrives here in rax. - // live: rax - // live: tmp1 - __ mov(tmp1, rax); - __ shrptr(tmp1, ShenandoahHeapRegion::region_size_bytes_shift_jint()); - // live: tmp2 - __ movptr(tmp2, (intptr_t) ShenandoahHeap::in_cset_fast_test_addr()); - __ movbool(tmp2, Address(tmp2, tmp1, Address::times_1)); - // unlive: tmp1 - __ testbool(tmp2); - // unlive: tmp2 - __ jccb(Assembler::notZero, resolve_oop); - - __ pop(tmp2); - __ pop(tmp1); - __ ret(0); + gen_cset_check(cgen->assembler(), rax, tmp1, tmp2, done); __ bind(resolve_oop); - __ movptr(tmp2, Address(rax, oopDesc::mark_offset_in_bytes())); - // Test if both lowest bits are set. We trick it by negating the bits - // then test for both bits clear. - __ notptr(tmp2); - __ testb(tmp2, markOopDesc::marked_value); - __ jccb(Assembler::notZero, slow_path); - // Clear both lower bits. It's still inverted, so set them, and then invert back. - __ orptr(tmp2, markOopDesc::marked_value); - __ notptr(tmp2); - // At this point, tmp2 contains the decoded forwarding pointer. - __ mov(rax, tmp2); + gen_resolved_check(cgen->assembler(), rax, tmp2, slow_path); + __ bind(done); __ pop(tmp2); __ pop(tmp1); __ ret(0); @@ -1031,10 +965,7 @@ __ push(r15); #endif - save_vector_registers(cgen->assembler()); - __ movptr(rdi, rax); - __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier), rdi); - restore_vector_registers(cgen->assembler()); + __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier), rax); #ifdef _LP64 __ pop(r15); diff --git a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.hpp b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.hpp --- a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.hpp +++ b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.hpp @@ -64,8 +64,16 @@ address generate_shenandoah_lrb(StubCodeGenerator* cgen); - void save_vector_registers(MacroAssembler* masm); - void restore_vector_registers(MacroAssembler* masm); + // Generate cset check. If obj is not in cset, branch to done label, otherwise fall through + // obj: Register holding the oop, preserved + // tmp1, tmp2: temp registers, trashed + void gen_cset_check(MacroAssembler* masm, Register obj, Register tmp1, Register tmp2, Label& done); + + // Generate check if object is resolved. Branch to resolved label, if not. Otherwise return resolved + // object in obj register. + // obj: object, resolved object on normal return + // tmp: temp register, trashed + void gen_resolved_check(MacroAssembler* masm, Register obj, Register tmp, Label& not_resolved); public: static address shenandoah_lrb(); @@ -75,6 +83,7 @@ void gen_pre_barrier_stub(LIR_Assembler* ce, ShenandoahPreBarrierStub* stub); void gen_load_reference_barrier_stub(LIR_Assembler* ce, ShenandoahLoadReferenceBarrierStub* stub); void generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm); + void generate_c1_load_reference_barrier_runtime_stub(StubAssembler* sasm); #endif void load_reference_barrier(MacroAssembler* masm, Register dst); diff --git a/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.cpp b/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.cpp --- a/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.cpp +++ b/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.cpp @@ -114,8 +114,10 @@ obj = ensure_in_register(gen, obj); assert(obj->is_register(), "must be a register at this point"); - LIR_Opr result = gen->new_register(T_OBJECT); + LIR_Opr result = gen->result_register_for(obj->value_type()); __ move(obj, result); + LIR_Opr tmp1 = gen->new_register(T_OBJECT); + LIR_Opr tmp2 = gen->new_register(T_OBJECT); LIR_Opr thrd = gen->getThreadPointer(); LIR_Address* active_flag_addr = @@ -140,7 +142,7 @@ } __ cmp(lir_cond_notEqual, flag_val, LIR_OprFact::intConst(0)); - CodeStub* slow = new ShenandoahLoadReferenceBarrierStub(obj, result); + CodeStub* slow = new ShenandoahLoadReferenceBarrierStub(obj, result, tmp1, tmp2); __ branch(lir_cond_notEqual, T_INT, slow); __ branch_destination(slow->continuation()); @@ -238,11 +240,25 @@ } }; +class C1ShenandoahLoadReferenceBarrierCodeGenClosure : public StubAssemblerCodeGenClosure { + virtual OopMapSet* generate_code(StubAssembler* sasm) { + ShenandoahBarrierSetAssembler* bs = (ShenandoahBarrierSetAssembler*)BarrierSet::barrier_set()->barrier_set_assembler(); + bs->generate_c1_load_reference_barrier_runtime_stub(sasm); + return NULL; + } +}; + void ShenandoahBarrierSetC1::generate_c1_runtime_stubs(BufferBlob* buffer_blob) { C1ShenandoahPreBarrierCodeGenClosure pre_code_gen_cl; _pre_barrier_c1_runtime_code_blob = Runtime1::generate_blob(buffer_blob, -1, "shenandoah_pre_barrier_slow", false, &pre_code_gen_cl); + if (ShenandoahLoadRefBarrier) { + C1ShenandoahLoadReferenceBarrierCodeGenClosure lrb_code_gen_cl; + _load_reference_barrier_rt_code_blob = Runtime1::generate_blob(buffer_blob, -1, + "shenandoah_load_reference_barrier_slow", + false, &lrb_code_gen_cl); + } } const char* ShenandoahBarrierSetC1::rtcall_name_for_address(address entry) { diff --git a/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.hpp b/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.hpp --- a/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.hpp +++ b/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.hpp @@ -90,23 +90,31 @@ private: LIR_Opr _obj; LIR_Opr _result; + LIR_Opr _tmp1; + LIR_Opr _tmp2; public: - ShenandoahLoadReferenceBarrierStub(LIR_Opr obj, LIR_Opr result) : - _obj(obj), _result(result) + ShenandoahLoadReferenceBarrierStub(LIR_Opr obj, LIR_Opr result, LIR_Opr tmp1, LIR_Opr tmp2) : + _obj(obj), _result(result), _tmp1(tmp1), _tmp2(tmp2) { assert(_obj->is_register(), "should be register"); assert(_result->is_register(), "should be register"); + assert(_tmp1->is_register(), "should be register"); + assert(_tmp2->is_register(), "should be register"); } LIR_Opr obj() const { return _obj; } LIR_Opr result() const { return _result; } + LIR_Opr tmp1() const { return _tmp1; } + LIR_Opr tmp2() const { return _tmp2; } virtual void emit_code(LIR_Assembler* e); virtual void visit(LIR_OpVisitState* visitor) { visitor->do_slow_case(); visitor->do_input(_obj); visitor->do_temp(_result); + visitor->do_temp(_tmp1); + visitor->do_temp(_tmp2); } #ifndef PRODUCT virtual void print_name(outputStream* out) const { out->print("ShenandoahLoadReferenceBarrierStub"); } @@ -174,6 +182,7 @@ class ShenandoahBarrierSetC1 : public BarrierSetC1 { private: CodeBlob* _pre_barrier_c1_runtime_code_blob; + CodeBlob* _load_reference_barrier_rt_code_blob; void pre_barrier(LIRGenerator* gen, CodeEmitInfo* info, DecoratorSet decorators, LIR_Opr addr_opr, LIR_Opr pre_val); @@ -186,6 +195,7 @@ public: CodeBlob* pre_barrier_c1_runtime_code_blob() { return _pre_barrier_c1_runtime_code_blob; } + CodeBlob* load_reference_barrier_rt_code_blob() { return _load_reference_barrier_rt_code_blob; } protected: