# HG changeset patch # User rkennke # Date 1568832979 -7200 # Wed Sep 18 20:56:19 2019 +0200 # Node ID 69289957896337aa20ff581df9f60efd7536b178 # Parent 786d56b6de04310c2775c80f06affe4b43e187f8 [backport] 8231086: Shenandoah: Stronger invariant for object-arraycopy Reviewed-by: shade diff -r 786d56b6de04 src/cpu/aarch64/vm/shenandoahBarrierSetAssembler_aarch64.cpp --- a/src/cpu/aarch64/vm/shenandoahBarrierSetAssembler_aarch64.cpp Mon Feb 10 17:04:04 2020 +0100 +++ b/src/cpu/aarch64/vm/shenandoahBarrierSetAssembler_aarch64.cpp Thu Feb 20 16:57:24 2020 +0100 @@ -40,6 +40,45 @@ #define __ masm-> +void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, bool dest_uninitialized, + Register src, Register dst, Register count) { + if ((ShenandoahSATBBarrier && !dest_uninitialized) || ShenandoahLoadRefBarrier) { + + Label done; + + // Avoid calling runtime if count == 0 + __ cbz(count, done); + + // Is marking active? + Address gc_state(rthread, in_bytes(JavaThread::gc_state_offset())); + __ ldrb(rscratch1, gc_state); + if (dest_uninitialized) { + __ tbz(rscratch2, ShenandoahHeap::HAS_FORWARDED_BITPOS, done); + } else { + __ mov(rscratch2, ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::MARKING); + __ tst(rscratch1, rscratch2); + __ br(Assembler::EQ, done); + } + + __ push_call_clobbered_registers(); + if (UseCompressedOops) { + if (dest_uninitialized) { + __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_duinit_narrow_oop_entry), src, dst, count); + } else { + __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_narrow_oop_entry), src, dst, count); + } + } else { + if (dest_uninitialized) { + __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_duinit_oop_entry), src, dst, count); + } else { + __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_oop_entry), src, dst, count); + } + } + __ pop_call_clobbered_registers(); + __ bind(done); + } +} + void ShenandoahBarrierSetAssembler::resolve_forward_pointer(MacroAssembler* masm, Register dst, Register tmp) { assert(ShenandoahCASBarrier, "should be enabled"); Label is_null; diff -r 786d56b6de04 src/cpu/aarch64/vm/shenandoahBarrierSetAssembler_aarch64.hpp --- a/src/cpu/aarch64/vm/shenandoahBarrierSetAssembler_aarch64.hpp Mon Feb 10 17:04:04 2020 +0100 +++ b/src/cpu/aarch64/vm/shenandoahBarrierSetAssembler_aarch64.hpp Thu Feb 20 16:57:24 2020 +0100 @@ -52,6 +52,8 @@ void load_reference_barrier(MacroAssembler* masm, Register dst); + virtual void arraycopy_prologue(MacroAssembler* masm, bool dest_uninitialized, + Register src, Register dst, Register count); virtual void cmpxchg_oop(MacroAssembler* masm, Register addr, Register expected, Register new_val, bool acquire, bool release, bool weak, bool is_cae, diff -r 786d56b6de04 src/cpu/aarch64/vm/stubGenerator_aarch64.cpp --- a/src/cpu/aarch64/vm/stubGenerator_aarch64.cpp Mon Feb 10 17:04:04 2020 +0100 +++ b/src/cpu/aarch64/vm/stubGenerator_aarch64.cpp Thu Feb 20 16:57:24 2020 +0100 @@ -47,6 +47,9 @@ #ifdef COMPILER2 #include "opto/runtime.hpp" #endif +#if INCLUDE_ALL_GCS +#include "shenandoahBarrierSetAssembler_aarch64.hpp" +#endif // Declaration and definition of StubGenerator (no .hpp file). // For a more detailed description of the stub routine structure @@ -600,12 +603,11 @@ // // Destroy no registers except rscratch1 and rscratch2 // - void gen_write_ref_array_pre_barrier(Register addr, Register count, bool dest_uninitialized) { + void gen_write_ref_array_pre_barrier(Register src, Register addr, Register count, bool dest_uninitialized) { BarrierSet* bs = Universe::heap()->barrier_set(); switch (bs->kind()) { case BarrierSet::G1SATBCT: case BarrierSet::G1SATBCTLogging: - case BarrierSet::ShenandoahBarrierSet: // Don't generate the call if we statically know that the target is uninitialized if (!dest_uninitialized) { __ push_call_clobbered_registers(); @@ -630,6 +632,9 @@ case BarrierSet::CardTableExtension: case BarrierSet::ModRef: break; + case BarrierSet::ShenandoahBarrierSet: + ShenandoahBarrierSetAssembler::bsasm()->arraycopy_prologue(_masm, dest_uninitialized, src, addr, count); + break; default: ShouldNotReachHere(); @@ -653,8 +658,7 @@ switch (bs->kind()) { case BarrierSet::G1SATBCT: case BarrierSet::G1SATBCTLogging: - case BarrierSet::ShenandoahBarrierSet: - + { __ push_call_clobbered_registers(); // must compute element count unless barrier set interface is changed (other platforms supply count) @@ -692,6 +696,8 @@ __ br(Assembler::GE, L_loop); } break; + case BarrierSet::ShenandoahBarrierSet: + break; default: ShouldNotReachHere(); @@ -1408,7 +1414,7 @@ if (is_oop) { __ push(RegSet::of(d, count), sp); // no registers are destroyed by this call - gen_write_ref_array_pre_barrier(d, count, dest_uninitialized); + gen_write_ref_array_pre_barrier(s, d, count, dest_uninitialized); } copy_memory(aligned, s, d, count, rscratch1, size); if (is_oop) { @@ -1464,7 +1470,7 @@ if (is_oop) { __ push(RegSet::of(d, count), sp); // no registers are destroyed by this call - gen_write_ref_array_pre_barrier(d, count, dest_uninitialized); + gen_write_ref_array_pre_barrier(s, d, count, dest_uninitialized); } copy_memory(aligned, s, d, count, rscratch1, -size); if (is_oop) { @@ -1806,7 +1812,7 @@ } #endif //ASSERT - gen_write_ref_array_pre_barrier(to, count, dest_uninitialized); + gen_write_ref_array_pre_barrier(from, to, count, dest_uninitialized); // save the original count __ mov(count_save, count); diff -r 786d56b6de04 src/cpu/x86/vm/shenandoahBarrierSetAssembler_x86.cpp --- a/src/cpu/x86/vm/shenandoahBarrierSetAssembler_x86.cpp Mon Feb 10 17:04:04 2020 +0100 +++ b/src/cpu/x86/vm/shenandoahBarrierSetAssembler_x86.cpp Thu Feb 20 16:57:24 2020 +0100 @@ -40,6 +40,70 @@ #define __ masm-> +void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, bool dest_uninitialized, + Register src, Register dst, Register count) { + + if ((ShenandoahSATBBarrier && !dest_uninitialized) || ShenandoahLoadRefBarrier) { +#ifdef _LP64 + Register thread = r15_thread; +#else + Register thread = rax; + if (thread == src || thread == dst || thread == count) { + thread = rbx; + } + if (thread == src || thread == dst || thread == count) { + thread = rcx; + } + if (thread == src || thread == dst || thread == count) { + thread = rdx; + } + __ push(thread); + __ get_thread(thread); +#endif + assert_different_registers(src, dst, count, thread); + + Label done; + // Short-circuit if count == 0. + __ testptr(count, count); + __ jcc(Assembler::zero, done); + + // Avoid runtime call when not marking. + Address gc_state(thread, in_bytes(JavaThread::gc_state_offset())); + int flags = ShenandoahHeap::HAS_FORWARDED; + if (!dest_uninitialized) { + flags |= ShenandoahHeap::MARKING; + } + __ testb(gc_state, flags); + __ jcc(Assembler::zero, done); + + __ pusha(); // push registers +#ifdef _LP64 + assert(src == rdi, "expected"); + assert(dst == rsi, "expected"); + // commented-out for generate_conjoint_long_oop_copy(), call_VM_leaf() will move + // register into right place. + // assert(count == rdx, "expected"); + if (UseCompressedOops) { + if (dest_uninitialized) { + __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_duinit_narrow_oop_entry), src, dst, count); + } else { + __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_narrow_oop_entry), src, dst, count); + } + } else +#endif + { + if (dest_uninitialized) { + __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_duinit_oop_entry), src, dst, count); + } else { + __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_oop_entry), src, dst, count); + } + } + __ popa(); + __ bind(done); + NOT_LP64(__ pop(thread);) + } +} + void ShenandoahBarrierSetAssembler::resolve_forward_pointer(MacroAssembler* masm, Register dst, Register tmp) { assert(ShenandoahCASBarrier, "should be enabled"); Label is_null; diff -r 786d56b6de04 src/cpu/x86/vm/shenandoahBarrierSetAssembler_x86.hpp --- a/src/cpu/x86/vm/shenandoahBarrierSetAssembler_x86.hpp Mon Feb 10 17:04:04 2020 +0100 +++ b/src/cpu/x86/vm/shenandoahBarrierSetAssembler_x86.hpp Thu Feb 20 16:57:24 2020 +0100 @@ -53,6 +53,8 @@ void load_reference_barrier(MacroAssembler* masm, Register dst); + virtual void arraycopy_prologue(MacroAssembler* masm, bool dest_uninitialized, + Register src, Register dst, Register count); virtual void cmpxchg_oop(MacroAssembler* masm, Register res, Address addr, Register oldval, Register newval, bool exchange, Register tmp1, Register tmp2); diff -r 786d56b6de04 src/cpu/x86/vm/stubGenerator_x86_32.cpp --- a/src/cpu/x86/vm/stubGenerator_x86_32.cpp Mon Feb 10 17:04:04 2020 +0100 +++ b/src/cpu/x86/vm/stubGenerator_x86_32.cpp Thu Feb 20 16:57:24 2020 +0100 @@ -43,6 +43,9 @@ #ifdef COMPILER2 #include "opto/runtime.hpp" #endif +#if INCLUDE_ALL_GCS +#include "shenandoahBarrierSetAssembler_x86.hpp" +#endif // Declaration and definition of StubGenerator (no .hpp file). // For a more detailed description of the stub routine structure @@ -703,13 +706,12 @@ // Input: // start - starting address // count - element count - void gen_write_ref_array_pre_barrier(Register start, Register count, bool uninitialized_target) { + void gen_write_ref_array_pre_barrier(Register src, Register start, Register count, bool uninitialized_target) { assert_different_registers(start, count); BarrierSet* bs = Universe::heap()->barrier_set(); switch (bs->kind()) { case BarrierSet::G1SATBCT: case BarrierSet::G1SATBCTLogging: - case BarrierSet::ShenandoahBarrierSet: // With G1, don't generate the call if we statically know that the target in uninitialized if (!uninitialized_target) { __ pusha(); // push registers @@ -722,6 +724,9 @@ case BarrierSet::CardTableExtension: case BarrierSet::ModRef: break; + case BarrierSet::ShenandoahBarrierSet: + ShenandoahBarrierSetAssembler::bsasm()->arraycopy_prologue(_masm, uninitialized_target, src, start, count); + break; default : ShouldNotReachHere(); @@ -743,7 +748,6 @@ switch (bs->kind()) { case BarrierSet::G1SATBCT: case BarrierSet::G1SATBCTLogging: - case BarrierSet::ShenandoahBarrierSet: { __ pusha(); // push registers __ call_VM_leaf(CAST_FROM_FN_PTR(address, BarrierSet::static_write_ref_array_post), @@ -775,6 +779,7 @@ } break; case BarrierSet::ModRef: + case BarrierSet::ShenandoahBarrierSet: break; default : ShouldNotReachHere(); @@ -940,7 +945,7 @@ if (t == T_OBJECT) { __ testl(count, count); __ jcc(Assembler::zero, L_0_count); - gen_write_ref_array_pre_barrier(to, count, dest_uninitialized); + gen_write_ref_array_pre_barrier(from, to, count, dest_uninitialized); __ mov(saved_to, to); // save 'to' } @@ -1119,7 +1124,7 @@ if (t == T_OBJECT) { __ testl(count, count); __ jcc(Assembler::zero, L_0_count); - gen_write_ref_array_pre_barrier(dst, count, dest_uninitialized); + gen_write_ref_array_pre_barrier(src, dst, count, dest_uninitialized); } // copy from high to low @@ -1466,7 +1471,7 @@ Address elem_klass_addr(elem, oopDesc::klass_offset_in_bytes()); // Copy from low to high addresses, indexed from the end of each array. - gen_write_ref_array_pre_barrier(to, count, dest_uninitialized); + gen_write_ref_array_pre_barrier(from, to, count, dest_uninitialized); __ lea(end_from, end_from_addr); __ lea(end_to, end_to_addr); assert(length == count, ""); // else fix next line: diff -r 786d56b6de04 src/cpu/x86/vm/stubGenerator_x86_64.cpp --- a/src/cpu/x86/vm/stubGenerator_x86_64.cpp Mon Feb 10 17:04:04 2020 +0100 +++ b/src/cpu/x86/vm/stubGenerator_x86_64.cpp Thu Feb 20 16:57:24 2020 +0100 @@ -1181,12 +1181,11 @@ // // Destroy no registers! // - void gen_write_ref_array_pre_barrier(Register addr, Register count, bool dest_uninitialized) { + void gen_write_ref_array_pre_barrier(Register src, Register addr, Register count, bool dest_uninitialized) { BarrierSet* bs = Universe::heap()->barrier_set(); switch (bs->kind()) { case BarrierSet::G1SATBCT: case BarrierSet::G1SATBCTLogging: - case BarrierSet::ShenandoahBarrierSet: // With G1, don't generate the call if we statically know that the target in uninitialized if (!dest_uninitialized) { __ pusha(); // push registers @@ -1210,6 +1209,9 @@ case BarrierSet::CardTableExtension: case BarrierSet::ModRef: break; + case BarrierSet::ShenandoahBarrierSet: + ShenandoahBarrierSetAssembler::bsasm()->arraycopy_prologue(_masm, dest_uninitialized, src, addr, count); + break; default: ShouldNotReachHere(); @@ -1232,7 +1234,6 @@ switch (bs->kind()) { case BarrierSet::G1SATBCT: case BarrierSet::G1SATBCTLogging: - case BarrierSet::ShenandoahBarrierSet: { __ pusha(); // push registers (overkill) if (c_rarg0 == count) { // On win64 c_rarg0 == rcx @@ -1272,6 +1273,8 @@ __ jcc(Assembler::greaterEqual, L_loop); } break; + case BarrierSet::ShenandoahBarrierSet: + break; default: ShouldNotReachHere(); @@ -1889,7 +1892,7 @@ // r9 and r10 may be used to save non-volatile registers if (is_oop) { __ movq(saved_to, to); - gen_write_ref_array_pre_barrier(to, count, dest_uninitialized); + gen_write_ref_array_pre_barrier(from, to, count, dest_uninitialized); } // 'from', 'to' and 'count' are now valid @@ -1977,7 +1980,7 @@ if (is_oop) { // no registers are destroyed by this call - gen_write_ref_array_pre_barrier(to, count, dest_uninitialized); + gen_write_ref_array_pre_barrier(from, to, count, dest_uninitialized); } assert_clean_int(count, rax); // Make sure 'count' is clean int. @@ -2075,7 +2078,7 @@ // Save to and count for store barrier __ movptr(saved_count, qword_count); // no registers are destroyed by this call - gen_write_ref_array_pre_barrier(to, qword_count, dest_uninitialized); + gen_write_ref_array_pre_barrier(from, to, qword_count, dest_uninitialized); } // Copy from low to high addresses. Use 'to' as scratch. @@ -2162,7 +2165,7 @@ // Save to and count for store barrier __ movptr(saved_count, qword_count); // No registers are destroyed by this call - gen_write_ref_array_pre_barrier(to, saved_count, dest_uninitialized); + gen_write_ref_array_pre_barrier(from, to, saved_count, dest_uninitialized); } __ jmp(L_copy_bytes); @@ -2336,7 +2339,7 @@ Address from_element_addr(end_from, count, TIMES_OOP, 0); Address to_element_addr(end_to, count, TIMES_OOP, 0); - gen_write_ref_array_pre_barrier(to, count, dest_uninitialized); + gen_write_ref_array_pre_barrier(from, to, count, dest_uninitialized); // Copy from low to high addresses, indexed from the end of each array. __ lea(end_from, end_from_addr); diff -r 786d56b6de04 src/share/vm/c1/c1_Runtime1.cpp --- a/src/share/vm/c1/c1_Runtime1.cpp Mon Feb 10 17:04:04 2020 +0100 +++ b/src/share/vm/c1/c1_Runtime1.cpp Thu Feb 20 16:57:24 2020 +0100 @@ -57,7 +57,9 @@ #include "runtime/vframeArray.hpp" #include "utilities/copy.hpp" #include "utilities/events.hpp" - +#ifdef INCLUDE_ALL_GCS +#include "gc_implementation/shenandoah/shenandoahBarrierSet.inline.hpp" +#endif // Implementation of StubAssembler @@ -1307,6 +1309,13 @@ BarrierSet* bs = Universe::heap()->barrier_set(); assert(bs->has_write_ref_array_opt(), "Barrier set must have ref array opt"); assert(bs->has_write_ref_array_pre_opt(), "For pre-barrier as well."); + +#ifdef INCLUDE_ALL_GCS + if (UseShenandoahGC) { + ShenandoahBarrierSet::barrier_set()->arraycopy_pre(src_addr, dst_addr, length); + } +#endif + if (src == dst) { // same object, no check bs->write_ref_array_pre(dst_addr, length); diff -r 786d56b6de04 src/share/vm/gc_implementation/shenandoah/shenandoahBarrierSet.cpp --- a/src/share/vm/gc_implementation/shenandoah/shenandoahBarrierSet.cpp Mon Feb 10 17:04:04 2020 +0100 +++ b/src/share/vm/gc_implementation/shenandoah/shenandoahBarrierSet.cpp Thu Feb 20 16:57:24 2020 +0100 @@ -46,33 +46,6 @@ #include "shenandoahBarrierSetAssembler_stub.hpp" #endif -template -class ShenandoahUpdateRefsForOopClosure: public ExtendedOopClosure { -private: - ShenandoahHeap* _heap; - ShenandoahBarrierSet* _bs; - - template - inline void do_oop_work(T* p) { - oop o; - if (STOREVAL_EVAC_BARRIER) { - o = _heap->evac_update_with_forwarded(p); - if (!oopDesc::is_null(o)) { - _bs->enqueue(o); - } - } else { - _heap->maybe_update_with_forwarded(p); - } - } -public: - ShenandoahUpdateRefsForOopClosure() : _heap(ShenandoahHeap::heap()), _bs(ShenandoahBarrierSet::barrier_set()) { - assert(UseShenandoahGC && ShenandoahCloneBarrier, "should be enabled"); - } - - virtual void do_oop(oop* p) { do_oop_work(p); } - virtual void do_oop(narrowOop* p) { do_oop_work(p); } -}; - ShenandoahBarrierSet::ShenandoahBarrierSet(ShenandoahHeap* heap) : BarrierSet(), _heap(heap), @@ -160,65 +133,6 @@ // return *v; } -template -void ShenandoahBarrierSet::write_ref_array_loop(HeapWord* start, size_t count) { - assert(UseShenandoahGC && ShenandoahCloneBarrier, "should be enabled"); - ShenandoahUpdateRefsForOopClosure cl; - T* dst = (T*) start; - for (size_t i = 0; i < count; i++) { - cl.do_oop(dst++); - } -} - -void ShenandoahBarrierSet::write_ref_array(HeapWord* start, size_t count) { - assert(UseShenandoahGC, "should be enabled"); - if (!ShenandoahCloneBarrier) return; - if (!need_update_refs_barrier()) return; - - if (_heap->is_concurrent_traversal_in_progress()) { - ShenandoahEvacOOMScope oom_evac_scope; - if (UseCompressedOops) { - write_ref_array_loop(start, count); - } else { - write_ref_array_loop(start, count); - } - } else { - if (UseCompressedOops) { - write_ref_array_loop(start, count); - } else { - write_ref_array_loop(start, count); - } - } -} - -template -void ShenandoahBarrierSet::write_ref_array_pre_work(T* dst, size_t count) { - assert (UseShenandoahGC && ShenandoahSATBBarrier, "Should be enabled"); - - shenandoah_assert_not_in_cset_loc_except(dst, _heap->cancelled_gc()); - - if (! JavaThread::satb_mark_queue_set().is_active()) return; - T* elem_ptr = dst; - for (size_t i = 0; i < count; i++, elem_ptr++) { - T heap_oop = oopDesc::load_heap_oop(elem_ptr); - if (!oopDesc::is_null(heap_oop)) { - enqueue(oopDesc::decode_heap_oop_not_null(heap_oop)); - } - } -} - -void ShenandoahBarrierSet::write_ref_array_pre(oop* dst, int count, bool dest_uninitialized) { - if (! dest_uninitialized && ShenandoahSATBBarrier) { - write_ref_array_pre_work(dst, (size_t)count); - } -} - -void ShenandoahBarrierSet::write_ref_array_pre(narrowOop* dst, int count, bool dest_uninitialized) { - if (! dest_uninitialized && ShenandoahSATBBarrier) { - write_ref_array_pre_work(dst, (size_t)count); - } -} - template inline void ShenandoahBarrierSet::inline_write_ref_field_pre(T* field, oop newVal) { newVal = load_reference_barrier(newVal); @@ -247,28 +161,6 @@ shenandoah_assert_not_in_cset_except (v, o, o == NULL || _heap->cancelled_gc() || !_heap->is_concurrent_mark_in_progress()); } -void ShenandoahBarrierSet::write_region_work(MemRegion mr) { - assert(UseShenandoahGC, "should be enabled"); - if (!ShenandoahCloneBarrier) return; - if (! need_update_refs_barrier()) return; - - // This is called for cloning an object (see jvm.cpp) after the clone - // has been made. We are not interested in any 'previous value' because - // it would be NULL in any case. But we *are* interested in any oop* - // that potentially need to be updated. - - oop obj = oop(mr.start()); - shenandoah_assert_correct(NULL, obj); - if (_heap->is_concurrent_traversal_in_progress()) { - ShenandoahEvacOOMScope oom_evac_scope; - ShenandoahUpdateRefsForOopClosure cl; - obj->oop_iterate(&cl); - } else { - ShenandoahUpdateRefsForOopClosure cl; - obj->oop_iterate(&cl); - } -} - oop ShenandoahBarrierSet::load_reference_barrier_not_null(oop obj) { assert(obj != NULL, ""); if (ShenandoahLoadRefBarrier && _heap->has_forwarded_objects()) { diff -r 786d56b6de04 src/share/vm/gc_implementation/shenandoah/shenandoahBarrierSet.hpp --- a/src/share/vm/gc_implementation/shenandoah/shenandoahBarrierSet.hpp Mon Feb 10 17:04:04 2020 +0100 +++ b/src/share/vm/gc_implementation/shenandoah/shenandoahBarrierSet.hpp Thu Feb 20 16:57:24 2020 +0100 @@ -81,15 +81,17 @@ void write_prim_array(MemRegion mr) shenandoah_not_implemented; void write_prim_field(HeapWord* hw, size_t s , juint x, juint y) shenandoah_not_implemented; bool write_prim_needs_barrier(HeapWord* hw, size_t s, juint x, juint y) shenandoah_not_implemented_return(false); - void write_ref_array(HeapWord* start, size_t count); - void write_ref_array_work(MemRegion r) shenandoah_not_implemented; + + void write_ref_array_work(MemRegion mr) {} template void - write_ref_array_pre_work(T* dst, size_t count); + write_ref_array_pre_work(T* src, T* dst, size_t count, bool dest_uninitialized); - void write_ref_array_pre(oop* dst, int count, bool dest_uninitialized); - - void write_ref_array_pre(narrowOop* dst, int count, bool dest_uninitialized); + inline void arraycopy_pre(oop* src, oop* dst, size_t count); + inline void arraycopy_pre(narrowOop* src, narrowOop* dst, size_t count); + inline void arraycopy_update(oop* src, size_t count); + inline void arraycopy_update(narrowOop* src, size_t count); + inline void clone_barrier(oop src); // We export this to make it available in cases where the static // type of the barrier set is known. Note that it is non-virtual. @@ -101,7 +103,7 @@ void write_ref_field_pre_work(void* field, oop new_val) shenandoah_not_implemented; void write_ref_field_work(void* v, oop o, bool release = false); - void write_region_work(MemRegion mr); + void write_region_work(MemRegion mr) {}; static inline oop resolve_forwarded_not_null(oop p); static inline oop resolve_forwarded(oop p); @@ -120,8 +122,12 @@ private: inline bool need_update_refs_barrier(); - template - void write_ref_array_loop(HeapWord* start, size_t count); + template + inline void arraycopy_pre_work(T* src, T* dst, size_t count); + template + inline void arraycopy_work(T* src, size_t count); + template + inline void arraycopy_update_impl(T* src, size_t count); oop load_reference_barrier_impl(oop obj); diff -r 786d56b6de04 src/share/vm/gc_implementation/shenandoah/shenandoahBarrierSet.inline.hpp --- a/src/share/vm/gc_implementation/shenandoah/shenandoahBarrierSet.inline.hpp Mon Feb 10 17:04:04 2020 +0100 +++ b/src/share/vm/gc_implementation/shenandoah/shenandoahBarrierSet.inline.hpp Thu Feb 20 16:57:24 2020 +0100 @@ -25,8 +25,12 @@ #define SHARE_VM_GC_SHENANDOAH_SHENANDOAHBARRIERSET_INLINE_HPP #include "gc_implementation/shenandoah/shenandoahBarrierSet.hpp" +#include "gc_implementation/shenandoah/shenandoahCollectionSet.inline.hpp" #include "gc_implementation/shenandoah/shenandoahForwarding.inline.hpp" #include "gc_implementation/shenandoah/shenandoahHeap.inline.hpp" +#include "gc_implementation/shenandoah/shenandoahMarkingContext.inline.hpp" +#include "memory/iterator.inline.hpp" +#include "oops/oop.inline.hpp" bool ShenandoahBarrierSet::need_update_refs_barrier() { return _heap->is_update_refs_in_progress() || @@ -45,4 +49,137 @@ } } +template +void ShenandoahBarrierSet::arraycopy_work(T* src, size_t count) { + JavaThread* thread = JavaThread::current(); + ObjPtrQueue& queue = thread->satb_mark_queue(); + ShenandoahMarkingContext* ctx = _heap->marking_context(); + const ShenandoahCollectionSet* const cset = _heap->collection_set(); + T* end = src + count; + for (T* elem_ptr = src; elem_ptr < end; elem_ptr++) { + T o = oopDesc::load_heap_oop(elem_ptr); + if (!oopDesc::is_null(o)) { + oop obj = oopDesc::decode_heap_oop_not_null(o); + if (HAS_FWD && cset->is_in((HeapWord *) obj)) { + assert(_heap->has_forwarded_objects(), "only get here with forwarded objects"); + oop fwd = resolve_forwarded_not_null(obj); + if (EVAC && obj == fwd) { + fwd = _heap->evacuate_object(obj, thread); + } + assert(obj != fwd || _heap->cancelled_gc(), "must be forwarded"); + oop witness = ShenandoahHeap::cas_oop(fwd, elem_ptr, o); + obj = fwd; + } + if (ENQUEUE && !ctx->is_marked(obj)) { + queue.enqueue_known_active(obj); + } + } + } +} + +template +void ShenandoahBarrierSet::arraycopy_pre_work(T* src, T* dst, size_t count) { + if (_heap->is_concurrent_mark_in_progress()) { + if (_heap->has_forwarded_objects()) { + arraycopy_work(dst, count); + } else { + arraycopy_work(dst, count); + } + } + + arraycopy_update_impl(src, count); +} + +void ShenandoahBarrierSet::arraycopy_pre(oop* src, oop* dst, size_t count) { + arraycopy_pre_work(src, dst, count); +} + +void ShenandoahBarrierSet::arraycopy_pre(narrowOop* src, narrowOop* dst, size_t count) { + arraycopy_pre_work(src, dst, count); +} + +template +void ShenandoahBarrierSet::arraycopy_update_impl(T* src, size_t count) { + if (_heap->is_evacuation_in_progress()) { + ShenandoahEvacOOMScope oom_evac; + arraycopy_work(src, count); + } else if (_heap->is_concurrent_traversal_in_progress()){ + ShenandoahEvacOOMScope oom_evac; + arraycopy_work(src, count); + } else if (_heap->has_forwarded_objects()) { + arraycopy_work(src, count); + } +} + +void ShenandoahBarrierSet::arraycopy_update(oop* src, size_t count) { + arraycopy_update_impl(src, count); +} + +void ShenandoahBarrierSet::arraycopy_update(narrowOop* src, size_t count) { + arraycopy_update_impl(src, count); +} + +template +class ShenandoahUpdateRefsForOopClosure: public ExtendedOopClosure { +private: + ShenandoahHeap* const _heap; + ShenandoahBarrierSet* const _bs; + const ShenandoahCollectionSet* const _cset; + Thread* const _thread; + + template + inline void do_oop_work(T* p) { + T o = oopDesc::load_heap_oop(p); + if (!oopDesc::is_null(o)) { + oop obj = oopDesc::decode_heap_oop_not_null(o); + if (_cset->is_in((HeapWord *)obj)) { + oop fwd = _bs->resolve_forwarded_not_null(obj); + if (EVAC && obj == fwd) { + fwd = _heap->evacuate_object(obj, _thread); + } + if (ENQUEUE) { + _bs->enqueue(fwd); + } + assert(obj != fwd || _heap->cancelled_gc(), "must be forwarded"); + ShenandoahHeap::cas_oop(fwd, p, o); + } + + } + } +public: + ShenandoahUpdateRefsForOopClosure() : + _heap(ShenandoahHeap::heap()), + _bs(ShenandoahBarrierSet::barrier_set()), + _cset(_heap->collection_set()), + _thread(Thread::current()) { + } + + virtual void do_oop(oop* p) { do_oop_work(p); } + virtual void do_oop(narrowOop* p) { do_oop_work(p); } +}; + +void ShenandoahBarrierSet::clone_barrier(oop obj) { + assert(ShenandoahCloneBarrier, "only get here with clone barriers enabled"); + if (!_heap->has_forwarded_objects()) return; + + // This is called for cloning an object (see jvm.cpp) after the clone + // has been made. We are not interested in any 'previous value' because + // it would be NULL in any case. But we *are* interested in any oop* + // that potentially need to be updated. + + shenandoah_assert_correct(NULL, obj); + if (_heap->is_evacuation_in_progress()) { + ShenandoahEvacOOMScope evac_scope; + ShenandoahUpdateRefsForOopClosure cl; + obj->oop_iterate(&cl); + } else if (_heap->is_concurrent_traversal_in_progress()) { + ShenandoahEvacOOMScope evac_scope; + ShenandoahUpdateRefsForOopClosure cl; + obj->oop_iterate(&cl); + } else { + ShenandoahUpdateRefsForOopClosure cl; + obj->oop_iterate(&cl); + } +} + #endif //SHARE_VM_GC_SHENANDOAH_SHENANDOAHBARRIERSET_INLINE_HPP diff -r 786d56b6de04 src/share/vm/gc_implementation/shenandoah/shenandoahHeap.hpp --- a/src/share/vm/gc_implementation/shenandoah/shenandoahHeap.hpp Mon Feb 10 17:04:04 2020 +0100 +++ b/src/share/vm/gc_implementation/shenandoah/shenandoahHeap.hpp Thu Feb 20 16:57:24 2020 +0100 @@ -674,6 +674,7 @@ static inline oop cas_oop(oop n, narrowOop* addr, oop c); static inline oop cas_oop(oop n, oop* addr, oop c); + static inline oop cas_oop(oop n, narrowOop* addr, narrowOop c); void trash_humongous_region_at(ShenandoahHeapRegion *r); diff -r 786d56b6de04 src/share/vm/gc_implementation/shenandoah/shenandoahHeap.inline.hpp --- a/src/share/vm/gc_implementation/shenandoah/shenandoahHeap.inline.hpp Mon Feb 10 17:04:04 2020 +0100 +++ b/src/share/vm/gc_implementation/shenandoah/shenandoahHeap.inline.hpp Thu Feb 20 16:57:24 2020 +0100 @@ -126,6 +126,11 @@ return (oop) Atomic::cmpxchg_ptr(n, addr, c); } +inline oop ShenandoahHeap::cas_oop(oop n, narrowOop* addr, narrowOop c) { + narrowOop val = oopDesc::encode_heap_oop(n); + return oopDesc::decode_heap_oop((narrowOop) Atomic::cmpxchg(val, addr, c)); +} + inline oop ShenandoahHeap::cas_oop(oop n, narrowOop* addr, oop c) { assert(is_ptr_aligned(addr, sizeof(narrowOop)), err_msg("Address should be aligned: " PTR_FORMAT, p2i(addr))); narrowOop cmp = oopDesc::encode_heap_oop(c); diff -r 786d56b6de04 src/share/vm/gc_implementation/shenandoah/shenandoahRuntime.cpp --- a/src/share/vm/gc_implementation/shenandoah/shenandoahRuntime.cpp Mon Feb 10 17:04:04 2020 +0100 +++ b/src/share/vm/gc_implementation/shenandoah/shenandoahRuntime.cpp Thu Feb 20 16:57:24 2020 +0100 @@ -22,26 +22,29 @@ */ #include "precompiled.hpp" -#include "gc_implementation/shenandoah/shenandoahBarrierSet.hpp" +#include "gc_implementation/shenandoah/shenandoahBarrierSet.inline.hpp" #include "gc_implementation/shenandoah/shenandoahRuntime.hpp" #include "runtime/interfaceSupport.hpp" #include "oops/oop.inline.hpp" -void ShenandoahRuntime::write_ref_array_pre_oop_entry(oop* dst, size_t length) { +void ShenandoahRuntime::write_ref_array_pre_oop_entry(oop* src, oop* dst, size_t length) { ShenandoahBarrierSet *bs = ShenandoahBarrierSet::barrier_set(); - assert(length <= INT_MAX, err_msg("sanity: " SIZE_FORMAT, length)); - bs->write_ref_array_pre(dst, (int)length, false); + bs->arraycopy_pre(src, dst, length); } -void ShenandoahRuntime::write_ref_array_pre_narrow_oop_entry(narrowOop* dst, size_t length) { +void ShenandoahRuntime::write_ref_array_pre_narrow_oop_entry(narrowOop* src, narrowOop* dst, size_t length) { ShenandoahBarrierSet *bs = ShenandoahBarrierSet::barrier_set(); - assert(length <= INT_MAX, err_msg("sanity: " SIZE_FORMAT, length)); - bs->write_ref_array_pre(dst, (int)length, false); + bs->arraycopy_pre(src, dst, length); } -void ShenandoahRuntime::write_ref_array_post_entry(HeapWord* dst, size_t length) { +void ShenandoahRuntime::write_ref_array_pre_duinit_oop_entry(oop* src, oop* dst, size_t length) { ShenandoahBarrierSet *bs = ShenandoahBarrierSet::barrier_set(); - bs->ShenandoahBarrierSet::write_ref_array(dst, length); + bs->arraycopy_update(src, length); +} + +void ShenandoahRuntime::write_ref_array_pre_duinit_narrow_oop_entry(narrowOop* src, narrowOop* dst, size_t length) { + ShenandoahBarrierSet *bs = ShenandoahBarrierSet::barrier_set(); + bs->arraycopy_update(src, length); } // Shenandoah pre write barrier slowpath @@ -68,6 +71,10 @@ // Shenandoah clone barrier: makes sure that references point to to-space // in cloned objects. -JRT_LEAF(void, ShenandoahRuntime::shenandoah_clone_barrier(oopDesc* obj)) - ShenandoahBarrierSet::barrier_set()->write_region(MemRegion((HeapWord*) obj, obj->size())); +JRT_LEAF(void, ShenandoahRuntime::shenandoah_clone_barrier(oopDesc* s, oopDesc* d, size_t length)) + oop src = oop(s); + oop dst = oop(d); + shenandoah_assert_correct(NULL, src); + shenandoah_assert_correct(NULL, dst); + ShenandoahBarrierSet::barrier_set()->clone_barrier(src); JRT_END diff -r 786d56b6de04 src/share/vm/gc_implementation/shenandoah/shenandoahRuntime.hpp --- a/src/share/vm/gc_implementation/shenandoah/shenandoahRuntime.hpp Mon Feb 10 17:04:04 2020 +0100 +++ b/src/share/vm/gc_implementation/shenandoah/shenandoahRuntime.hpp Thu Feb 20 16:57:24 2020 +0100 @@ -33,15 +33,16 @@ class ShenandoahRuntime : public AllStatic { public: - static void write_ref_array_pre_oop_entry(oop* dst, size_t length); - static void write_ref_array_pre_narrow_oop_entry(narrowOop* dst, size_t length); - static void write_ref_array_post_entry(HeapWord* dst, size_t length); + static void write_ref_array_pre_oop_entry(oop* src, oop* dst, size_t length); + static void write_ref_array_pre_narrow_oop_entry(narrowOop* src, narrowOop* dst, size_t length); + static void write_ref_array_pre_duinit_oop_entry(oop* src, oop* dst, size_t length); + static void write_ref_array_pre_duinit_narrow_oop_entry(narrowOop* src, narrowOop* dst, size_t length); static void write_ref_field_pre_entry(oopDesc* orig, JavaThread* thread); static oopDesc* load_reference_barrier_JRT(oopDesc* src); static oopDesc* load_reference_barrier_interpreter(oopDesc* src); - static void shenandoah_clone_barrier(oopDesc* obj); + static void shenandoah_clone_barrier(oopDesc* s, oopDesc* d, size_t length); }; #endif // SHARE_VM_GC_SHENANDOAH_SHENANDOAHRUNTIME_HPP diff -r 786d56b6de04 src/share/vm/oops/objArrayKlass.cpp --- a/src/share/vm/oops/objArrayKlass.cpp Mon Feb 10 17:04:04 2020 +0100 +++ b/src/share/vm/oops/objArrayKlass.cpp Thu Feb 20 16:57:24 2020 +0100 @@ -56,6 +56,7 @@ #include "gc_implementation/parallelScavenge/psCompactionManager.hpp" #include "gc_implementation/parallelScavenge/psPromotionManager.inline.hpp" #include "gc_implementation/parallelScavenge/psScavenge.inline.hpp" +#include "gc_implementation/shenandoah/shenandoahBarrierSet.inline.hpp" #include "gc_implementation/shenandoah/shenandoahOopClosures.inline.hpp" #include "oops/oop.pcgc.inline.hpp" #endif // INCLUDE_ALL_GCS @@ -244,6 +245,12 @@ assert(bs->has_write_ref_array_opt(), "Barrier set must have ref array opt"); assert(bs->has_write_ref_array_pre_opt(), "For pre-barrier as well."); +#ifdef INCLUDE_ALL_GCS + if (UseShenandoahGC) { + ShenandoahBarrierSet::barrier_set()->arraycopy_pre(src, dst, length); + } +#endif + if (s == d) { // since source and destination are equal we do not need conversion checks. assert(length > 0, "sanity check"); diff -r 786d56b6de04 src/share/vm/opto/library_call.cpp --- a/src/share/vm/opto/library_call.cpp Mon Feb 10 17:04:04 2020 +0100 +++ b/src/share/vm/opto/library_call.cpp Thu Feb 20 16:57:24 2020 +0100 @@ -44,6 +44,7 @@ #include "utilities/macros.hpp" #if INCLUDE_ALL_GCS #include "gc_implementation/shenandoah/shenandoahBarrierSetC2.hpp" +#include "gc_implementation/shenandoah/shenandoahRuntime.hpp" #include "gc_implementation/shenandoah/shenandoahSupport.hpp" #endif @@ -4549,21 +4550,23 @@ countx = _gvn.transform(new (C) SubXNode(countx, MakeConX(base_off))); countx = _gvn.transform(new (C) URShiftXNode(countx, intcon(LogBytesPerLong) )); +#ifdef INCLUDE_ALL_GCS + if (UseShenandoahGC && ShenandoahCloneBarrier) { + // Make sure that references in the cloned object are updated for Shenandoah. + make_runtime_call(RC_LEAF|RC_NO_FP, + OptoRuntime::shenandoah_clone_barrier_Type(), + CAST_FROM_FN_PTR(address, ShenandoahRuntime::shenandoah_clone_barrier), + "shenandoah_clone_barrier", TypePtr::BOTTOM, + src, dest, countx); + } +#endif + const TypePtr* raw_adr_type = TypeRawPtr::BOTTOM; bool disjoint_bases = true; generate_unchecked_arraycopy(raw_adr_type, T_LONG, disjoint_bases, src, NULL, dest, NULL, countx, /*dest_uninitialized*/true); - if (UseShenandoahGC && ShenandoahCloneBarrier) { - // Make sure that references in the cloned object are updated for Shenandoah. - make_runtime_call(RC_LEAF|RC_NO_FP, - OptoRuntime::shenandoah_clone_barrier_Type(), - CAST_FROM_FN_PTR(address, SharedRuntime::shenandoah_clone_barrier), - "shenandoah_clone_barrier", TypePtr::BOTTOM, - alloc_obj); - } - // If necessary, emit some card marks afterwards. (Non-arrays only.) if (card_mark) { assert(!is_array, ""); diff -r 786d56b6de04 src/share/vm/opto/runtime.cpp --- a/src/share/vm/opto/runtime.cpp Mon Feb 10 17:04:04 2020 +0100 +++ b/src/share/vm/opto/runtime.cpp Thu Feb 20 16:57:24 2020 +0100 @@ -583,9 +583,11 @@ } const TypeFunc *OptoRuntime::shenandoah_clone_barrier_Type() { - const Type **fields = TypeTuple::fields(1); - fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL; // original field value - const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+1, fields); + const Type **fields = TypeTuple::fields(3); + fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL; // src + fields[TypeFunc::Parms+1] = TypeInstPtr::NOTNULL; // dst + fields[TypeFunc::Parms+2] = TypeInt::INT; // length + const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+3, fields); // create result type (range) fields = TypeTuple::fields(0); diff -r 786d56b6de04 src/share/vm/prims/jvm.cpp --- a/src/share/vm/prims/jvm.cpp Mon Feb 10 17:04:04 2020 +0100 +++ b/src/share/vm/prims/jvm.cpp Thu Feb 20 16:57:24 2020 +0100 @@ -93,6 +93,7 @@ #if INCLUDE_ALL_GCS #include "gc_implementation/g1/g1SATBCardTableModRefBS.hpp" +#include "gc_implementation/shenandoah/shenandoahBarrierSet.inline.hpp" #endif // INCLUDE_ALL_GCS #include @@ -644,6 +645,12 @@ new_obj_oop = CollectedHeap::obj_allocate(klass, size, CHECK_NULL); } +#ifdef INCLUDE_ALL_GCS + if (UseShenandoahGC && ShenandoahCloneBarrier) { + ShenandoahBarrierSet::barrier_set()->clone_barrier(obj()); + } +#endif + // 4839641 (4840070): We must do an oop-atomic copy, because if another thread // is modifying a reference field in the clonee, a non-oop-atomic copy might // be suspended in the middle of copying the pointer and end up with parts diff -r 786d56b6de04 src/share/vm/runtime/sharedRuntime.cpp --- a/src/share/vm/runtime/sharedRuntime.cpp Mon Feb 10 17:04:04 2020 +0100 +++ b/src/share/vm/runtime/sharedRuntime.cpp Thu Feb 20 16:57:24 2020 +0100 @@ -239,12 +239,6 @@ thread->dirty_card_queue().enqueue(card_addr); JRT_END -// Shenandoah clone barrier: makes sure that references point to to-space -// in cloned objects. -JRT_LEAF(void, SharedRuntime::shenandoah_clone_barrier(oopDesc* obj)) - oopDesc::bs()->write_region(MemRegion((HeapWord*) obj, obj->size())); -JRT_END - #endif // INCLUDE_ALL_GCS diff -r 786d56b6de04 src/share/vm/runtime/sharedRuntime.hpp --- a/src/share/vm/runtime/sharedRuntime.hpp Mon Feb 10 17:04:04 2020 +0100 +++ b/src/share/vm/runtime/sharedRuntime.hpp Thu Feb 20 16:57:24 2020 +0100 @@ -180,7 +180,6 @@ // G1 write barriers static void g1_wb_pre(oopDesc* orig, JavaThread *thread); static void g1_wb_post(void* card_addr, JavaThread* thread); - static void shenandoah_clone_barrier(oopDesc* obj); #endif // INCLUDE_ALL_GCS // exception handling and implicit exceptions