# HG changeset patch # Parent ab5759fe0f1e13a3d7e89fb9f954377141d123d3 diff -r ab5759fe0f1e src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp --- a/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp Thu May 02 15:59:41 2019 -0400 +++ b/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp Thu May 02 16:56:00 2019 -0400 @@ -45,68 +45,85 @@ void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop, Register src, Register dst, Register count, Register klass, Label& L_exit, RegSet saved_regs) { + bool checkcast = (decorators & ARRAYCOPY_CHECKCAST) != 0; + bool disjoint = (decorators & ARRAYCOPY_DISJOINT) != 0; + bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0; + if (is_oop) { - bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0; - if (ShenandoahSATBBarrier && !dest_uninitialized && !ShenandoahHeap::heap()->heuristics()->can_do_traversal_gc()) { - - Label done; - - // Avoid calling runtime if count == 0 - __ cbz(count, done); - - // Is marking active? + Label fastpath; Address gc_state(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset())); __ ldrb(rscratch1, gc_state); - __ tbz(rscratch1, ShenandoahHeap::MARKING_BITPOS, done); + uint64_t flags = ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::TRAVERSAL; + if (!dest_uninitialized) { + flags |= ShenandoahHeap::MARKING; + } + __ mov(rscratch2, flags); + __ andw(rscratch1, rscratch1, rscratch2); + __ cbz(rscratch1, fastpath); __ push(saved_regs, sp); - if (count == c_rarg0) { - if (dst == c_rarg1) { - // exactly backwards!! - __ mov(rscratch1, c_rarg0); - __ mov(c_rarg0, c_rarg1); - __ mov(c_rarg1, rscratch1); + // Registers should already be in the right places for runtime calls. + assert(src == c_rarg0, "from must be 1st arg"); + assert(dst == c_rarg1, "to must be 2nd arg"); + assert(count == c_rarg2, "count must be 3rd arg"); + if (checkcast) { + assert(klass != noreg, "must have klass reg"); + __ mov(c_rarg3, klass); + // No conjoint case in checkcast + assert(disjoint, "checkcast always disjoint"); + if (dest_uninitialized) { + if (UseCompressedOops) { + __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_narrow_uninit_disjoint_checkcast), 4); } else { - __ mov(c_rarg1, count); - __ mov(c_rarg0, dst); + __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_oop_uninit_disjoint_checkcast), 4); } } else { - __ mov(c_rarg0, dst); - __ mov(c_rarg1, count); + if (UseCompressedOops) { + __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_narrow_disjoint_checkcast), 4); + } else { + __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_oop_disjoint_checkcast), 4); + } } - if (UseCompressedOops) { - __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_narrow_oop_entry), 2); + // Preserve return value + __ mov(rscratch1, r0); + } else { + if (disjoint) { + if (dest_uninitialized) { + if (UseCompressedOops) { + __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_narrow_uninit_disjoint), 3); + } else { + __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_oop_uninit_disjoint), 3); + } + } else { + if (UseCompressedOops) { + __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_narrow_disjoint), 3); + } else { + __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_oop_disjoint), 3); + } + } } else { - __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_oop_entry), 2); + if (dest_uninitialized) { + if (UseCompressedOops) { + __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_narrow_uninit_conjoint), 3); + } else { + __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_oop_uninit_conjoint), 3); + } + } else { + if (UseCompressedOops) { + __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_narrow_conjoint), 3); + } else { + __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_oop_conjoint), 3); + } + } } - __ pop(saved_regs, sp); - __ bind(done); } - } -} - -void ShenandoahBarrierSetAssembler::arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop, - Register start, Register count, Register scratch, RegSet saved_regs) { - if (is_oop) { - Label done; - - // Avoid calling runtime if count == 0 - __ cbz(count, done); - - // Is updating references? - Address gc_state(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset())); - __ ldrb(rscratch1, gc_state); - __ tbz(rscratch1, ShenandoahHeap::UPDATEREFS_BITPOS, done); - - __ push(saved_regs, sp); - assert_different_registers(start, count, scratch); - assert_different_registers(c_rarg0, count); - __ mov(c_rarg0, start); - __ mov(c_rarg1, count); - __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_post_entry), 2); __ pop(saved_regs, sp); - - __ bind(done); + if (checkcast) { + // Restore return value into count register + __ mov(count, rscratch1); + } + __ b(L_exit); + __ bind(fastpath); } } diff -r ab5759fe0f1e src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.hpp --- a/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.hpp Thu May 02 15:59:41 2019 -0400 +++ b/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.hpp Thu May 02 16:56:00 2019 -0400 @@ -75,8 +75,6 @@ virtual void arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop, Register src, Register dst, Register count, Register klass, Label& L_exit, RegSet saved_regs); - virtual void arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop, - Register start, Register count, Register tmp, RegSet saved_regs); virtual void load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Register dst, Address src, Register tmp1, Register tmp_thread); virtual void store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, diff -r ab5759fe0f1e src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp --- a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp Thu May 02 15:59:41 2019 -0400 +++ b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp Thu May 02 16:56:00 2019 -0400 @@ -43,144 +43,113 @@ address ShenandoahBarrierSetAssembler::_shenandoah_lrb = NULL; +#ifdef _LP64 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Register src, Register dst, Register count, Register klass, Label& L_exit) { bool checkcast = (decorators & ARRAYCOPY_CHECKCAST) != 0; bool disjoint = (decorators & ARRAYCOPY_DISJOINT) != 0; - bool obj_int = type == T_OBJECT LP64_ONLY(&& UseCompressedOops); bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0; if (type == T_OBJECT || type == T_ARRAY) { -#ifdef _LP64 - if (!checkcast) { - if (!obj_int) { - // Save count for barrier - __ movptr(r11, count); - } else if (disjoint) { - // Save dst in r11 in the disjoint case - __ movq(r11, dst); + Label fastpath; + Address gc_state(r15_thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset())); + int flags = ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::TRAVERSAL; + if (!dest_uninitialized) { + flags |= ShenandoahHeap::MARKING; + } + __ testb(gc_state, flags); + __ jcc(Assembler::zero, fastpath); + __ pusha(); + + // Registers should already be in the right places for runtime calls. + assert(src == c_rarg0, "from must be 1st arg"); + assert(dst == c_rarg1, "to must be 2nd arg"); + assert(count == c_rarg2, "count must be 3rd arg"); + if (checkcast) { + assert(klass != noreg, "must have klass reg"); + __ movptr(c_rarg3, klass); + // No conjoint case in checkcast + assert(disjoint, "checkcast always disjoint"); + if (dest_uninitialized) { + if (UseCompressedOops) { + __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_narrow_uninit_disjoint_checkcast), 4); + } else { + __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_oop_uninit_disjoint_checkcast), 4); + } + } else { + if (UseCompressedOops) { + __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_narrow_disjoint_checkcast), 4); + } else { + __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_oop_disjoint_checkcast), 4); + } + } + } else { + if (disjoint) { + if (dest_uninitialized) { + if (UseCompressedOops) { + __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_narrow_uninit_disjoint), 3); + } else { + __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_oop_uninit_disjoint), 3); + } + } else { + if (UseCompressedOops) { + __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_narrow_disjoint), 3); + } else { + __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_oop_disjoint), 3); + } + } + } else { + if (dest_uninitialized) { + if (UseCompressedOops) { + __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_narrow_uninit_conjoint), 3); + } else { + __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_oop_uninit_conjoint), 3); + } + } else { + if (UseCompressedOops) { + __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_narrow_conjoint), 3); + } else { + __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_oop_conjoint), 3); + } + } } } -#else - if (disjoint) { - __ mov(rdx, dst); // save 'to' + + { + // popa, but preserving return value from checkcast calls + __ movq(r15, Address(rsp, 0)); + __ movq(r14, Address(rsp, wordSize)); + __ movq(r13, Address(rsp, 2 * wordSize)); + __ movq(r12, Address(rsp, 3 * wordSize)); + __ movq(r11, Address(rsp, 4 * wordSize)); + __ movq(r10, Address(rsp, 5 * wordSize)); + __ movq(r9, Address(rsp, 6 * wordSize)); + __ movq(r8, Address(rsp, 7 * wordSize)); + __ movq(rdi, Address(rsp, 8 * wordSize)); + __ movq(rsi, Address(rsp, 9 * wordSize)); + __ movq(rbp, Address(rsp, 10 * wordSize)); + // skip rsp + __ movq(rbx, Address(rsp, 12 * wordSize)); + __ movq(rdx, Address(rsp, 13 * wordSize)); + __ movq(rcx, Address(rsp, 14 * wordSize)); + if (!checkcast) { + __ movq(rax, Address(rsp, 15 * wordSize)); + } + __ addq(rsp, 16 * wordSize); } -#endif - - if (ShenandoahSATBBarrier && !dest_uninitialized && !ShenandoahHeap::heap()->heuristics()->can_do_traversal_gc()) { - Register thread = NOT_LP64(rax) LP64_ONLY(r15_thread); -#ifndef _LP64 - __ push(thread); - __ get_thread(thread); -#endif - - Label done; - // Short-circuit if count == 0. - __ testptr(count, count); - __ jcc(Assembler::zero, done); - - // Avoid runtime call when not marking. - Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset())); - __ testb(gc_state, ShenandoahHeap::MARKING); - __ jcc(Assembler::zero, done); - - __ pusha(); // push registers -#ifdef _LP64 - if (count == c_rarg0) { - if (dst == c_rarg1) { - // exactly backwards!! - __ xchgptr(c_rarg1, c_rarg0); - } else { - __ movptr(c_rarg1, count); - __ movptr(c_rarg0, dst); - } - } else { - __ movptr(c_rarg0, dst); - __ movptr(c_rarg1, count); - } - if (UseCompressedOops) { - __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_narrow_oop_entry), 2); - } else { - __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_oop_entry), 2); - } -#else - __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_oop_entry), - dst, count); -#endif - __ popa(); - __ bind(done); - NOT_LP64(__ pop(thread);) - } - } - -} - -void ShenandoahBarrierSetAssembler::arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, BasicType type, - Register src, Register dst, Register count) { - bool checkcast = (decorators & ARRAYCOPY_CHECKCAST) != 0; - bool disjoint = (decorators & ARRAYCOPY_DISJOINT) != 0; - bool obj_int = type == T_OBJECT LP64_ONLY(&& UseCompressedOops); - Register tmp = rax; - - if (type == T_OBJECT || type == T_ARRAY) { -#ifdef _LP64 - if (!checkcast) { - if (!obj_int) { - // Save count for barrier - count = r11; - } else if (disjoint && obj_int) { - // Use the saved dst in the disjoint case - dst = r11; - } - } else { - tmp = rscratch1; - } -#else - if (disjoint) { - __ mov(dst, rdx); // restore 'to' - } -#endif - - Register thread = NOT_LP64(rax) LP64_ONLY(r15_thread); -#ifndef _LP64 - __ push(thread); - __ get_thread(thread); -#endif - - // Short-circuit if count == 0. - Label done; - __ testptr(count, count); - __ jcc(Assembler::zero, done); - - // Skip runtime call if no forwarded objects. - Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset())); - __ testb(gc_state, ShenandoahHeap::UPDATEREFS); - __ jcc(Assembler::zero, done); - - __ pusha(); // push registers (overkill) -#ifdef _LP64 - if (c_rarg0 == count) { // On win64 c_rarg0 == rcx - assert_different_registers(c_rarg1, dst); - __ mov(c_rarg1, count); - __ mov(c_rarg0, dst); - } else { - assert_different_registers(c_rarg0, count); - __ mov(c_rarg0, dst); - __ mov(c_rarg1, count); - } - __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_post_entry), 2); -#else - __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_post_entry), - dst, count); -#endif - __ popa(); - - __ bind(done); - NOT_LP64(__ pop(thread);) + __ jmp(L_exit); // Skip fastpath + __ bind(fastpath); } } +#else +// 32 bit - not yet implemented +void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, BasicType type, + Register src, Register dst, Register count, Label& L_exit, Register klass) { + Unimplemented(); +} +#endif void ShenandoahBarrierSetAssembler::shenandoah_write_barrier_pre(MacroAssembler* masm, Register obj, Register pre_val, diff -r ab5759fe0f1e src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.hpp --- a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.hpp Thu May 02 15:59:41 2019 -0400 +++ b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.hpp Thu May 02 16:56:00 2019 -0400 @@ -84,8 +84,6 @@ bool exchange, Register tmp1, Register tmp2); virtual void arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Register src, Register dst, Register count, Register klass, Label& L_exit); - virtual void arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, BasicType type, - Register src, Register dst, Register count); virtual void load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Register dst, Address src, Register tmp1, Register tmp_thread); virtual void store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, diff -r ab5759fe0f1e src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp --- a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp Thu May 02 15:59:41 2019 -0400 +++ b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp Thu May 02 16:56:00 2019 -0400 @@ -96,73 +96,6 @@ return true; } -template -void ShenandoahBarrierSet::write_ref_array_loop(HeapWord* start, size_t count) { - assert(UseShenandoahGC && ShenandoahCloneBarrier, "should be enabled"); - ShenandoahUpdateRefsForOopClosure cl; - T* dst = (T*) start; - for (size_t i = 0; i < count; i++) { - cl.do_oop(dst++); - } -} - -void ShenandoahBarrierSet::write_ref_array(HeapWord* start, size_t count) { - assert(_heap->is_update_refs_in_progress(), "should not be here otherwise"); - assert(count > 0, "Should have been filtered before"); - - if (_heap->is_concurrent_traversal_in_progress()) { - ShenandoahEvacOOMScope oom_evac_scope; - if (UseCompressedOops) { - write_ref_array_loop(start, count); - } else { - write_ref_array_loop(start, count); - } - } else { - if (UseCompressedOops) { - write_ref_array_loop(start, count); - } else { - write_ref_array_loop(start, count); - } - } -} - -template -void ShenandoahBarrierSet::write_ref_array_pre_work(T* dst, size_t count) { - shenandoah_assert_not_in_cset_loc_except(dst, _heap->cancelled_gc()); - assert(ShenandoahThreadLocalData::satb_mark_queue(Thread::current()).is_active(), "Shouldn't be here otherwise"); - assert(ShenandoahSATBBarrier, "Shouldn't be here otherwise"); - assert(count > 0, "Should have been filtered before"); - - Thread* thread = Thread::current(); - ShenandoahMarkingContext* ctx = _heap->marking_context(); - bool has_forwarded = _heap->has_forwarded_objects(); - T* elem_ptr = dst; - for (size_t i = 0; i < count; i++, elem_ptr++) { - T heap_oop = RawAccess<>::oop_load(elem_ptr); - if (!CompressedOops::is_null(heap_oop)) { - oop obj = CompressedOops::decode_not_null(heap_oop); - if (has_forwarded) { - obj = resolve_forwarded_not_null(obj); - } - if (!ctx->is_marked(obj)) { - ShenandoahThreadLocalData::satb_mark_queue(thread).enqueue_known_active(obj); - } - } - } -} - -void ShenandoahBarrierSet::write_ref_array_pre(oop* dst, size_t count, bool dest_uninitialized) { - if (! dest_uninitialized) { - write_ref_array_pre_work(dst, count); - } -} - -void ShenandoahBarrierSet::write_ref_array_pre(narrowOop* dst, size_t count, bool dest_uninitialized) { - if (! dest_uninitialized) { - write_ref_array_pre_work(dst, count); - } -} - template inline void ShenandoahBarrierSet::inline_write_ref_field_pre(T* field, oop new_val) { shenandoah_assert_not_in_cset_loc_except(field, _heap->cancelled_gc()); diff -r ab5759fe0f1e src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.hpp --- a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.hpp Thu May 02 15:59:41 2019 -0400 +++ b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.hpp Thu May 02 16:56:00 2019 -0400 @@ -61,15 +61,6 @@ bool is_aligned(HeapWord* hw); - void write_ref_array(HeapWord* start, size_t count); - - template void - write_ref_array_pre_work(T* dst, size_t count); - - void write_ref_array_pre(oop* dst, size_t count, bool dest_uninitialized); - - void write_ref_array_pre(narrowOop* dst, size_t count, bool dest_uninitialized); - // We export this to make it available in cases where the static // type of the barrier set is known. Note that it is non-virtual. template inline void inline_write_ref_field_pre(T* field, oop new_val); @@ -99,10 +90,10 @@ void enqueue(oop obj); + template + size_t arraycopy_entry(T* src, T* dst, size_t length, Klass* bound, bool checkcast, bool disjoint, bool uninit); + private: - template - void write_ref_array_loop(HeapWord* start, size_t count); - oop load_reference_barrier_impl(oop obj); static void keep_alive_if_weak(DecoratorSet decorators, oop value) { @@ -115,19 +106,19 @@ } template - bool arraycopy_loop_1(T* src, T* dst, size_t length, Klass* bound, + size_t arraycopy_loop_1(T* src, T* dst, size_t length, Klass* bound, bool checkcast, bool satb, bool disjoint, ShenandoahBarrierSet::ArrayCopyStoreValMode storeval_mode); template - bool arraycopy_loop_2(T* src, T* dst, size_t length, Klass* bound, + size_t arraycopy_loop_2(T* src, T* dst, size_t length, Klass* bound, bool satb, bool disjoint, ShenandoahBarrierSet::ArrayCopyStoreValMode storeval_mode); template - bool arraycopy_loop_3(T* src, T* dst, size_t length, Klass* bound, + size_t arraycopy_loop_3(T* src, T* dst, size_t length, Klass* bound, bool disjoint, ShenandoahBarrierSet::ArrayCopyStoreValMode storeval_mode); template - bool arraycopy_loop(T* src, T* dst, size_t length, Klass* bound, bool disjoint); + size_t arraycopy_loop(T* src, T* dst, size_t length, Klass* bound, bool disjoint); template bool arraycopy_element(T* cur_src, T* cur_dst, Klass* bound, Thread* const thread, ShenandoahMarkingContext* const ctx); diff -r ab5759fe0f1e src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp --- a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp Thu May 02 15:59:41 2019 -0400 +++ b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp Thu May 02 16:56:00 2019 -0400 @@ -172,7 +172,42 @@ } template -bool ShenandoahBarrierSet::arraycopy_loop_1(T* src, T* dst, size_t length, Klass* bound, +inline size_t ShenandoahBarrierSet::arraycopy_entry(T* src, T* dst, size_t length, Klass* bound, bool checkcast, bool disjoint, bool uninit) { + ShenandoahHeap* heap = ShenandoahHeap::heap(); + bool satb = ShenandoahSATBBarrier && heap->is_concurrent_mark_in_progress() && !uninit; + ArrayCopyStoreValMode storeval_mode; + if (heap->has_forwarded_objects()) { + if (heap->is_concurrent_traversal_in_progress()) { + storeval_mode = WRITE_BARRIER; + } else if (heap->is_concurrent_mark_in_progress() || heap->is_update_refs_in_progress()) { + storeval_mode = READ_BARRIER; + } else { + assert(heap->is_idle() || heap->is_evacuation_in_progress(), "must not have anything in progress"); + storeval_mode = NONE; // E.g. during evac or outside cycle + } + } else { + assert(heap->is_stable() || heap->is_concurrent_mark_in_progress(), "must not have anything in progress"); + storeval_mode = NONE; + } + +// tty->print("arraycopy "); +// if (satb) tty->print("satb "); +// if (checkcast) tty->print("checkcast "); +// if (disjoint) tty->print("disjoint "); +// if (uninit) tty->print("uninit "); +// if (storeval_mode == READ_BARRIER) tty->print("RB "); +// if (storeval_mode == WRITE_BARRIER) tty->print("WB "); +// tty->print_cr(" "); + +// if (!satb && !checkcast && storeval_mode == NONE) { +// // Short-circuit to bulk copy. +// return ArrayAccess::oop_arraycopy_raw(src, dst, length); +// } + return arraycopy_loop_1(src, dst, length, bound, checkcast, satb, disjoint, storeval_mode); +} + +template +size_t ShenandoahBarrierSet::arraycopy_loop_1(T* src, T* dst, size_t length, Klass* bound, bool checkcast, bool satb, bool disjoint, ShenandoahBarrierSet::ArrayCopyStoreValMode storeval_mode) { if (checkcast) { @@ -183,7 +218,7 @@ } template -bool ShenandoahBarrierSet::arraycopy_loop_2(T* src, T* dst, size_t length, Klass* bound, +size_t ShenandoahBarrierSet::arraycopy_loop_2(T* src, T* dst, size_t length, Klass* bound, bool satb, bool disjoint, ShenandoahBarrierSet::ArrayCopyStoreValMode storeval_mode) { if (satb) { @@ -194,7 +229,7 @@ } template -bool ShenandoahBarrierSet::arraycopy_loop_3(T* src, T* dst, size_t length, Klass* bound, bool disjoint, +size_t ShenandoahBarrierSet::arraycopy_loop_3(T* src, T* dst, size_t length, Klass* bound, bool disjoint, ShenandoahBarrierSet::ArrayCopyStoreValMode storeval_mode) { switch (storeval_mode) { case NONE: @@ -210,7 +245,7 @@ } template -bool ShenandoahBarrierSet::arraycopy_loop(T* src, T* dst, size_t length, Klass* bound, bool disjoint) { +size_t ShenandoahBarrierSet::arraycopy_loop(T* src, T* dst, size_t length, Klass* bound, bool disjoint) { Thread* thread = Thread::current(); ShenandoahMarkingContext* ctx = _heap->marking_context(); ShenandoahEvacOOMScope oom_evac_scope; @@ -240,7 +275,12 @@ T* src_end = src + length; for (; cur_src < src_end; cur_src++, cur_dst++) { if (!arraycopy_element(cur_src, cur_dst, bound, thread, ctx)) { - return false; +// tty->print_cr("cur_src: " SIZE_FORMAT, p2i(cur_src)); +// tty->print_cr("src: " SIZE_FORMAT, p2i(src)); +// tty->print_cr("cur_src - src: " SIZE_FORMAT, cur_src - src); +// tty->print_cr("~(cur_src - src): " SIZE_FORMAT, ~(cur_src - src)); +// ShouldNotReachHere(); + return ~(cur_src - src); } } } else { @@ -249,11 +289,12 @@ T* cur_dst = dst + length - 1; for (; cur_src >= src; cur_src--, cur_dst--) { if (!arraycopy_element(cur_src, cur_dst, bound, thread, ctx)) { - return false; +// ShouldNotReachHere(); + return ~((src + length - 1) - cur_src); } } } - return true; + return 0; } template @@ -330,36 +371,14 @@ bool ShenandoahBarrierSet::AccessBarrier::oop_arraycopy_in_heap(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw, arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw, size_t length) { - ShenandoahHeap* heap = ShenandoahHeap::heap(); - bool satb = ShenandoahSATBBarrier && heap->is_concurrent_mark_in_progress(); bool checkcast = HasDecorator::value; bool disjoint = HasDecorator::value; - ArrayCopyStoreValMode storeval_mode; - if (heap->has_forwarded_objects()) { - if (heap->is_concurrent_traversal_in_progress()) { - storeval_mode = WRITE_BARRIER; - } else if (heap->is_update_refs_in_progress()) { - storeval_mode = READ_BARRIER; - } else { - assert(heap->is_idle() || heap->is_evacuation_in_progress(), "must not have anything in progress"); - storeval_mode = NONE; // E.g. during evac or outside cycle - } - } else { - assert(heap->is_stable() || heap->is_concurrent_mark_in_progress(), "must not have anything in progress"); - storeval_mode = NONE; - } - - if (!satb && !checkcast && storeval_mode == NONE) { - // Short-circuit to bulk copy. - return Raw::oop_arraycopy(src_obj, src_offset_in_bytes, src_raw, dst_obj, dst_offset_in_bytes, dst_raw, length); - } - src_raw = arrayOopDesc::obj_offset_to_raw(src_obj, src_offset_in_bytes, src_raw); dst_raw = arrayOopDesc::obj_offset_to_raw(dst_obj, dst_offset_in_bytes, dst_raw); Klass* bound = objArrayOop(dst_obj)->element_klass(); ShenandoahBarrierSet* bs = ShenandoahBarrierSet::barrier_set(); - return bs->arraycopy_loop_1(src_raw, dst_raw, length, bound, checkcast, satb, disjoint, storeval_mode); + return bs->arraycopy_entry(src_raw, dst_raw, length, bound, checkcast, disjoint, false) == 0; } #endif // SHARE_GC_SHENANDOAH_SHENANDOAHBARRIERSET_INLINE_HPP diff -r ab5759fe0f1e src/hotspot/share/gc/shenandoah/shenandoahRuntime.cpp --- a/src/hotspot/share/gc/shenandoah/shenandoahRuntime.cpp Thu May 02 15:59:41 2019 -0400 +++ b/src/hotspot/share/gc/shenandoah/shenandoahRuntime.cpp Thu May 02 16:56:00 2019 -0400 @@ -28,19 +28,65 @@ #include "runtime/interfaceSupport.inline.hpp" #include "oops/oop.inline.hpp" -void ShenandoahRuntime::write_ref_array_pre_oop_entry(oop* dst, size_t length) { - ShenandoahBarrierSet *bs = ShenandoahBarrierSet::barrier_set(); - bs->write_ref_array_pre(dst, length, false); +void ShenandoahRuntime::arraycopy_oop_disjoint(oop* src, oop* dst, size_t length) { + //tty->print_cr("arraycopy_oop_disjoint"); + ShenandoahBarrierSet::barrier_set()->arraycopy_entry(src, dst, length, NULL, false /*checkcast*/, true /*disjoint*/, false /*uninit*/); } -void ShenandoahRuntime::write_ref_array_pre_narrow_oop_entry(narrowOop* dst, size_t length) { - ShenandoahBarrierSet *bs = ShenandoahBarrierSet::barrier_set(); - bs->write_ref_array_pre(dst, length, false); +void ShenandoahRuntime::arraycopy_narrow_disjoint(narrowOop* src, narrowOop* dst, size_t length) { + //tty->print_cr("arraycopy_narrow_disjoint"); + ShenandoahBarrierSet::barrier_set()->arraycopy_entry(src, dst, length, NULL, false /*checkcast*/, true /*disjoint*/, false /*uninit*/); } -void ShenandoahRuntime::write_ref_array_post_entry(HeapWord* dst, size_t length) { - ShenandoahBarrierSet *bs = ShenandoahBarrierSet::barrier_set(); - bs->ShenandoahBarrierSet::write_ref_array(dst, length); +void ShenandoahRuntime::arraycopy_oop_uninit_disjoint(oop* src, oop* dst, size_t length) { + //tty->print_cr("arraycopy_oop_uninit_disjoint"); + ShenandoahBarrierSet::barrier_set()->arraycopy_entry(src, dst, length, NULL, false /*checkcast*/, true /*disjoint*/, true /*uninit*/); +} + +void ShenandoahRuntime::arraycopy_narrow_uninit_disjoint(narrowOop* src, narrowOop* dst, size_t length) { + //tty->print_cr("arraycopy_narrow_uninit_disjoint"); + ShenandoahBarrierSet::barrier_set()->arraycopy_entry(src, dst, length, NULL, false /*checkcast*/, true /*disjoint*/, true /*uninit*/); +} + +void ShenandoahRuntime::arraycopy_oop_conjoint(oop* src, oop* dst, size_t length) { + //tty->print_cr("arraycopy_oop_conjoint"); + ShenandoahBarrierSet::barrier_set()->arraycopy_entry(src, dst, length, NULL, false /*checkcast*/, false /*disjoint*/, false /*uninit*/); +} + +void ShenandoahRuntime::arraycopy_narrow_conjoint(narrowOop* src, narrowOop* dst, size_t length) { + //tty->print_cr("arraycopy_narrow_conjoint"); + ShenandoahBarrierSet::barrier_set()->arraycopy_entry(src, dst, length, NULL, false /*checkcast*/, false /*disjoint*/, false /*uninit*/); +} + +void ShenandoahRuntime::arraycopy_oop_uninit_conjoint(oop* src, oop* dst, size_t length) { + //tty->print_cr("arraycopy_oop_uninit_conjoint"); + ShenandoahBarrierSet::barrier_set()->arraycopy_entry(src, dst, length, NULL, false /*checkcast*/, false /*disjoint*/, true /*uninit*/); +} + +void ShenandoahRuntime::arraycopy_narrow_uninit_conjoint(narrowOop* src, narrowOop* dst, size_t length) { + //tty->print_cr("arraycopy_narrow_uninit_conjoint"); + ShenandoahBarrierSet::barrier_set()->arraycopy_entry(src, dst, length, NULL, false /*checkcast*/, false /*disjoint*/, true /*uninit*/); +} + +size_t ShenandoahRuntime::arraycopy_oop_disjoint_checkcast(oop* src, oop* dst, size_t length, Klass* klass) { + //tty->print_cr("arraycopy_oop_disjoint_checkcast"); + // klass->print(); + return fixup_arraycopy_return(ShenandoahBarrierSet::barrier_set()->arraycopy_entry(src, dst, length, klass, true /*checkcast*/, true /*disjoint*/, false /*uninit*/)); +} + +size_t ShenandoahRuntime::arraycopy_narrow_disjoint_checkcast(narrowOop* src, narrowOop* dst, size_t length, Klass* klass) { + //tty->print_cr("arraycopy_narrow_disjoint_checkcast"); + return fixup_arraycopy_return(ShenandoahBarrierSet::barrier_set()->arraycopy_entry(src, dst, length, klass, true /*checkcast*/, true /*disjoint*/, false /*uninit*/)); +} + +size_t ShenandoahRuntime::arraycopy_oop_uninit_disjoint_checkcast(oop* src, oop* dst, size_t length, Klass* klass) { + //tty->print_cr("arraycopy_oop_uninit_disjoint_checkcast"); + return fixup_arraycopy_return(ShenandoahBarrierSet::barrier_set()->arraycopy_entry(src, dst, length, klass, true /*checkcast*/, true /*disjoint*/, true /*uninit*/)); +} + +size_t ShenandoahRuntime::arraycopy_narrow_uninit_disjoint_checkcast(narrowOop* src, narrowOop* dst, size_t length, Klass* klass) { + //tty->print_cr("arraycopy_narrow_uninit_disjoint_checkcast"); + return fixup_arraycopy_return(ShenandoahBarrierSet::barrier_set()->arraycopy_entry(src, dst, length, klass, true /*checkcast*/, true /*disjoint*/, true /*uninit*/)); } // Shenandoah pre write barrier slowpath diff -r ab5759fe0f1e src/hotspot/share/gc/shenandoah/shenandoahRuntime.hpp --- a/src/hotspot/share/gc/shenandoah/shenandoahRuntime.hpp Thu May 02 15:59:41 2019 -0400 +++ b/src/hotspot/share/gc/shenandoah/shenandoahRuntime.hpp Thu May 02 16:56:00 2019 -0400 @@ -31,10 +31,25 @@ class oopDesc; class ShenandoahRuntime : public AllStatic { +private: + static size_t fixup_arraycopy_return(size_t retval) { + return retval == 0 ? 0 : retval + 1; + } public: - static void write_ref_array_pre_oop_entry(oop* dst, size_t length); - static void write_ref_array_pre_narrow_oop_entry(narrowOop* dst, size_t length); - static void write_ref_array_post_entry(HeapWord* dst, size_t length); + static void arraycopy_oop_disjoint(oop* src, oop* dst, size_t length); + static void arraycopy_narrow_disjoint(narrowOop* src, narrowOop* dst, size_t length); + static void arraycopy_oop_uninit_disjoint(oop* src, oop* dst, size_t length); + static void arraycopy_narrow_uninit_disjoint(narrowOop* src, narrowOop* dst, size_t length); + static void arraycopy_oop_conjoint(oop* src, oop* dst, size_t length); + static void arraycopy_narrow_conjoint(narrowOop* src, narrowOop* dst, size_t length); + static void arraycopy_oop_uninit_conjoint(oop* src, oop* dst, size_t length); + static void arraycopy_narrow_uninit_conjoint(narrowOop* src, narrowOop* dst, size_t length); + + static size_t arraycopy_oop_disjoint_checkcast(oop* src, oop* dst, size_t length, Klass* klass); + static size_t arraycopy_narrow_disjoint_checkcast(narrowOop* src, narrowOop* dst, size_t length, Klass* klass); + static size_t arraycopy_oop_uninit_disjoint_checkcast(oop* src, oop* dst, size_t length, Klass* klass); + static size_t arraycopy_narrow_uninit_disjoint_checkcast(narrowOop* src, narrowOop* dst, size_t length, Klass* klass); + static void write_ref_field_pre_entry(oopDesc* orig, JavaThread* thread); static oopDesc* load_reference_barrier_JRT(oopDesc* src);