< prev index next >
src/hotspot/cpu/x86/stubGenerator_x86_64.cpp
8198949_arraycopy
#include "precompiled.hpp"
#include "asm/macroAssembler.hpp"
#include "asm/macroAssembler.inline.hpp"
#include "ci/ciUtilities.hpp"
! #include "gc/shared/cardTable.hpp"
! #include "gc/shared/cardTableModRefBS.hpp"
#include "interpreter/interpreter.hpp"
#include "nativeInst_x86.hpp"
#include "oops/instanceOop.hpp"
#include "oops/method.hpp"
#include "oops/objArrayKlass.hpp"
#include "precompiled.hpp"
#include "asm/macroAssembler.hpp"
#include "asm/macroAssembler.inline.hpp"
#include "ci/ciUtilities.hpp"
! #include "gc/shared/barrierSet.hpp"
! #include "gc/shared/barrierSetCodeGen.hpp"
#include "interpreter/interpreter.hpp"
#include "nativeInst_x86.hpp"
#include "oops/instanceOop.hpp"
#include "oops/method.hpp"
#include "oops/objArrayKlass.hpp"
***************
__ movptr(rdi, saved_rdi);
__ movptr(rsi, saved_rsi);
#endif
}
- // Generate code for an array write pre barrier
- //
- // addr - starting address
- // count - element count
- // tmp - scratch register
- //
- // Destroy no registers!
- //
- void gen_write_ref_array_pre_barrier(Register addr, Register count, bool dest_uninitialized) {
- BarrierSet* bs = Universe::heap()->barrier_set();
- switch (bs->kind()) {
- case BarrierSet::G1BarrierSet:
- // With G1, don't generate the call if we statically know that the target in uninitialized
- if (!dest_uninitialized) {
- Label filtered;
- Address in_progress(r15_thread, in_bytes(JavaThread::satb_mark_queue_offset() +
- SATBMarkQueue::byte_offset_of_active()));
- // Is marking active?
- if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) {
- __ cmpl(in_progress, 0);
- } else {
- assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption");
- __ cmpb(in_progress, 0);
- }
- __ jcc(Assembler::equal, filtered);
-
- __ pusha(); // push registers
- if (count == c_rarg0) {
- if (addr == c_rarg1) {
- // exactly backwards!!
- __ xchgptr(c_rarg1, c_rarg0);
- } else {
- __ movptr(c_rarg1, count);
- __ movptr(c_rarg0, addr);
- }
- } else {
- __ movptr(c_rarg0, addr);
- __ movptr(c_rarg1, count);
- }
- __ call_VM_leaf(CAST_FROM_FN_PTR(address, BarrierSet::static_write_ref_array_pre), 2);
- __ popa();
-
- __ bind(filtered);
- }
- break;
- case BarrierSet::CardTableModRef:
- break;
- default:
- ShouldNotReachHere();
-
- }
- }
-
- //
- // Generate code for an array write post barrier
- //
- // Input:
- // start - register containing starting address of destination array
- // count - elements count
- // scratch - scratch register
- //
- // The input registers are overwritten.
- //
- void gen_write_ref_array_post_barrier(Register start, Register count, Register scratch) {
- assert_different_registers(start, count, scratch);
- BarrierSet* bs = Universe::heap()->barrier_set();
- switch (bs->kind()) {
- case BarrierSet::G1BarrierSet:
- {
- __ pusha(); // push registers (overkill)
- if (c_rarg0 == count) { // On win64 c_rarg0 == rcx
- assert_different_registers(c_rarg1, start);
- __ mov(c_rarg1, count);
- __ mov(c_rarg0, start);
- } else {
- assert_different_registers(c_rarg0, count);
- __ mov(c_rarg0, start);
- __ mov(c_rarg1, count);
- }
- __ call_VM_leaf(CAST_FROM_FN_PTR(address, BarrierSet::static_write_ref_array_post), 2);
- __ popa();
- }
- break;
- case BarrierSet::CardTableModRef:
- {
- Label L_loop, L_done;
- const Register end = count;
-
- __ testl(count, count);
- __ jcc(Assembler::zero, L_done); // zero count - nothing to do
-
- __ leaq(end, Address(start, count, TIMES_OOP, 0)); // end == start+count*oop_size
- __ subptr(end, BytesPerHeapOop); // end - 1 to make inclusive
- __ shrptr(start, CardTable::card_shift);
- __ shrptr(end, CardTable::card_shift);
- __ subptr(end, start); // end --> cards count
-
- int64_t disp = ci_card_table_address_as<int64_t>();
- __ mov64(scratch, disp);
- __ addptr(start, scratch);
- __ BIND(L_loop);
- __ movb(Address(start, count, Address::times_1), 0);
- __ decrement(count);
- __ jcc(Assembler::greaterEqual, L_loop);
- __ BIND(L_done);
- }
- break;
- default:
- ShouldNotReachHere();
-
- }
- }
-
// Copy big chunks forward
//
// Inputs:
// end_from - source arrays end address
***************
const Register count = rdx; // elements count
const Register dword_count = rcx;
const Register qword_count = count;
const Register end_from = from; // source array end address
const Register end_to = to; // destination array end address
- const Register saved_to = r11; // saved destination array address
// End pointers are inclusive, and if count is not zero they point
// to the last unit copied: end_to[0] := end_from[0]
__ enter(); // required for proper stackwalking of RuntimeStub frame
assert_clean_int(c_rarg2, rax); // Make sure 'count' is clean int.
***************
BLOCK_COMMENT("Entry:");
}
setup_arg_regs(); // from => rdi, to => rsi, count => rdx
// r9 and r10 may be used to save non-volatile registers
! if (is_oop) {
! __ movq(saved_to, to);
! gen_write_ref_array_pre_barrier(to, count, dest_uninitialized);
}
// 'from', 'to' and 'count' are now valid
__ movptr(dword_count, count);
__ shrptr(count, 1); // count => qword_count
// Copy from low to high addresses. Use 'to' as scratch.
BLOCK_COMMENT("Entry:");
}
setup_arg_regs(); // from => rdi, to => rsi, count => rdx
// r9 and r10 may be used to save non-volatile registers
!
! BarrierSetCodeGen *bs = Universe::heap()->barrier_set()->code_gen();
! DecoratorSet decorators = ARRAYCOPY_DISJOINT;
! BasicType type = is_oop ? T_OBJECT : T_INT;
! if (dest_uninitialized) {
! decorators |= AS_DEST_NOT_INITIALIZED;
! }
! if (aligned) {
! decorators |= ARRAYCOPY_ALIGNED;
}
+ bs->arraycopy_prologue(_masm, decorators, type, from, to, count);
+
// 'from', 'to' and 'count' are now valid
__ movptr(dword_count, count);
__ shrptr(count, 1); // count => qword_count
// Copy from low to high addresses. Use 'to' as scratch.
***************
__ jccb(Assembler::zero, L_exit);
__ movl(rax, Address(end_from, 8));
__ movl(Address(end_to, 8), rax);
__ BIND(L_exit);
! if (is_oop) {
! gen_write_ref_array_post_barrier(saved_to, dword_count, rax);
! }
restore_arg_regs();
inc_counter_np(SharedRuntime::_jint_array_copy_ctr); // Update counter after rscratch1 is free
__ vzeroupper();
__ xorptr(rax, rax); // return 0
__ leave(); // required for proper stackwalking of RuntimeStub frame
__ jccb(Assembler::zero, L_exit);
__ movl(rax, Address(end_from, 8));
__ movl(Address(end_to, 8), rax);
__ BIND(L_exit);
! bs->arraycopy_epilogue(_masm, decorators, type, from, to, dword_count);
restore_arg_regs();
inc_counter_np(SharedRuntime::_jint_array_copy_ctr); // Update counter after rscratch1 is free
__ vzeroupper();
__ xorptr(rax, rax); // return 0
__ leave(); // required for proper stackwalking of RuntimeStub frame
***************
array_overlap_test(nooverlap_target, Address::times_4);
setup_arg_regs(); // from => rdi, to => rsi, count => rdx
// r9 and r10 may be used to save non-volatile registers
! if (is_oop) {
! // no registers are destroyed by this call
! gen_write_ref_array_pre_barrier(to, count, dest_uninitialized);
}
assert_clean_int(count, rax); // Make sure 'count' is clean int.
// 'from', 'to' and 'count' are now valid
__ movptr(dword_count, count);
__ shrptr(count, 1); // count => qword_count
array_overlap_test(nooverlap_target, Address::times_4);
setup_arg_regs(); // from => rdi, to => rsi, count => rdx
// r9 and r10 may be used to save non-volatile registers
! BarrierSetCodeGen *bs = Universe::heap()->barrier_set()->code_gen();
! DecoratorSet decorators = 0;
! BasicType type = is_oop ? T_OBJECT : T_INT;
! if (dest_uninitialized) {
! decorators |= AS_DEST_NOT_INITIALIZED;
}
+ if (aligned) {
+ decorators |= ARRAYCOPY_ALIGNED;
+ }
+
+ // no registers are destroyed by this call
+ bs->arraycopy_prologue(_masm, decorators, type, from, to, count);
assert_clean_int(count, rax); // Make sure 'count' is clean int.
// 'from', 'to' and 'count' are now valid
__ movptr(dword_count, count);
__ shrptr(count, 1); // count => qword_count
***************
// Copy in multi-bytes chunks
copy_bytes_backward(from, to, qword_count, rax, L_copy_bytes, L_copy_8_bytes);
__ BIND(L_exit);
! if (is_oop) {
! gen_write_ref_array_post_barrier(to, dword_count, rax);
! }
restore_arg_regs();
inc_counter_np(SharedRuntime::_jint_array_copy_ctr); // Update counter after rscratch1 is free
__ xorptr(rax, rax); // return 0
__ vzeroupper();
__ leave(); // required for proper stackwalking of RuntimeStub frame
// Copy in multi-bytes chunks
copy_bytes_backward(from, to, qword_count, rax, L_copy_bytes, L_copy_8_bytes);
__ BIND(L_exit);
! bs->arraycopy_epilogue(_masm, decorators, type, from, to, dword_count);
restore_arg_regs();
inc_counter_np(SharedRuntime::_jint_array_copy_ctr); // Update counter after rscratch1 is free
__ xorptr(rax, rax); // return 0
__ vzeroupper();
__ leave(); // required for proper stackwalking of RuntimeStub frame
***************
const Register from = rdi; // source array address
const Register to = rsi; // destination array address
const Register qword_count = rdx; // elements count
const Register end_from = from; // source array end address
const Register end_to = rcx; // destination array end address
- const Register saved_to = to;
const Register saved_count = r11;
// End pointers are inclusive, and if count is not zero they point
// to the last unit copied: end_to[0] := end_from[0]
__ enter(); // required for proper stackwalking of RuntimeStub frame
***************
}
setup_arg_regs(); // from => rdi, to => rsi, count => rdx
// r9 and r10 may be used to save non-volatile registers
// 'from', 'to' and 'qword_count' are now valid
! if (is_oop) {
! // Save to and count for store barrier
! __ movptr(saved_count, qword_count);
! // no registers are destroyed by this call
! gen_write_ref_array_pre_barrier(to, qword_count, dest_uninitialized);
}
// Copy from low to high addresses. Use 'to' as scratch.
__ lea(end_from, Address(from, qword_count, Address::times_8, -8));
__ lea(end_to, Address(to, qword_count, Address::times_8, -8));
__ negptr(qword_count);
__ jmp(L_copy_bytes);
}
setup_arg_regs(); // from => rdi, to => rsi, count => rdx
// r9 and r10 may be used to save non-volatile registers
// 'from', 'to' and 'qword_count' are now valid
!
! BarrierSetCodeGen *bs = Universe::heap()->barrier_set()->code_gen();
! DecoratorSet decorators = ARRAYCOPY_DISJOINT;
! BasicType type = is_oop ? T_OBJECT : T_LONG;
! if (dest_uninitialized) {
! decorators |= AS_DEST_NOT_INITIALIZED;
! }
! if (aligned) {
! decorators |= ARRAYCOPY_ALIGNED;
}
+ bs->arraycopy_prologue(_masm, decorators, type, from, to, qword_count);
+
// Copy from low to high addresses. Use 'to' as scratch.
__ lea(end_from, Address(from, qword_count, Address::times_8, -8));
__ lea(end_to, Address(to, qword_count, Address::times_8, -8));
__ negptr(qword_count);
__ jmp(L_copy_bytes);
***************
}
// Copy in multi-bytes chunks
copy_bytes_forward(end_from, end_to, qword_count, rax, L_copy_bytes, L_copy_8_bytes);
- if (is_oop) {
__ BIND(L_exit);
! gen_write_ref_array_post_barrier(saved_to, saved_count, rax);
! }
restore_arg_regs();
if (is_oop) {
inc_counter_np(SharedRuntime::_oop_array_copy_ctr); // Update counter after rscratch1 is free
} else {
inc_counter_np(SharedRuntime::_jlong_array_copy_ctr); // Update counter after rscratch1 is free
}
// Copy in multi-bytes chunks
copy_bytes_forward(end_from, end_to, qword_count, rax, L_copy_bytes, L_copy_8_bytes);
__ BIND(L_exit);
! bs->arraycopy_epilogue(_masm, decorators, type, from, to, qword_count);
restore_arg_regs();
if (is_oop) {
inc_counter_np(SharedRuntime::_oop_array_copy_ctr); // Update counter after rscratch1 is free
} else {
inc_counter_np(SharedRuntime::_jlong_array_copy_ctr); // Update counter after rscratch1 is free
***************
array_overlap_test(nooverlap_target, Address::times_8);
setup_arg_regs(); // from => rdi, to => rsi, count => rdx
// r9 and r10 may be used to save non-volatile registers
// 'from', 'to' and 'qword_count' are now valid
! if (is_oop) {
! // Save to and count for store barrier
! __ movptr(saved_count, qword_count);
! // No registers are destroyed by this call
! gen_write_ref_array_pre_barrier(to, saved_count, dest_uninitialized);
}
__ jmp(L_copy_bytes);
// Copy trailing qwords
__ BIND(L_copy_8_bytes);
array_overlap_test(nooverlap_target, Address::times_8);
setup_arg_regs(); // from => rdi, to => rsi, count => rdx
// r9 and r10 may be used to save non-volatile registers
// 'from', 'to' and 'qword_count' are now valid
!
! BarrierSetCodeGen *bs = Universe::heap()->barrier_set()->code_gen();
! DecoratorSet decorators = ARRAYCOPY_DISJOINT;
! BasicType type = is_oop ? T_OBJECT : T_LONG;
! if (dest_uninitialized) {
! decorators |= AS_DEST_NOT_INITIALIZED;
}
+ if (aligned) {
+ decorators |= ARRAYCOPY_ALIGNED;
+ }
+
+ bs->arraycopy_prologue(_masm, decorators, type, from, to, qword_count);
__ jmp(L_copy_bytes);
// Copy trailing qwords
__ BIND(L_copy_8_bytes);
***************
}
// Copy in multi-bytes chunks
copy_bytes_backward(from, to, qword_count, rax, L_copy_bytes, L_copy_8_bytes);
- if (is_oop) {
__ BIND(L_exit);
! gen_write_ref_array_post_barrier(to, saved_count, rax);
! }
restore_arg_regs();
if (is_oop) {
inc_counter_np(SharedRuntime::_oop_array_copy_ctr); // Update counter after rscratch1 is free
} else {
inc_counter_np(SharedRuntime::_jlong_array_copy_ctr); // Update counter after rscratch1 is free
}
// Copy in multi-bytes chunks
copy_bytes_backward(from, to, qword_count, rax, L_copy_bytes, L_copy_8_bytes);
__ BIND(L_exit);
! bs->arraycopy_epilogue(_masm, decorators, type, from, to, qword_count);
restore_arg_regs();
if (is_oop) {
inc_counter_np(SharedRuntime::_oop_array_copy_ctr); // Update counter after rscratch1 is free
} else {
inc_counter_np(SharedRuntime::_jlong_array_copy_ctr); // Update counter after rscratch1 is free
***************
Address end_to_addr(to, length, TIMES_OOP, 0);
// Loop-variant addresses. They assume post-incremented count < 0.
Address from_element_addr(end_from, count, TIMES_OOP, 0);
Address to_element_addr(end_to, count, TIMES_OOP, 0);
! gen_write_ref_array_pre_barrier(to, count, dest_uninitialized);
// Copy from low to high addresses, indexed from the end of each array.
__ lea(end_from, end_from_addr);
__ lea(end_to, end_to_addr);
__ movptr(r14_length, length); // save a copy of the length
Address end_to_addr(to, length, TIMES_OOP, 0);
// Loop-variant addresses. They assume post-incremented count < 0.
Address from_element_addr(end_from, count, TIMES_OOP, 0);
Address to_element_addr(end_to, count, TIMES_OOP, 0);
! BarrierSetCodeGen *bs = Universe::heap()->barrier_set()->code_gen();
! DecoratorSet decorators = ARRAYCOPY_CHECKCAST;
! BasicType type = T_OBJECT;
! if (dest_uninitialized) {
! decorators |= AS_DEST_NOT_INITIALIZED;
! }
!
! bs->arraycopy_prologue(_masm, decorators, type, from, to, count);
// Copy from low to high addresses, indexed from the end of each array.
__ lea(end_from, end_from_addr);
__ lea(end_to, end_to_addr);
__ movptr(r14_length, length); // save a copy of the length
***************
// Come here on success only.
__ BIND(L_do_card_marks);
__ xorptr(rax, rax); // return 0 on success
__ BIND(L_post_barrier);
! gen_write_ref_array_post_barrier(to, r14_length, rscratch1);
// Common exit point (success or failure).
__ BIND(L_done);
__ movptr(r13, Address(rsp, saved_r13_offset * wordSize));
__ movptr(r14, Address(rsp, saved_r14_offset * wordSize));
// Come here on success only.
__ BIND(L_do_card_marks);
__ xorptr(rax, rax); // return 0 on success
__ BIND(L_post_barrier);
! bs->arraycopy_epilogue(_masm, decorators, type, from, to, r14_length);
// Common exit point (success or failure).
__ BIND(L_done);
__ movptr(r13, Address(rsp, saved_r13_offset * wordSize));
__ movptr(r14, Address(rsp, saved_r14_offset * wordSize));
< prev index next >