--- old/src/cpu/x86/vm/stubGenerator_x86_64.cpp 2017-04-25 16:43:57.311176506 +0200 +++ new/src/cpu/x86/vm/stubGenerator_x86_64.cpp 2017-04-25 16:43:57.163176511 +0200 @@ -25,6 +25,7 @@ #include "precompiled.hpp" #include "asm/macroAssembler.hpp" #include "asm/macroAssembler.inline.hpp" +#include "gc/shared/barrierSetCodeGen.hpp" #include "interpreter/interpreter.hpp" #include "nativeInst_x86.hpp" #include "oops/instanceOop.hpp" @@ -1186,108 +1187,6 @@ #endif } - // Generate code for an array write pre barrier - // - // addr - starting address - // count - element count - // tmp - scratch register - // - // Destroy no registers! - // - void gen_write_ref_array_pre_barrier(Register addr, Register count, bool dest_uninitialized) { - BarrierSet* bs = Universe::heap()->barrier_set(); - switch (bs->kind()) { - case BarrierSet::G1SATBCTLogging: - // With G1, don't generate the call if we statically know that the target in uninitialized - if (!dest_uninitialized) { - __ pusha(); // push registers - if (count == c_rarg0) { - if (addr == c_rarg1) { - // exactly backwards!! - __ xchgptr(c_rarg1, c_rarg0); - } else { - __ movptr(c_rarg1, count); - __ movptr(c_rarg0, addr); - } - } else { - __ movptr(c_rarg0, addr); - __ movptr(c_rarg1, count); - } - __ call_VM_leaf(CAST_FROM_FN_PTR(address, BarrierSet::static_write_ref_array_pre), 2); - __ popa(); - } - break; - case BarrierSet::CardTableForRS: - case BarrierSet::CardTableExtension: - case BarrierSet::ModRef: - break; - default: - ShouldNotReachHere(); - - } - } - - // - // Generate code for an array write post barrier - // - // Input: - // start - register containing starting address of destination array - // count - elements count - // scratch - scratch register - // - // The input registers are overwritten. - // - void gen_write_ref_array_post_barrier(Register start, Register count, Register scratch) { - assert_different_registers(start, count, scratch); - BarrierSet* bs = Universe::heap()->barrier_set(); - switch (bs->kind()) { - case BarrierSet::G1SATBCTLogging: - { - __ pusha(); // push registers (overkill) - if (c_rarg0 == count) { // On win64 c_rarg0 == rcx - assert_different_registers(c_rarg1, start); - __ mov(c_rarg1, count); - __ mov(c_rarg0, start); - } else { - assert_different_registers(c_rarg0, count); - __ mov(c_rarg0, start); - __ mov(c_rarg1, count); - } - __ call_VM_leaf(CAST_FROM_FN_PTR(address, BarrierSet::static_write_ref_array_post), 2); - __ popa(); - } - break; - case BarrierSet::CardTableForRS: - case BarrierSet::CardTableExtension: - { - CardTableModRefBS* ct = barrier_set_cast(bs); - assert(sizeof(*ct->byte_map_base) == sizeof(jbyte), "adjust this code"); - - Label L_loop; - const Register end = count; - - __ leaq(end, Address(start, count, TIMES_OOP, 0)); // end == start+count*oop_size - __ subptr(end, BytesPerHeapOop); // end - 1 to make inclusive - __ shrptr(start, CardTableModRefBS::card_shift); - __ shrptr(end, CardTableModRefBS::card_shift); - __ subptr(end, start); // end --> cards count - - int64_t disp = (int64_t) ct->byte_map_base; - __ mov64(scratch, disp); - __ addptr(start, scratch); - __ BIND(L_loop); - __ movb(Address(start, count, Address::times_1), 0); - __ decrement(count); - __ jcc(Assembler::greaterEqual, L_loop); - } - break; - default: - ShouldNotReachHere(); - - } - } - - // Copy big chunks forward // // Inputs: @@ -1895,7 +1794,6 @@ const Register qword_count = count; const Register end_from = from; // source array end address const Register end_to = to; // destination array end address - const Register saved_to = r11; // saved destination array address // End pointers are inclusive, and if count is not zero they point // to the last unit copied: end_to[0] := end_from[0] @@ -1910,10 +1808,18 @@ setup_arg_regs(); // from => rdi, to => rsi, count => rdx // r9 and r10 may be used to save non-volatile registers - if (is_oop) { - __ movq(saved_to, to); - gen_write_ref_array_pre_barrier(to, count, dest_uninitialized); + + BarrierSetCodeGen *bs = Universe::heap()->barrier_set()->code_gen(); + DecoratorSet decorators = DEST_COVARIANT | DEST_DISJOINT; + BasicType type = is_oop ? T_OBJECT : T_INT; + if (dest_uninitialized) { + decorators |= DEST_NOT_INITIALIZED; } + if (aligned) { + decorators |= ACCESS_ALIGNED; + } + + bs->arraycopy_prologue(_masm, decorators, type, from, to, count); // 'from', 'to' and 'count' are now valid __ movptr(dword_count, count); @@ -1940,9 +1846,7 @@ __ movl(Address(end_to, 8), rax); __ BIND(L_exit); - if (is_oop) { - gen_write_ref_array_post_barrier(saved_to, dword_count, rax); - } + bs->arraycopy_epilogue(_masm, decorators, type, from, to, dword_count); restore_arg_regs(); inc_counter_np(SharedRuntime::_jint_array_copy_ctr); // Update counter after rscratch1 is free __ xorptr(rax, rax); // return 0 @@ -1998,11 +1902,19 @@ setup_arg_regs(); // from => rdi, to => rsi, count => rdx // r9 and r10 may be used to save non-volatile registers - if (is_oop) { - // no registers are destroyed by this call - gen_write_ref_array_pre_barrier(to, count, dest_uninitialized); + BarrierSetCodeGen *bs = Universe::heap()->barrier_set()->code_gen(); + DecoratorSet decorators = DEST_COVARIANT | DEST_CONJOINT; + BasicType type = is_oop ? T_OBJECT : T_INT; + if (dest_uninitialized) { + decorators |= DEST_NOT_INITIALIZED; + } + if (aligned) { + decorators |= ACCESS_ALIGNED; } + // no registers are destroyed by this call + bs->arraycopy_prologue(_masm, decorators, type, from, to, count); + assert_clean_int(count, rax); // Make sure 'count' is clean int. // 'from', 'to' and 'count' are now valid __ movptr(dword_count, count); @@ -2037,9 +1949,7 @@ copy_bytes_backward(from, to, qword_count, rax, L_copy_bytes, L_copy_8_bytes); __ BIND(L_exit); - if (is_oop) { - gen_write_ref_array_post_barrier(to, dword_count, rax); - } + bs->arraycopy_epilogue(_masm, decorators, type, from, to, dword_count); restore_arg_regs(); inc_counter_np(SharedRuntime::_jint_array_copy_ctr); // Update counter after rscratch1 is free __ xorptr(rax, rax); // return 0 @@ -2076,7 +1986,6 @@ const Register qword_count = rdx; // elements count const Register end_from = from; // source array end address const Register end_to = rcx; // destination array end address - const Register saved_to = to; const Register saved_count = r11; // End pointers are inclusive, and if count is not zero they point // to the last unit copied: end_to[0] := end_from[0] @@ -2094,13 +2003,19 @@ setup_arg_regs(); // from => rdi, to => rsi, count => rdx // r9 and r10 may be used to save non-volatile registers // 'from', 'to' and 'qword_count' are now valid - if (is_oop) { - // Save to and count for store barrier - __ movptr(saved_count, qword_count); - // no registers are destroyed by this call - gen_write_ref_array_pre_barrier(to, qword_count, dest_uninitialized); + + BarrierSetCodeGen *bs = Universe::heap()->barrier_set()->code_gen(); + DecoratorSet decorators = DEST_COVARIANT | DEST_DISJOINT; + BasicType type = is_oop ? T_OBJECT : T_LONG; + if (dest_uninitialized) { + decorators |= DEST_NOT_INITIALIZED; + } + if (aligned) { + decorators |= ACCESS_ALIGNED; } + bs->arraycopy_prologue(_masm, decorators, type, from, to, qword_count); + // Copy from low to high addresses. Use 'to' as scratch. __ lea(end_from, Address(from, qword_count, Address::times_8, -8)); __ lea(end_to, Address(to, qword_count, Address::times_8, -8)); @@ -2127,10 +2042,8 @@ // Copy in multi-bytes chunks copy_bytes_forward(end_from, end_to, qword_count, rax, L_copy_bytes, L_copy_8_bytes); - if (is_oop) { __ BIND(L_exit); - gen_write_ref_array_post_barrier(saved_to, saved_count, rax); - } + bs->arraycopy_epilogue(_masm, decorators, type, from, to, qword_count); restore_arg_regs(); if (is_oop) { inc_counter_np(SharedRuntime::_oop_array_copy_ctr); // Update counter after rscratch1 is free @@ -2181,12 +2094,18 @@ setup_arg_regs(); // from => rdi, to => rsi, count => rdx // r9 and r10 may be used to save non-volatile registers // 'from', 'to' and 'qword_count' are now valid - if (is_oop) { - // Save to and count for store barrier - __ movptr(saved_count, qword_count); - // No registers are destroyed by this call - gen_write_ref_array_pre_barrier(to, saved_count, dest_uninitialized); + + BarrierSetCodeGen *bs = Universe::heap()->barrier_set()->code_gen(); + DecoratorSet decorators = DEST_COVARIANT | DEST_DISJOINT; + BasicType type = is_oop ? T_OBJECT : T_LONG; + if (dest_uninitialized) { + decorators |= DEST_NOT_INITIALIZED; } + if (aligned) { + decorators |= ACCESS_ALIGNED; + } + + bs->arraycopy_prologue(_masm, decorators, type, from, to, qword_count); __ jmp(L_copy_bytes); @@ -2210,10 +2129,8 @@ // Copy in multi-bytes chunks copy_bytes_backward(from, to, qword_count, rax, L_copy_bytes, L_copy_8_bytes); - if (is_oop) { __ BIND(L_exit); - gen_write_ref_array_post_barrier(to, saved_count, rax); - } + bs->arraycopy_epilogue(_masm, decorators, type, from, to, qword_count); restore_arg_regs(); if (is_oop) { inc_counter_np(SharedRuntime::_oop_array_copy_ctr); // Update counter after rscratch1 is free @@ -2359,7 +2276,14 @@ Address from_element_addr(end_from, count, TIMES_OOP, 0); Address to_element_addr(end_to, count, TIMES_OOP, 0); - gen_write_ref_array_pre_barrier(to, count, dest_uninitialized); + BarrierSetCodeGen *bs = Universe::heap()->barrier_set()->code_gen(); + DecoratorSet decorators = DEST_CONTRAVARIANT | DEST_DISJOINT; + BasicType type = T_OBJECT; + if (dest_uninitialized) { + decorators |= DEST_NOT_INITIALIZED; + } + + bs->arraycopy_prologue(_masm, decorators, type, from, to, count); // Copy from low to high addresses, indexed from the end of each array. __ lea(end_from, end_from_addr); @@ -2412,7 +2336,7 @@ __ xorptr(rax, rax); // return 0 on success __ BIND(L_post_barrier); - gen_write_ref_array_post_barrier(to, r14_length, rscratch1); + bs->arraycopy_epilogue(_masm, decorators, type, from, to, r14_length); // Common exit point (success or failure). __ BIND(L_done);