diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp index 4037287..8d5f474 100644 --- a/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp +++ b/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp @@ -26,8 +26,8 @@ #include "asm/macroAssembler.hpp" #include "asm/macroAssembler.inline.hpp" #include "ci/ciUtilities.hpp" -#include "gc/shared/cardTable.hpp" -#include "gc/shared/cardTableModRefBS.hpp" +#include "gc/shared/barrierSet.hpp" +#include "gc/shared/barrierSetCodeGen.hpp" #include "interpreter/interpreter.hpp" #include "nativeInst_x86.hpp" #include "oops/instanceOop.hpp" @@ -1190,119 +1190,6 @@ class StubGenerator: public StubCodeGenerator { #endif } - // Generate code for an array write pre barrier - // - // addr - starting address - // count - element count - // tmp - scratch register - // - // Destroy no registers! - // - void gen_write_ref_array_pre_barrier(Register addr, Register count, bool dest_uninitialized) { - BarrierSet* bs = Universe::heap()->barrier_set(); - switch (bs->kind()) { - case BarrierSet::G1BarrierSet: - // With G1, don't generate the call if we statically know that the target in uninitialized - if (!dest_uninitialized) { - Label filtered; - Address in_progress(r15_thread, in_bytes(JavaThread::satb_mark_queue_offset() + - SATBMarkQueue::byte_offset_of_active())); - // Is marking active? - if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) { - __ cmpl(in_progress, 0); - } else { - assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption"); - __ cmpb(in_progress, 0); - } - __ jcc(Assembler::equal, filtered); - - __ pusha(); // push registers - if (count == c_rarg0) { - if (addr == c_rarg1) { - // exactly backwards!! - __ xchgptr(c_rarg1, c_rarg0); - } else { - __ movptr(c_rarg1, count); - __ movptr(c_rarg0, addr); - } - } else { - __ movptr(c_rarg0, addr); - __ movptr(c_rarg1, count); - } - __ call_VM_leaf(CAST_FROM_FN_PTR(address, BarrierSet::static_write_ref_array_pre), 2); - __ popa(); - - __ bind(filtered); - } - break; - case BarrierSet::CardTableModRef: - break; - default: - ShouldNotReachHere(); - - } - } - - // - // Generate code for an array write post barrier - // - // Input: - // start - register containing starting address of destination array - // count - elements count - // scratch - scratch register - // - // The input registers are overwritten. - // - void gen_write_ref_array_post_barrier(Register start, Register count, Register scratch) { - assert_different_registers(start, count, scratch); - BarrierSet* bs = Universe::heap()->barrier_set(); - switch (bs->kind()) { - case BarrierSet::G1BarrierSet: - { - __ pusha(); // push registers (overkill) - if (c_rarg0 == count) { // On win64 c_rarg0 == rcx - assert_different_registers(c_rarg1, start); - __ mov(c_rarg1, count); - __ mov(c_rarg0, start); - } else { - assert_different_registers(c_rarg0, count); - __ mov(c_rarg0, start); - __ mov(c_rarg1, count); - } - __ call_VM_leaf(CAST_FROM_FN_PTR(address, BarrierSet::static_write_ref_array_post), 2); - __ popa(); - } - break; - case BarrierSet::CardTableModRef: - { - Label L_loop, L_done; - const Register end = count; - - __ testl(count, count); - __ jcc(Assembler::zero, L_done); // zero count - nothing to do - - __ leaq(end, Address(start, count, TIMES_OOP, 0)); // end == start+count*oop_size - __ subptr(end, BytesPerHeapOop); // end - 1 to make inclusive - __ shrptr(start, CardTable::card_shift); - __ shrptr(end, CardTable::card_shift); - __ subptr(end, start); // end --> cards count - - int64_t disp = ci_card_table_address_as(); - __ mov64(scratch, disp); - __ addptr(start, scratch); - __ BIND(L_loop); - __ movb(Address(start, count, Address::times_1), 0); - __ decrement(count); - __ jcc(Assembler::greaterEqual, L_loop); - __ BIND(L_done); - } - break; - default: - ShouldNotReachHere(); - - } - } - // Copy big chunks forward // @@ -1918,7 +1805,6 @@ class StubGenerator: public StubCodeGenerator { const Register qword_count = count; const Register end_from = from; // source array end address const Register end_to = to; // destination array end address - const Register saved_to = r11; // saved destination array address // End pointers are inclusive, and if count is not zero they point // to the last unit copied: end_to[0] := end_from[0] @@ -1933,10 +1819,18 @@ class StubGenerator: public StubCodeGenerator { setup_arg_regs(); // from => rdi, to => rsi, count => rdx // r9 and r10 may be used to save non-volatile registers - if (is_oop) { - __ movq(saved_to, to); - gen_write_ref_array_pre_barrier(to, count, dest_uninitialized); + + BarrierSetCodeGen *bs = Universe::heap()->barrier_set()->code_gen(); + DecoratorSet decorators = ARRAYCOPY_DISJOINT; + BasicType type = is_oop ? T_OBJECT : T_INT; + if (dest_uninitialized) { + decorators |= AS_DEST_NOT_INITIALIZED; } + if (aligned) { + decorators |= ARRAYCOPY_ALIGNED; + } + + bs->arraycopy_prologue(_masm, decorators, type, from, to, count); // 'from', 'to' and 'count' are now valid __ movptr(dword_count, count); @@ -1963,9 +1857,7 @@ class StubGenerator: public StubCodeGenerator { __ movl(Address(end_to, 8), rax); __ BIND(L_exit); - if (is_oop) { - gen_write_ref_array_post_barrier(saved_to, dword_count, rax); - } + bs->arraycopy_epilogue(_masm, decorators, type, from, to, dword_count); restore_arg_regs(); inc_counter_np(SharedRuntime::_jint_array_copy_ctr); // Update counter after rscratch1 is free __ vzeroupper(); @@ -2022,10 +1914,18 @@ class StubGenerator: public StubCodeGenerator { setup_arg_regs(); // from => rdi, to => rsi, count => rdx // r9 and r10 may be used to save non-volatile registers - if (is_oop) { - // no registers are destroyed by this call - gen_write_ref_array_pre_barrier(to, count, dest_uninitialized); + BarrierSetCodeGen *bs = Universe::heap()->barrier_set()->code_gen(); + DecoratorSet decorators = 0; + BasicType type = is_oop ? T_OBJECT : T_INT; + if (dest_uninitialized) { + decorators |= AS_DEST_NOT_INITIALIZED; } + if (aligned) { + decorators |= ARRAYCOPY_ALIGNED; + } + + // no registers are destroyed by this call + bs->arraycopy_prologue(_masm, decorators, type, from, to, count); assert_clean_int(count, rax); // Make sure 'count' is clean int. // 'from', 'to' and 'count' are now valid @@ -2062,9 +1962,7 @@ class StubGenerator: public StubCodeGenerator { copy_bytes_backward(from, to, qword_count, rax, L_copy_bytes, L_copy_8_bytes); __ BIND(L_exit); - if (is_oop) { - gen_write_ref_array_post_barrier(to, dword_count, rax); - } + bs->arraycopy_epilogue(_masm, decorators, type, from, to, dword_count); restore_arg_regs(); inc_counter_np(SharedRuntime::_jint_array_copy_ctr); // Update counter after rscratch1 is free __ xorptr(rax, rax); // return 0 @@ -2102,7 +2000,6 @@ class StubGenerator: public StubCodeGenerator { const Register qword_count = rdx; // elements count const Register end_from = from; // source array end address const Register end_to = rcx; // destination array end address - const Register saved_to = to; const Register saved_count = r11; // End pointers are inclusive, and if count is not zero they point // to the last unit copied: end_to[0] := end_from[0] @@ -2120,12 +2017,18 @@ class StubGenerator: public StubCodeGenerator { setup_arg_regs(); // from => rdi, to => rsi, count => rdx // r9 and r10 may be used to save non-volatile registers // 'from', 'to' and 'qword_count' are now valid - if (is_oop) { - // Save to and count for store barrier - __ movptr(saved_count, qword_count); - // no registers are destroyed by this call - gen_write_ref_array_pre_barrier(to, qword_count, dest_uninitialized); + + BarrierSetCodeGen *bs = Universe::heap()->barrier_set()->code_gen(); + DecoratorSet decorators = ARRAYCOPY_DISJOINT; + BasicType type = is_oop ? T_OBJECT : T_LONG; + if (dest_uninitialized) { + decorators |= AS_DEST_NOT_INITIALIZED; } + if (aligned) { + decorators |= ARRAYCOPY_ALIGNED; + } + + bs->arraycopy_prologue(_masm, decorators, type, from, to, qword_count); // Copy from low to high addresses. Use 'to' as scratch. __ lea(end_from, Address(from, qword_count, Address::times_8, -8)); @@ -2154,10 +2057,8 @@ class StubGenerator: public StubCodeGenerator { // Copy in multi-bytes chunks copy_bytes_forward(end_from, end_to, qword_count, rax, L_copy_bytes, L_copy_8_bytes); - if (is_oop) { __ BIND(L_exit); - gen_write_ref_array_post_barrier(saved_to, saved_count, rax); - } + bs->arraycopy_epilogue(_masm, decorators, type, from, to, qword_count); restore_arg_regs(); if (is_oop) { inc_counter_np(SharedRuntime::_oop_array_copy_ctr); // Update counter after rscratch1 is free @@ -2209,12 +2110,18 @@ class StubGenerator: public StubCodeGenerator { setup_arg_regs(); // from => rdi, to => rsi, count => rdx // r9 and r10 may be used to save non-volatile registers // 'from', 'to' and 'qword_count' are now valid - if (is_oop) { - // Save to and count for store barrier - __ movptr(saved_count, qword_count); - // No registers are destroyed by this call - gen_write_ref_array_pre_barrier(to, saved_count, dest_uninitialized); + + BarrierSetCodeGen *bs = Universe::heap()->barrier_set()->code_gen(); + DecoratorSet decorators = ARRAYCOPY_DISJOINT; + BasicType type = is_oop ? T_OBJECT : T_LONG; + if (dest_uninitialized) { + decorators |= AS_DEST_NOT_INITIALIZED; } + if (aligned) { + decorators |= ARRAYCOPY_ALIGNED; + } + + bs->arraycopy_prologue(_masm, decorators, type, from, to, qword_count); __ jmp(L_copy_bytes); @@ -2239,10 +2146,8 @@ class StubGenerator: public StubCodeGenerator { // Copy in multi-bytes chunks copy_bytes_backward(from, to, qword_count, rax, L_copy_bytes, L_copy_8_bytes); - if (is_oop) { __ BIND(L_exit); - gen_write_ref_array_post_barrier(to, saved_count, rax); - } + bs->arraycopy_epilogue(_masm, decorators, type, from, to, qword_count); restore_arg_regs(); if (is_oop) { inc_counter_np(SharedRuntime::_oop_array_copy_ctr); // Update counter after rscratch1 is free @@ -2389,7 +2294,14 @@ class StubGenerator: public StubCodeGenerator { Address from_element_addr(end_from, count, TIMES_OOP, 0); Address to_element_addr(end_to, count, TIMES_OOP, 0); - gen_write_ref_array_pre_barrier(to, count, dest_uninitialized); + BarrierSetCodeGen *bs = Universe::heap()->barrier_set()->code_gen(); + DecoratorSet decorators = ARRAYCOPY_CHECKCAST; + BasicType type = T_OBJECT; + if (dest_uninitialized) { + decorators |= AS_DEST_NOT_INITIALIZED; + } + + bs->arraycopy_prologue(_masm, decorators, type, from, to, count); // Copy from low to high addresses, indexed from the end of each array. __ lea(end_from, end_from_addr); @@ -2442,7 +2354,7 @@ class StubGenerator: public StubCodeGenerator { __ xorptr(rax, rax); // return 0 on success __ BIND(L_post_barrier); - gen_write_ref_array_post_barrier(to, r14_length, rscratch1); + bs->arraycopy_epilogue(_masm, decorators, type, from, to, r14_length); // Common exit point (success or failure). __ BIND(L_done);