BarrierSetCodeGen_v2 8198949_arraycopy
7 * 8 * This code is distributed in the hope that it will be useful, but WITHOUT 9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 11 * version 2 for more details (a copy is included in the LICENSE file that 12 * accompanied this code). 13 * 14 * You should have received a copy of the GNU General Public License version 15 * 2 along with this work; if not, write to the Free Software Foundation, 16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 17 * 18 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 19 * or visit www.oracle.com if you need additional information or have any 20 * questions. 21 * 22 */ 23 24 #include "precompiled.hpp" 25 #include "asm/macroAssembler.hpp" 26 #include "asm/macroAssembler.inline.hpp" 27 #include "gc/shared/cardTable.hpp" 28 #include "gc/shared/cardTableModRefBS.hpp" 29 #include "interpreter/interpreter.hpp" 30 #include "nativeInst_x86.hpp" 31 #include "oops/instanceOop.hpp" 32 #include "oops/method.hpp" 33 #include "oops/objArrayKlass.hpp" 34 #include "oops/oop.inline.hpp" 35 #include "prims/methodHandles.hpp" 36 #include "runtime/frame.inline.hpp" 37 #include "runtime/handles.inline.hpp" 38 #include "runtime/sharedRuntime.hpp" 39 #include "runtime/stubCodeGenerator.hpp" 40 #include "runtime/stubRoutines.hpp" 41 #include "runtime/thread.inline.hpp" 42 #ifdef COMPILER2 43 #include "opto/runtime.hpp" 44 #endif 45 46 // Declaration and definition of StubGenerator (no .hpp file). 47 // For a more detailed description of the stub routine structure |
7 * 8 * This code is distributed in the hope that it will be useful, but WITHOUT 9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 11 * version 2 for more details (a copy is included in the LICENSE file that 12 * accompanied this code). 13 * 14 * You should have received a copy of the GNU General Public License version 15 * 2 along with this work; if not, write to the Free Software Foundation, 16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 17 * 18 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 19 * or visit www.oracle.com if you need additional information or have any 20 * questions. 21 * 22 */ 23 24 #include "precompiled.hpp" 25 #include "asm/macroAssembler.hpp" 26 #include "asm/macroAssembler.inline.hpp" 27 #include "gc/shared/barrierSet.hpp" 28 #include "gc/shared/barrierSetCodeGen.hpp" 29 #include "interpreter/interpreter.hpp" 30 #include "nativeInst_x86.hpp" 31 #include "oops/instanceOop.hpp" 32 #include "oops/method.hpp" 33 #include "oops/objArrayKlass.hpp" 34 #include "oops/oop.inline.hpp" 35 #include "prims/methodHandles.hpp" 36 #include "runtime/frame.inline.hpp" 37 #include "runtime/handles.inline.hpp" 38 #include "runtime/sharedRuntime.hpp" 39 #include "runtime/stubCodeGenerator.hpp" 40 #include "runtime/stubRoutines.hpp" 41 #include "runtime/thread.inline.hpp" 42 #ifdef COMPILER2 43 #include "opto/runtime.hpp" 44 #endif 45 46 // Declaration and definition of StubGenerator (no .hpp file). 47 // For a more detailed description of the stub routine structure |
650 // return if everything seems ok
651 __ bind(exit);
652 __ movptr(rax, Address(rsp, 5 * wordSize)); // get saved rax, back
653 __ pop(rdx); // restore rdx
654 __ popf(); // restore EFLAGS
655 __ ret(3 * wordSize); // pop arguments
656
657 // handle errors
658 __ bind(error);
659 __ movptr(rax, Address(rsp, 5 * wordSize)); // get saved rax, back
660 __ pop(rdx); // get saved rdx back
661 __ popf(); // get saved EFLAGS off stack -- will be ignored
662 __ pusha(); // push registers (eip = return address & msg are already pushed)
663 BLOCK_COMMENT("call MacroAssembler::debug");
664 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::debug32)));
665 __ popa();
666 __ ret(3 * wordSize); // pop arguments
667 return start;
668 }
669
670 //
671 // Generate pre-barrier for array stores
672 //
673 // Input:
674 // start - starting address
675 // count - element count
676 void gen_write_ref_array_pre_barrier(Register start, Register count, bool uninitialized_target) {
677 assert_different_registers(start, count);
678 BarrierSet* bs = Universe::heap()->barrier_set();
679 switch (bs->kind()) {
680 #if INCLUDE_ALL_GCS
681 case BarrierSet::G1BarrierSet:
682 // With G1, don't generate the call if we statically know that the target in uninitialized
683 if (!uninitialized_target) {
684 Register thread = rax;
685 Label filtered;
686 __ push(thread);
687 __ get_thread(thread);
688 Address in_progress(thread, in_bytes(JavaThread::satb_mark_queue_offset() +
689 SATBMarkQueue::byte_offset_of_active()));
690 // Is marking active?
691 if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) {
692 __ cmpl(in_progress, 0);
693 } else {
694 assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption");
695 __ cmpb(in_progress, 0);
696 }
697 __ pop(thread);
698 __ jcc(Assembler::equal, filtered);
699
700 __ pusha(); // push registers
701 __ call_VM_leaf(CAST_FROM_FN_PTR(address, BarrierSet::static_write_ref_array_pre),
702 start, count);
703 __ popa();
704
705 __ bind(filtered);
706 }
707 break;
708 #endif // INCLUDE_ALL_GCS
709 case BarrierSet::CardTableModRef:
710 break;
711 default :
712 ShouldNotReachHere();
713
714 }
715 }
716
717
718 //
719 // Generate a post-barrier for an array store
720 //
721 // start - starting address
722 // count - element count
723 //
724 // The two input registers are overwritten.
725 //
726 void gen_write_ref_array_post_barrier(Register start, Register count) {
727 BarrierSet* bs = Universe::heap()->barrier_set();
728 assert_different_registers(start, count);
729 switch (bs->kind()) {
730 #if INCLUDE_ALL_GCS
731 case BarrierSet::G1BarrierSet:
732 {
733 __ pusha(); // push registers
734 __ call_VM_leaf(CAST_FROM_FN_PTR(address, BarrierSet::static_write_ref_array_post),
735 start, count);
736 __ popa();
737 }
738 break;
739 #endif // INCLUDE_ALL_GCS
740
741 case BarrierSet::CardTableModRef:
742 {
743 CardTableModRefBS* ctbs = barrier_set_cast<CardTableModRefBS>(bs);
744 CardTable* ct = ctbs->card_table();
745 assert(sizeof(*ct->byte_map_base()) == sizeof(jbyte), "adjust this code");
746
747 Label L_loop;
748 const Register end = count; // elements count; end == start+count-1
749 assert_different_registers(start, end);
750
751 __ lea(end, Address(start, count, Address::times_ptr, -wordSize));
752 __ shrptr(start, CardTable::card_shift);
753 __ shrptr(end, CardTable::card_shift);
754 __ subptr(end, start); // end --> count
755 __ BIND(L_loop);
756 intptr_t disp = (intptr_t) ct->byte_map_base();
757 Address cardtable(start, count, Address::times_1, disp);
758 __ movb(cardtable, 0);
759 __ decrement(count);
760 __ jcc(Assembler::greaterEqual, L_loop);
761 }
762 break;
763 case BarrierSet::ModRef:
764 break;
765 default :
766 ShouldNotReachHere();
767
768 }
769 }
770
771
772 // Copy 64 bytes chunks
773 //
774 // Inputs:
775 // from - source array address
776 // to_from - destination array address - from
777 // qword_count - 8-bytes element count, negative
778 //
779 void xmm_copy_forward(Register from, Register to_from, Register qword_count) {
780 assert( UseSSE >= 2, "supported cpu only" );
781 Label L_copy_64_bytes_loop, L_copy_64_bytes, L_copy_8_bytes, L_exit;
782 if (UseAVX > 2) {
783 __ push(rbx);
784 __ movl(rbx, 0xffff);
785 __ kmovwl(k1, rbx);
786 __ pop(rbx);
787 }
788 // Copy 64-byte chunks
789 __ jmpb(L_copy_64_bytes);
|
650 // return if everything seems ok
651 __ bind(exit);
652 __ movptr(rax, Address(rsp, 5 * wordSize)); // get saved rax, back
653 __ pop(rdx); // restore rdx
654 __ popf(); // restore EFLAGS
655 __ ret(3 * wordSize); // pop arguments
656
657 // handle errors
658 __ bind(error);
659 __ movptr(rax, Address(rsp, 5 * wordSize)); // get saved rax, back
660 __ pop(rdx); // get saved rdx back
661 __ popf(); // get saved EFLAGS off stack -- will be ignored
662 __ pusha(); // push registers (eip = return address & msg are already pushed)
663 BLOCK_COMMENT("call MacroAssembler::debug");
664 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::debug32)));
665 __ popa();
666 __ ret(3 * wordSize); // pop arguments
667 return start;
668 }
669
670
671 // Copy 64 bytes chunks
672 //
673 // Inputs:
674 // from - source array address
675 // to_from - destination array address - from
676 // qword_count - 8-bytes element count, negative
677 //
678 void xmm_copy_forward(Register from, Register to_from, Register qword_count) {
679 assert( UseSSE >= 2, "supported cpu only" );
680 Label L_copy_64_bytes_loop, L_copy_64_bytes, L_copy_8_bytes, L_exit;
681 if (UseAVX > 2) {
682 __ push(rbx);
683 __ movl(rbx, 0xffff);
684 __ kmovwl(k1, rbx);
685 __ pop(rbx);
686 }
687 // Copy 64-byte chunks
688 __ jmpb(L_copy_64_bytes);
|
918 const Register to = rdi; // destination array address
919 const Register count = rcx; // elements count
920 const Register to_from = to; // (to - from)
921 const Register saved_to = rdx; // saved destination array address
922
923 __ enter(); // required for proper stackwalking of RuntimeStub frame
924 __ push(rsi);
925 __ push(rdi);
926 __ movptr(from , Address(rsp, 12+ 4));
927 __ movptr(to , Address(rsp, 12+ 8));
928 __ movl(count, Address(rsp, 12+ 12));
929
930 if (entry != NULL) {
931 *entry = __ pc(); // Entry point from conjoint arraycopy stub.
932 BLOCK_COMMENT("Entry:");
933 }
934
935 if (t == T_OBJECT) {
936 __ testl(count, count);
937 __ jcc(Assembler::zero, L_0_count);
938 gen_write_ref_array_pre_barrier(to, count, dest_uninitialized);
939 __ mov(saved_to, to); // save 'to'
940 }
941
942 __ subptr(to, from); // to --> to_from
943 __ cmpl(count, 2<<shift); // Short arrays (< 8 bytes) copy by element
944 __ jcc(Assembler::below, L_copy_4_bytes); // use unsigned cmp
945 if (!UseUnalignedLoadStores && !aligned && (t == T_BYTE || t == T_SHORT)) {
946 // align source address at 4 bytes address boundary
947 if (t == T_BYTE) {
948 // One byte misalignment happens only for byte arrays
949 __ testl(from, 1);
950 __ jccb(Assembler::zero, L_skip_align1);
951 __ movb(rax, Address(from, 0));
952 __ movb(Address(from, to_from, Address::times_1, 0), rax);
953 __ increment(from);
954 __ decrement(count);
955 __ BIND(L_skip_align1);
956 }
957 // Two bytes misalignment happens only for byte and short (char) arrays
958 __ testl(from, 2);
959 __ jccb(Assembler::zero, L_skip_align2);
960 __ movw(rax, Address(from, 0));
|
817 const Register to = rdi; // destination array address
818 const Register count = rcx; // elements count
819 const Register to_from = to; // (to - from)
820 const Register saved_to = rdx; // saved destination array address
821
822 __ enter(); // required for proper stackwalking of RuntimeStub frame
823 __ push(rsi);
824 __ push(rdi);
825 __ movptr(from , Address(rsp, 12+ 4));
826 __ movptr(to , Address(rsp, 12+ 8));
827 __ movl(count, Address(rsp, 12+ 12));
828
829 if (entry != NULL) {
830 *entry = __ pc(); // Entry point from conjoint arraycopy stub.
831 BLOCK_COMMENT("Entry:");
832 }
833
834 if (t == T_OBJECT) {
835 __ testl(count, count);
836 __ jcc(Assembler::zero, L_0_count);
837 }
838
839 DecoratorSet decorators = ARRAYCOPY_DISJOINT;
840 if (dest_uninitialized) {
841 decorators |= AS_DEST_NOT_INITIALIZED;
842 }
843 if (aligned) {
844 decorators |= ARRAYCOPY_ALIGNED;
845 }
846
847 BarrierSetCodeGen *bs = Universe::heap()->barrier_set()->code_gen();
848 bs->arraycopy_prologue(_masm, decorators, t, from, to, count);
849
850 __ subptr(to, from); // to --> to_from
851 __ cmpl(count, 2<<shift); // Short arrays (< 8 bytes) copy by element
852 __ jcc(Assembler::below, L_copy_4_bytes); // use unsigned cmp
853 if (!UseUnalignedLoadStores && !aligned && (t == T_BYTE || t == T_SHORT)) {
854 // align source address at 4 bytes address boundary
855 if (t == T_BYTE) {
856 // One byte misalignment happens only for byte arrays
857 __ testl(from, 1);
858 __ jccb(Assembler::zero, L_skip_align1);
859 __ movb(rax, Address(from, 0));
860 __ movb(Address(from, to_from, Address::times_1, 0), rax);
861 __ increment(from);
862 __ decrement(count);
863 __ BIND(L_skip_align1);
864 }
865 // Two bytes misalignment happens only for byte and short (char) arrays
866 __ testl(from, 2);
867 __ jccb(Assembler::zero, L_skip_align2);
868 __ movw(rax, Address(from, 0));
|
1006 __ testl(count, 1<<(shift-1));
1007 __ jccb(Assembler::zero, L_copy_byte);
1008 __ movw(rax, Address(from, 0));
1009 __ movw(Address(from, to_from, Address::times_1, 0), rax);
1010 if (t == T_BYTE) {
1011 __ addptr(from, 2);
1012 __ BIND(L_copy_byte);
1013 // copy tailing byte
1014 __ testl(count, 1);
1015 __ jccb(Assembler::zero, L_exit);
1016 __ movb(rax, Address(from, 0));
1017 __ movb(Address(from, to_from, Address::times_1, 0), rax);
1018 __ BIND(L_exit);
1019 } else {
1020 __ BIND(L_copy_byte);
1021 }
1022 } else {
1023 __ BIND(L_copy_2_bytes);
1024 }
1025
1026 if (t == T_OBJECT) {
1027 __ movl(count, Address(rsp, 12+12)); // reread 'count'
1028 __ mov(to, saved_to); // restore 'to'
1029 gen_write_ref_array_post_barrier(to, count);
1030 __ BIND(L_0_count);
1031 }
1032 inc_copy_counter_np(t);
1033 __ pop(rdi);
1034 __ pop(rsi);
1035 __ leave(); // required for proper stackwalking of RuntimeStub frame
1036 __ vzeroupper();
1037 __ xorptr(rax, rax); // return 0
1038 __ ret(0);
1039 return start;
1040 }
1041
1042
1043 address generate_fill(BasicType t, bool aligned, const char *name) {
1044 __ align(CodeEntryAlignment);
1045 StubCodeMark mark(this, "StubRoutines", name);
1046 address start = __ pc();
1047
1048 BLOCK_COMMENT("Entry:");
|
914 __ testl(count, 1<<(shift-1));
915 __ jccb(Assembler::zero, L_copy_byte);
916 __ movw(rax, Address(from, 0));
917 __ movw(Address(from, to_from, Address::times_1, 0), rax);
918 if (t == T_BYTE) {
919 __ addptr(from, 2);
920 __ BIND(L_copy_byte);
921 // copy tailing byte
922 __ testl(count, 1);
923 __ jccb(Assembler::zero, L_exit);
924 __ movb(rax, Address(from, 0));
925 __ movb(Address(from, to_from, Address::times_1, 0), rax);
926 __ BIND(L_exit);
927 } else {
928 __ BIND(L_copy_byte);
929 }
930 } else {
931 __ BIND(L_copy_2_bytes);
932 }
933
934 __ movl(count, Address(rsp, 12+12)); // reread 'count'
935 bs->arraycopy_epilogue(_masm, decorators, t, from, to, count);
936
937 if (t == T_OBJECT) {
938 __ BIND(L_0_count);
939 }
940 inc_copy_counter_np(t);
941 __ pop(rdi);
942 __ pop(rsi);
943 __ leave(); // required for proper stackwalking of RuntimeStub frame
944 __ vzeroupper();
945 __ xorptr(rax, rax); // return 0
946 __ ret(0);
947 return start;
948 }
949
950
951 address generate_fill(BasicType t, bool aligned, const char *name) {
952 __ align(CodeEntryAlignment);
953 StubCodeMark mark(this, "StubRoutines", name);
954 address start = __ pc();
955
956 BLOCK_COMMENT("Entry:");
|
1098 if (entry != NULL) {
1099 *entry = __ pc(); // Entry point from generic arraycopy stub.
1100 BLOCK_COMMENT("Entry:");
1101 }
1102
1103 // nooverlap_target expects arguments in rsi and rdi.
1104 __ mov(from, src);
1105 __ mov(to , dst);
1106
1107 // arrays overlap test: dispatch to disjoint stub if necessary.
1108 RuntimeAddress nooverlap(nooverlap_target);
1109 __ cmpptr(dst, src);
1110 __ lea(end, Address(src, count, sf, 0)); // src + count * elem_size
1111 __ jump_cc(Assembler::belowEqual, nooverlap);
1112 __ cmpptr(dst, end);
1113 __ jump_cc(Assembler::aboveEqual, nooverlap);
1114
1115 if (t == T_OBJECT) {
1116 __ testl(count, count);
1117 __ jcc(Assembler::zero, L_0_count);
1118 gen_write_ref_array_pre_barrier(dst, count, dest_uninitialized);
1119 }
1120
1121 // copy from high to low
1122 __ cmpl(count, 2<<shift); // Short arrays (< 8 bytes) copy by element
1123 __ jcc(Assembler::below, L_copy_4_bytes); // use unsigned cmp
1124 if (t == T_BYTE || t == T_SHORT) {
1125 // Align the end of destination array at 4 bytes address boundary
1126 __ lea(end, Address(dst, count, sf, 0));
1127 if (t == T_BYTE) {
1128 // One byte misalignment happens only for byte arrays
1129 __ testl(end, 1);
1130 __ jccb(Assembler::zero, L_skip_align1);
1131 __ decrement(count);
1132 __ movb(rdx, Address(from, count, sf, 0));
1133 __ movb(Address(to, count, sf, 0), rdx);
1134 __ BIND(L_skip_align1);
1135 }
1136 // Two bytes misalignment happens only for byte and short (char) arrays
1137 __ testl(end, 2);
1138 __ jccb(Assembler::zero, L_skip_align2);
1139 __ subptr(count, 1<<(shift-1));
|
1006 if (entry != NULL) {
1007 *entry = __ pc(); // Entry point from generic arraycopy stub.
1008 BLOCK_COMMENT("Entry:");
1009 }
1010
1011 // nooverlap_target expects arguments in rsi and rdi.
1012 __ mov(from, src);
1013 __ mov(to , dst);
1014
1015 // arrays overlap test: dispatch to disjoint stub if necessary.
1016 RuntimeAddress nooverlap(nooverlap_target);
1017 __ cmpptr(dst, src);
1018 __ lea(end, Address(src, count, sf, 0)); // src + count * elem_size
1019 __ jump_cc(Assembler::belowEqual, nooverlap);
1020 __ cmpptr(dst, end);
1021 __ jump_cc(Assembler::aboveEqual, nooverlap);
1022
1023 if (t == T_OBJECT) {
1024 __ testl(count, count);
1025 __ jcc(Assembler::zero, L_0_count);
1026 }
1027
1028 DecoratorSet decorators = 0;
1029 if (dest_uninitialized) {
1030 decorators |= AS_DEST_NOT_INITIALIZED;
1031 }
1032 if (aligned) {
1033 decorators |= ARRAYCOPY_ALIGNED;
1034 }
1035
1036 BarrierSetCodeGen *bs = Universe::heap()->barrier_set()->code_gen();
1037 bs->arraycopy_prologue(_masm, decorators, t, from, to, count);
1038
1039 // copy from high to low
1040 __ cmpl(count, 2<<shift); // Short arrays (< 8 bytes) copy by element
1041 __ jcc(Assembler::below, L_copy_4_bytes); // use unsigned cmp
1042 if (t == T_BYTE || t == T_SHORT) {
1043 // Align the end of destination array at 4 bytes address boundary
1044 __ lea(end, Address(dst, count, sf, 0));
1045 if (t == T_BYTE) {
1046 // One byte misalignment happens only for byte arrays
1047 __ testl(end, 1);
1048 __ jccb(Assembler::zero, L_skip_align1);
1049 __ decrement(count);
1050 __ movb(rdx, Address(from, count, sf, 0));
1051 __ movb(Address(to, count, sf, 0), rdx);
1052 __ BIND(L_skip_align1);
1053 }
1054 // Two bytes misalignment happens only for byte and short (char) arrays
1055 __ testl(end, 2);
1056 __ jccb(Assembler::zero, L_skip_align2);
1057 __ subptr(count, 1<<(shift-1));
|
1198 // copy prefix dword
1199 __ testl(count, 1<<(shift-1));
1200 __ jccb(Assembler::zero, L_copy_byte);
1201 __ movw(rdx, Address(from, count, sf, -2));
1202 __ movw(Address(to, count, sf, -2), rdx);
1203 if (t == T_BYTE) {
1204 __ subl(count, 1<<(shift-1));
1205 __ BIND(L_copy_byte);
1206 // copy prefix byte
1207 __ testl(count, 1);
1208 __ jccb(Assembler::zero, L_exit);
1209 __ movb(rdx, Address(from, 0));
1210 __ movb(Address(to, 0), rdx);
1211 __ BIND(L_exit);
1212 } else {
1213 __ BIND(L_copy_byte);
1214 }
1215 } else {
1216 __ BIND(L_copy_2_bytes);
1217 }
1218 if (t == T_OBJECT) {
1219 __ movl2ptr(count, Address(rsp, 12+12)); // reread count
1220 gen_write_ref_array_post_barrier(to, count);
1221 __ BIND(L_0_count);
1222 }
1223 inc_copy_counter_np(t);
1224 __ pop(rdi);
1225 __ pop(rsi);
1226 __ leave(); // required for proper stackwalking of RuntimeStub frame
1227 __ xorptr(rax, rax); // return 0
1228 __ ret(0);
1229 return start;
1230 }
1231
1232
1233 address generate_disjoint_long_copy(address* entry, const char *name) {
1234 __ align(CodeEntryAlignment);
1235 StubCodeMark mark(this, "StubRoutines", name);
1236 address start = __ pc();
1237
1238 Label L_copy_8_bytes, L_copy_8_bytes_loop;
1239 const Register from = rax; // source array address
|
1116 // copy prefix dword
1117 __ testl(count, 1<<(shift-1));
1118 __ jccb(Assembler::zero, L_copy_byte);
1119 __ movw(rdx, Address(from, count, sf, -2));
1120 __ movw(Address(to, count, sf, -2), rdx);
1121 if (t == T_BYTE) {
1122 __ subl(count, 1<<(shift-1));
1123 __ BIND(L_copy_byte);
1124 // copy prefix byte
1125 __ testl(count, 1);
1126 __ jccb(Assembler::zero, L_exit);
1127 __ movb(rdx, Address(from, 0));
1128 __ movb(Address(to, 0), rdx);
1129 __ BIND(L_exit);
1130 } else {
1131 __ BIND(L_copy_byte);
1132 }
1133 } else {
1134 __ BIND(L_copy_2_bytes);
1135 }
1136
1137 __ movl2ptr(count, Address(rsp, 12+12)); // reread count
1138 bs->arraycopy_epilogue(_masm, decorators, t, from, to, count);
1139
1140 if (t == T_OBJECT) {
1141 __ BIND(L_0_count);
1142 }
1143 inc_copy_counter_np(t);
1144 __ pop(rdi);
1145 __ pop(rsi);
1146 __ leave(); // required for proper stackwalking of RuntimeStub frame
1147 __ xorptr(rax, rax); // return 0
1148 __ ret(0);
1149 return start;
1150 }
1151
1152
1153 address generate_disjoint_long_copy(address* entry, const char *name) {
1154 __ align(CodeEntryAlignment);
1155 StubCodeMark mark(this, "StubRoutines", name);
1156 address start = __ pc();
1157
1158 Label L_copy_8_bytes, L_copy_8_bytes_loop;
1159 const Register from = rax; // source array address
|
1445 //--------------------------------------------------------------- 1446 // Assembler stub will be used for this call to arraycopy 1447 // if the two arrays are subtypes of Object[] but the 1448 // destination array type is not equal to or a supertype 1449 // of the source type. Each element must be separately 1450 // checked. 1451 1452 // Loop-invariant addresses. They are exclusive end pointers. 1453 Address end_from_addr(from, length, Address::times_ptr, 0); 1454 Address end_to_addr(to, length, Address::times_ptr, 0); 1455 1456 Register end_from = from; // re-use 1457 Register end_to = to; // re-use 1458 Register count = length; // re-use 1459 1460 // Loop-variant addresses. They assume post-incremented count < 0. 1461 Address from_element_addr(end_from, count, Address::times_ptr, 0); 1462 Address to_element_addr(end_to, count, Address::times_ptr, 0); 1463 Address elem_klass_addr(elem, oopDesc::klass_offset_in_bytes()); 1464 1465 // Copy from low to high addresses, indexed from the end of each array. 1466 gen_write_ref_array_pre_barrier(to, count, dest_uninitialized); 1467 __ lea(end_from, end_from_addr); 1468 __ lea(end_to, end_to_addr); 1469 assert(length == count, ""); // else fix next line: 1470 __ negptr(count); // negate and test the length 1471 __ jccb(Assembler::notZero, L_load_element); 1472 1473 // Empty array: Nothing to do. 1474 __ xorptr(rax, rax); // return 0 on (trivial) success 1475 __ jmp(L_done); 1476 1477 // ======== begin loop ======== 1478 // (Loop is rotated; its entry is L_load_element.) 1479 // Loop control: 1480 // for (count = -count; count != 0; count++) 1481 // Base pointers src, dst are biased by 8*count,to last element. 1482 __ align(OptoLoopAlignment); 1483 1484 __ BIND(L_store_element); 1485 __ movptr(to_element_addr, elem); // store the oop |
1365 //--------------------------------------------------------------- 1366 // Assembler stub will be used for this call to arraycopy 1367 // if the two arrays are subtypes of Object[] but the 1368 // destination array type is not equal to or a supertype 1369 // of the source type. Each element must be separately 1370 // checked. 1371 1372 // Loop-invariant addresses. They are exclusive end pointers. 1373 Address end_from_addr(from, length, Address::times_ptr, 0); 1374 Address end_to_addr(to, length, Address::times_ptr, 0); 1375 1376 Register end_from = from; // re-use 1377 Register end_to = to; // re-use 1378 Register count = length; // re-use 1379 1380 // Loop-variant addresses. They assume post-incremented count < 0. 1381 Address from_element_addr(end_from, count, Address::times_ptr, 0); 1382 Address to_element_addr(end_to, count, Address::times_ptr, 0); 1383 Address elem_klass_addr(elem, oopDesc::klass_offset_in_bytes()); 1384 1385 DecoratorSet decorators = ARRAYCOPY_CHECKCAST; 1386 if (dest_uninitialized) { 1387 decorators |= AS_DEST_NOT_INITIALIZED; 1388 } 1389 1390 BasicType type = T_OBJECT; 1391 BarrierSetCodeGen *bs = Universe::heap()->barrier_set()->code_gen(); 1392 bs->arraycopy_prologue(_masm, decorators, type, from, to, count); 1393 1394 // Copy from low to high addresses, indexed from the end of each array. 1395 __ lea(end_from, end_from_addr); 1396 __ lea(end_to, end_to_addr); 1397 assert(length == count, ""); // else fix next line: 1398 __ negptr(count); // negate and test the length 1399 __ jccb(Assembler::notZero, L_load_element); 1400 1401 // Empty array: Nothing to do. 1402 __ xorptr(rax, rax); // return 0 on (trivial) success 1403 __ jmp(L_done); 1404 1405 // ======== begin loop ======== 1406 // (Loop is rotated; its entry is L_load_element.) 1407 // Loop control: 1408 // for (count = -count; count != 0; count++) 1409 // Base pointers src, dst are biased by 8*count,to last element. 1410 __ align(OptoLoopAlignment); 1411 1412 __ BIND(L_store_element); 1413 __ movptr(to_element_addr, elem); // store the oop |
1503
1504 // It was a real error; we must depend on the caller to finish the job.
1505 // Register "count" = -1 * number of *remaining* oops, length_arg = *total* oops.
1506 // Emit GC store barriers for the oops we have copied (length_arg + count),
1507 // and report their number to the caller.
1508 assert_different_registers(to, count, rax);
1509 Label L_post_barrier;
1510 __ addl(count, length_arg); // transfers = (length - remaining)
1511 __ movl2ptr(rax, count); // save the value
1512 __ notptr(rax); // report (-1^K) to caller (does not affect flags)
1513 __ jccb(Assembler::notZero, L_post_barrier);
1514 __ jmp(L_done); // K == 0, nothing was copied, skip post barrier
1515
1516 // Come here on success only.
1517 __ BIND(L_do_card_marks);
1518 __ xorptr(rax, rax); // return 0 on success
1519 __ movl2ptr(count, length_arg);
1520
1521 __ BIND(L_post_barrier);
1522 __ movptr(to, to_arg); // reload
1523 gen_write_ref_array_post_barrier(to, count);
1524
1525 // Common exit point (success or failure).
1526 __ BIND(L_done);
1527 __ pop(rbx);
1528 __ pop(rdi);
1529 __ pop(rsi);
1530 inc_counter_np(SharedRuntime::_checkcast_array_copy_ctr);
1531 __ leave(); // required for proper stackwalking of RuntimeStub frame
1532 __ ret(0);
1533
1534 return start;
1535 }
1536
1537 //
1538 // Generate 'unsafe' array copy stub
1539 // Though just as safe as the other stubs, it takes an unscaled
1540 // size_t argument instead of an element count.
1541 //
1542 // Input:
|
1431
1432 // It was a real error; we must depend on the caller to finish the job.
1433 // Register "count" = -1 * number of *remaining* oops, length_arg = *total* oops.
1434 // Emit GC store barriers for the oops we have copied (length_arg + count),
1435 // and report their number to the caller.
1436 assert_different_registers(to, count, rax);
1437 Label L_post_barrier;
1438 __ addl(count, length_arg); // transfers = (length - remaining)
1439 __ movl2ptr(rax, count); // save the value
1440 __ notptr(rax); // report (-1^K) to caller (does not affect flags)
1441 __ jccb(Assembler::notZero, L_post_barrier);
1442 __ jmp(L_done); // K == 0, nothing was copied, skip post barrier
1443
1444 // Come here on success only.
1445 __ BIND(L_do_card_marks);
1446 __ xorptr(rax, rax); // return 0 on success
1447 __ movl2ptr(count, length_arg);
1448
1449 __ BIND(L_post_barrier);
1450 __ movptr(to, to_arg); // reload
1451 bs->arraycopy_epilogue(_masm, decorators, type, from, to, count);
1452
1453 // Common exit point (success or failure).
1454 __ BIND(L_done);
1455 __ pop(rbx);
1456 __ pop(rdi);
1457 __ pop(rsi);
1458 inc_counter_np(SharedRuntime::_checkcast_array_copy_ctr);
1459 __ leave(); // required for proper stackwalking of RuntimeStub frame
1460 __ ret(0);
1461
1462 return start;
1463 }
1464
1465 //
1466 // Generate 'unsafe' array copy stub
1467 // Though just as safe as the other stubs, it takes an unscaled
1468 // size_t argument instead of an element count.
1469 //
1470 // Input:
|