--- old/src/hotspot/cpu/aarch64/aarch64.ad 2018-03-15 19:39:53.736289987 +0300 +++ new/src/hotspot/cpu/aarch64/aarch64.ad 2018-03-15 19:39:53.572284486 +0300 @@ -16165,9 +16165,8 @@ format %{ "String Equals $str1,$str2,$cnt -> $result" %} ins_encode %{ // Count is in 8-bit bytes; non-Compact chars are 16 bits. - __ arrays_equals($str1$$Register, $str2$$Register, - $result$$Register, $cnt$$Register, - 1, /*is_string*/true); + __ string_equals($str1$$Register, $str2$$Register, + $result$$Register, $cnt$$Register, 1); %} ins_pipe(pipe_class_memory); %} @@ -16182,42 +16181,42 @@ format %{ "String Equals $str1,$str2,$cnt -> $result" %} ins_encode %{ // Count is in 8-bit bytes; non-Compact chars are 16 bits. - __ asrw($cnt$$Register, $cnt$$Register, 1); - __ arrays_equals($str1$$Register, $str2$$Register, - $result$$Register, $cnt$$Register, - 2, /*is_string*/true); + __ string_equals($str1$$Register, $str2$$Register, + $result$$Register, $cnt$$Register, 2); %} ins_pipe(pipe_class_memory); %} instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result, - iRegP_R10 tmp, rFlagsReg cr) + iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3, + iRegP_R10 tmp, rFlagsReg cr) %{ predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL); match(Set result (AryEq ary1 ary2)); - effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, KILL cr); + effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr); format %{ "Array Equals $ary1,ary2 -> $result // KILL $tmp" %} ins_encode %{ __ arrays_equals($ary1$$Register, $ary2$$Register, - $result$$Register, $tmp$$Register, - 1, /*is_string*/false); + $tmp1$$Register, $tmp2$$Register, $tmp3$$Register, + $result$$Register, $tmp$$Register, 1); %} ins_pipe(pipe_class_memory); %} instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result, - iRegP_R10 tmp, rFlagsReg cr) + iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3, + iRegP_R10 tmp, rFlagsReg cr) %{ predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU); match(Set result (AryEq ary1 ary2)); - effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, KILL cr); + effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr); format %{ "Array Equals $ary1,ary2 -> $result // KILL $tmp" %} ins_encode %{ __ arrays_equals($ary1$$Register, $ary2$$Register, - $result$$Register, $tmp$$Register, - 2, /*is_string*/false); + $tmp1$$Register, $tmp2$$Register, $tmp3$$Register, + $result$$Register, $tmp$$Register, 2); %} ins_pipe(pipe_class_memory); %} --- old/src/hotspot/cpu/aarch64/globals_aarch64.hpp 2018-03-15 19:39:54.188305130 +0300 +++ new/src/hotspot/cpu/aarch64/globals_aarch64.hpp 2018-03-15 19:39:54.020299511 +0300 @@ -147,6 +147,10 @@ "Use CRC32 instructions for CRC32 computation") \ product(bool, UseSIMDForMemoryOps, false, \ "Use SIMD instructions in generated memory move code") \ + product(bool, UseSIMDForArrayEquals, true, \ + "Use SIMD instructions in generated array equals code") \ + product(bool, UseSimpleArrayEquals, false, \ + "Use simpliest and shortest implementation for array equals") \ product(bool, AvoidUnalignedAccesses, false, \ "Avoid generating unaligned memory accesses") \ product(bool, UseLSE, false, \ --- old/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp 2018-03-15 19:39:54.512315964 +0300 +++ new/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp 2018-03-15 19:39:54.344310347 +0300 @@ -5181,28 +5181,11 @@ BIND(DONE); } -// Compare Strings or char/byte arrays. - -// is_string is true iff this is a string comparison. - -// For Strings we're passed the address of the first characters in a1 -// and a2 and the length in cnt1. - -// For byte and char arrays we're passed the arrays themselves and we -// have to extract length fields and do null checks here. - -// elem_size is the element size in bytes: either 1 or 2. - -// There are two implementations. For arrays >= 8 bytes, all -// comparisons (including the final one, which may overlap) are -// performed 8 bytes at a time. For arrays < 8 bytes, we compare a -// halfword, then a short, and then a byte. - -void MacroAssembler::arrays_equals(Register a1, Register a2, - Register result, Register cnt1, - int elem_size, bool is_string) +void MacroAssembler::arrays_equals(Register a1, Register a2, Register tmp3, + Register tmp4, Register tmp5, Register result, + Register cnt1, int elem_size) { - Label SAME, DONE, SHORT, NEXT_WORD, ONE; + Label DONE; Register tmp1 = rscratch1; Register tmp2 = rscratch2; Register cnt2 = tmp2; // cnt2 only used in array length compare @@ -5211,6 +5194,7 @@ int length_offset = arrayOopDesc::length_offset_in_bytes(); int base_offset = arrayOopDesc::base_offset_in_bytes(elem_size == 2 ? T_CHAR : T_BYTE); + int stubBytesThreshold = 3 * 64 + (UseSIMDForArrayEquals ? 0 : 16); assert(elem_size == 1 || elem_size == 2, "must be char or byte"); assert_different_registers(a1, a2, result, cnt1, rscratch1, rscratch2); @@ -5219,43 +5203,229 @@ { const char kind = (elem_size == 2) ? 'U' : 'L'; char comment[64]; - snprintf(comment, sizeof comment, "%s%c%s {", - is_string ? "string_equals" : "array_equals", - kind, "{"); + snprintf(comment, sizeof comment, "array_equals%c{", kind); BLOCK_COMMENT(comment); } #endif - - mov(result, false); - - if (!is_string) { - // if (a==a2) + if (UseSimpleArrayEquals) { + Label NEXT_WORD, SHORT, SAME, TAIL03, TAIL01, A_MIGHT_BE_NULL, A_IS_NOT_NULL; + // if (a1==a2) // return true; - eor(rscratch1, a1, a2); - cbz(rscratch1, SAME); // if (a==null || a2==null) // return false; - cbz(a1, DONE); - cbz(a2, DONE); + // a1 & a2 == 0 means (some-pointer is null) or + // (very-rare-or-even-probably-impossible-pointer-values) + // so, we can save one branch in most cases + eor(rscratch1, a1, a2); + tst(a1, a2); + mov(result, false); + cbz(rscratch1, SAME); + br(EQ, A_MIGHT_BE_NULL); // if (a1.length != a2.length) // return false; + bind(A_IS_NOT_NULL); ldrw(cnt1, Address(a1, length_offset)); ldrw(cnt2, Address(a2, length_offset)); - eorw(tmp1, cnt1, cnt2); - cbnzw(tmp1, DONE); - + eorw(tmp5, cnt1, cnt2); + cbnzw(tmp5, DONE); lea(a1, Address(a1, base_offset)); lea(a2, Address(a2, base_offset)); + // Check for short strings, i.e. smaller than wordSize. + subs(cnt1, cnt1, elem_per_word); + br(Assembler::LT, SHORT); + // Main 8 byte comparison loop. + bind(NEXT_WORD); { + ldr(tmp1, Address(post(a1, wordSize))); + ldr(tmp2, Address(post(a2, wordSize))); + subs(cnt1, cnt1, elem_per_word); + eor(tmp5, tmp1, tmp2); + cbnz(tmp5, DONE); + } br(GT, NEXT_WORD); + // Last longword. In the case where length == 4 we compare the + // same longword twice, but that's still faster than another + // conditional branch. + // cnt1 could be 0, -1, -2, -3, -4 for chars; -4 only happens when + // length == 4. + if (log_elem_size > 0) + lsl(cnt1, cnt1, log_elem_size); + ldr(tmp3, Address(a1, cnt1)); + ldr(tmp4, Address(a2, cnt1)); + eor(tmp5, tmp3, tmp4); + cbnz(tmp5, DONE); + b(SAME); + bind(A_MIGHT_BE_NULL); + // in case both a1 and a2 are not-null, proceed with loads + cbz(a1, DONE); + cbz(a2, DONE); + b(A_IS_NOT_NULL); + bind(SHORT); + + tbz(cnt1, 2 - log_elem_size, TAIL03); // 0-7 bytes left. + { + ldrw(tmp1, Address(post(a1, 4))); + ldrw(tmp2, Address(post(a2, 4))); + eorw(tmp5, tmp1, tmp2); + cbnzw(tmp5, DONE); + } + bind(TAIL03); + tbz(cnt1, 1 - log_elem_size, TAIL01); // 0-3 bytes left. + { + ldrh(tmp3, Address(post(a1, 2))); + ldrh(tmp4, Address(post(a2, 2))); + eorw(tmp5, tmp3, tmp4); + cbnzw(tmp5, DONE); + } + bind(TAIL01); + if (elem_size == 1) { // Only needed when comparing byte arrays. + tbz(cnt1, 0, SAME); // 0-1 bytes left. + { + ldrb(tmp1, a1); + ldrb(tmp2, a2); + eorw(tmp5, tmp1, tmp2); + cbnzw(tmp5, DONE); + } + } + bind(SAME); + mov(result, true); + } else { + Label NEXT_DWORD, A_IS_NULL, SHORT, TAIL, TAIL2, STUB, EARLY_OUT, + CSET_EQ, LAST_CHECK, LEN_IS_ZERO, SAME; + cbz(a1, A_IS_NULL); + ldrw(cnt1, Address(a1, length_offset)); + cbz(a2, A_IS_NULL); + ldrw(cnt2, Address(a2, length_offset)); + mov(result, false); + // on most CPUs a2 is still "locked"(surprisingly) in ldrw and it's + // faster to perform another branch before comparing a1 and a2 + cmp(cnt1, elem_per_word); + br(LE, SHORT); // short or same + cmp(a1, a2); + br(EQ, SAME); + ldr(tmp3, Address(pre(a1, base_offset))); + cmp(cnt1, stubBytesThreshold); + br(GE, STUB); + ldr(tmp4, Address(pre(a2, base_offset))); + sub(tmp5, zr, cnt1, LSL, 3 + log_elem_size); + cmp(cnt2, cnt1); + br(NE, DONE); + + // Main 16 byte comparison loop with 2 exits + bind(NEXT_DWORD); { + ldr(tmp1, Address(pre(a1, wordSize))); + ldr(tmp2, Address(pre(a2, wordSize))); + subs(cnt1, cnt1, 2 * elem_per_word); + br(LE, TAIL); + eor(tmp4, tmp3, tmp4); + cbnz(tmp4, DONE); + ldr(tmp3, Address(pre(a1, wordSize))); + ldr(tmp4, Address(pre(a2, wordSize))); + cmp(cnt1, elem_per_word); + br(LE, TAIL2); + cmp(tmp1, tmp2); + } br(EQ, NEXT_DWORD); + b(DONE); + + bind(TAIL); + eor(tmp4, tmp3, tmp4); + eor(tmp2, tmp1, tmp2); + lslv(tmp2, tmp2, tmp5); + orr(tmp5, tmp4, tmp2); + cmp(tmp5, zr); + b(CSET_EQ); + + bind(TAIL2); + eor(tmp2, tmp1, tmp2); + cbnz(tmp2, DONE); + b(LAST_CHECK); + + bind(STUB); + ldr(tmp4, Address(pre(a2, base_offset))); + cmp(cnt2, cnt1); + br(NE, DONE); + if (elem_size == 2) { // convert to byte counter + lsl(cnt1, cnt1, 1); + } + eor(tmp5, tmp3, tmp4); + cbnz(tmp5, DONE); + RuntimeAddress stub = RuntimeAddress(StubRoutines::aarch64::large_array_equals()); + assert(stub.target() != NULL, "array_equals_long stub has not been generated"); + trampoline_call(stub); + b(DONE); + + bind(SAME); + mov(result, true); + b(DONE); + bind(A_IS_NULL); + // a1 or a2 is null. if a2 == a2 then return true. else return false + cmp(a1, a2); + b(CSET_EQ); + bind(EARLY_OUT); + // (a1 != null && a2 == null) || (a1 != null && a2 != null && a1 == a2) + // so, if a2 == null => return false(0), else return true, so we can return a2 + mov(result, a2); + b(DONE); + bind(LEN_IS_ZERO); + cmp(cnt2, zr); + b(CSET_EQ); + bind(SHORT); + cbz(cnt1, LEN_IS_ZERO); + sub(tmp5, zr, cnt1, LSL, 3 + log_elem_size); + ldr(tmp3, Address(a1, base_offset)); + ldr(tmp4, Address(a2, base_offset)); + bind(LAST_CHECK); + eor(tmp4, tmp3, tmp4); + lslv(tmp5, tmp4, tmp5); + cmp(tmp5, zr); + bind(CSET_EQ); + cset(result, EQ); } + // That's it. + bind(DONE); + + BLOCK_COMMENT("} array_equals"); +} + +// Compare Strings + +// For Strings we're passed the address of the first characters in a1 +// and a2 and the length in cnt1. +// elem_size is the element size in bytes: either 1 or 2. +// There are two implementations. For arrays >= 8 bytes, all +// comparisons (including the final one, which may overlap) are +// performed 8 bytes at a time. For strings < 8 bytes, we compare a +// halfword, then a short, and then a byte. + +void MacroAssembler::string_equals(Register a1, Register a2, + Register result, Register cnt1, int elem_size) +{ + Label SAME, DONE, SHORT, NEXT_WORD; + Register tmp1 = rscratch1; + Register tmp2 = rscratch2; + Register cnt2 = tmp2; // cnt2 only used in array length compare + + assert(elem_size == 1 || elem_size == 2, "must be 2 or 1 byte"); + assert_different_registers(a1, a2, result, cnt1, rscratch1, rscratch2); + +#ifndef PRODUCT + { + const char kind = (elem_size == 2) ? 'U' : 'L'; + char comment[64]; + snprintf(comment, sizeof comment, "{string_equals%c", kind); + BLOCK_COMMENT(comment); + } +#endif + + mov(result, false); + // Check for short strings, i.e. smaller than wordSize. - subs(cnt1, cnt1, elem_per_word); + subs(cnt1, cnt1, wordSize); br(Assembler::LT, SHORT); // Main 8 byte comparison loop. bind(NEXT_WORD); { ldr(tmp1, Address(post(a1, wordSize))); ldr(tmp2, Address(post(a2, wordSize))); - subs(cnt1, cnt1, elem_per_word); + subs(cnt1, cnt1, wordSize); eor(tmp1, tmp1, tmp2); cbnz(tmp1, DONE); } br(GT, NEXT_WORD); @@ -5264,18 +5434,16 @@ // conditional branch. // cnt1 could be 0, -1, -2, -3, -4 for chars; -4 only happens when // length == 4. - if (log_elem_size > 0) - lsl(cnt1, cnt1, log_elem_size); ldr(tmp1, Address(a1, cnt1)); ldr(tmp2, Address(a2, cnt1)); - eor(tmp1, tmp1, tmp2); - cbnz(tmp1, DONE); + eor(tmp2, tmp1, tmp2); + cbnz(tmp2, DONE); b(SAME); bind(SHORT); Label TAIL03, TAIL01; - tbz(cnt1, 2 - log_elem_size, TAIL03); // 0-7 bytes left. + tbz(cnt1, 2, TAIL03); // 0-7 bytes left. { ldrw(tmp1, Address(post(a1, 4))); ldrw(tmp2, Address(post(a2, 4))); @@ -5283,7 +5451,7 @@ cbnzw(tmp1, DONE); } bind(TAIL03); - tbz(cnt1, 1 - log_elem_size, TAIL01); // 0-3 bytes left. + tbz(cnt1, 1, TAIL01); // 0-3 bytes left. { ldrh(tmp1, Address(post(a1, 2))); ldrh(tmp2, Address(post(a2, 2))); @@ -5291,7 +5459,7 @@ cbnzw(tmp1, DONE); } bind(TAIL01); - if (elem_size == 1) { // Only needed when comparing byte arrays. + if (elem_size == 1) { // Only needed when comparing 1-byte elements tbz(cnt1, 0, SAME); // 0-1 bytes left. { ldrb(tmp1, a1); @@ -5306,7 +5474,7 @@ // That's it. bind(DONE); - BLOCK_COMMENT(is_string ? "} string_equals" : "} array_equals"); + BLOCK_COMMENT("} string_equals"); } --- old/src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp 2018-03-15 19:39:54.888328538 +0300 +++ new/src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp 2018-03-15 19:39:54.712322653 +0300 @@ -1225,9 +1225,11 @@ void has_negatives(Register ary1, Register len, Register result); - void arrays_equals(Register a1, Register a2, - Register result, Register cnt1, - int elem_size, bool is_string); + void arrays_equals(Register a1, Register a2, Register result, Register cnt1, + Register tmp1, Register tmp2, Register tmp3, int elem_size); + + void string_equals(Register a1, Register a2, Register result, Register cnt1, + int elem_size); void fill_words(Register base, Register cnt, Register value); void zero_words(Register base, u_int64_t cnt); --- old/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp 2018-03-15 19:39:55.212339354 +0300 +++ new/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp 2018-03-15 19:39:55.048333885 +0300 @@ -3888,6 +3888,182 @@ __ ret(lr); return entry; } + + void generate_large_array_equals_loop_nonsimd(int loopThreshold, + bool usePrefetch, Label &NOT_EQUAL) { + Register a1 = r1, a2 = r2, result = r0, cnt1 = r10, tmp1 = rscratch1, + tmp2 = rscratch2, tmp3 = r3, tmp4 = r4, tmp5 = r5, tmp6 = r11, + tmp7 = r12, tmp8 = r13; + Label LOOP; + + __ ldp(tmp1, tmp3, Address(__ post(a1, 2 * wordSize))); + __ ldp(tmp2, tmp4, Address(__ post(a2, 2 * wordSize))); + __ bind(LOOP); + if (usePrefetch) { + __ prfm(Address(a1, SoftwarePrefetchHintDistance)); + __ prfm(Address(a2, SoftwarePrefetchHintDistance)); + } + __ ldp(tmp5, tmp7, Address(__ post(a1, 2 * wordSize))); + __ eor(tmp1, tmp1, tmp2); + __ eor(tmp3, tmp3, tmp4); + __ ldp(tmp6, tmp8, Address(__ post(a2, 2 * wordSize))); + __ orr(tmp1, tmp1, tmp3); + __ cbnz(tmp1, NOT_EQUAL); + __ ldp(tmp1, tmp3, Address(__ post(a1, 2 * wordSize))); + __ eor(tmp5, tmp5, tmp6); + __ eor(tmp7, tmp7, tmp8); + __ ldp(tmp2, tmp4, Address(__ post(a2, 2 * wordSize))); + __ orr(tmp5, tmp5, tmp7); + __ cbnz(tmp5, NOT_EQUAL); + __ ldp(tmp5, tmp7, Address(__ post(a1, 2 * wordSize))); + __ eor(tmp1, tmp1, tmp2); + __ eor(tmp3, tmp3, tmp4); + __ ldp(tmp6, tmp8, Address(__ post(a2, 2 * wordSize))); + __ orr(tmp1, tmp1, tmp3); + __ cbnz(tmp1, NOT_EQUAL); + __ ldp(tmp1, tmp3, Address(__ post(a1, 2 * wordSize))); + __ eor(tmp5, tmp5, tmp6); + __ sub(cnt1, cnt1, 8 * wordSize); + __ eor(tmp7, tmp7, tmp8); + __ ldp(tmp2, tmp4, Address(__ post(a2, 2 * wordSize))); + __ cmp(cnt1, loopThreshold); + __ orr(tmp5, tmp5, tmp7); + __ cbnz(tmp5, NOT_EQUAL); + __ br(__ GE, LOOP); + // post-loop + __ eor(tmp1, tmp1, tmp2); + __ eor(tmp3, tmp3, tmp4); + __ orr(tmp1, tmp1, tmp3); + __ sub(cnt1, cnt1, 2 * wordSize); + __ cbnz(tmp1, NOT_EQUAL); + } + + void generate_large_array_equals_loop_simd(int loopThreshold, + bool usePrefetch, Label &NOT_EQUAL) { + Register a1 = r1, a2 = r2, result = r0, cnt1 = r10, tmp1 = rscratch1, + tmp2 = rscratch2; + Label LOOP; + + __ bind(LOOP); + if (usePrefetch) { + __ prfm(Address(a1, SoftwarePrefetchHintDistance)); + __ prfm(Address(a2, SoftwarePrefetchHintDistance)); + } + __ ld1(v0, v1, v2, v3, __ T2D, Address(__ post(a1, 4 * 2 * wordSize))); + __ sub(cnt1, cnt1, 8 * wordSize); + __ ld1(v4, v5, v6, v7, __ T2D, Address(__ post(a2, 4 * 2 * wordSize))); + __ cmp(cnt1, loopThreshold); + __ eor(v0, __ T16B, v0, v4); + __ eor(v1, __ T16B, v1, v5); + __ eor(v2, __ T16B, v2, v6); + __ eor(v3, __ T16B, v3, v7); + __ orr(v0, __ T16B, v0, v1); + __ orr(v1, __ T16B, v2, v3); + __ orr(v0, __ T16B, v0, v1); + __ umov(tmp1, v0, __ D, 0); + __ umov(tmp2, v0, __ D, 1); + __ orr(tmp1, tmp1, tmp2); + __ cbnz(tmp1, NOT_EQUAL); + __ br(__ GE, LOOP); + } + + // a1 = r1 - array1 address + // a2 = r2 - array2 address + // result = r0 - return value. Already contains "false" + // cnt1 = r10 - amount of elements left to check, reduced by wordSize + // r3-r5 are reserved temporary registers + address generate_large_array_equals() { + StubCodeMark mark(this, "StubRoutines", "large_array_equals"); + Register a1 = r1, a2 = r2, result = r0, cnt1 = r10, tmp1 = rscratch1, + tmp2 = rscratch2, tmp3 = r3, tmp4 = r4, tmp5 = r5, tmp6 = r11, + tmp7 = r12, tmp8 = r13; + Label TAIL, NOT_EQUAL, EQUAL, NOT_EQUAL_NO_POP, NO_PREFETCH_LARGE_LOOP, + SMALL_LOOP, POST_LOOP; + const int PRE_LOOP_SIZE = UseSIMDForArrayEquals ? 0 : 16; + // calculate if at least 32 prefetched bytes are used + int prefetchLoopThreshold = SoftwarePrefetchHintDistance + 32; + int nonPrefetchLoopThreshold = (64 + PRE_LOOP_SIZE); + RegSet spilled_regs = RegSet::range(tmp6, tmp8); + assert_different_registers(a1, a2, result, cnt1, tmp1, tmp2, tmp3, tmp4, + tmp5, tmp6, tmp7, tmp8); + + __ align(CodeEntryAlignment); + address entry = __ pc(); + __ enter(); + __ sub(cnt1, cnt1, wordSize); // first 8 bytes were loaded outside of stub + // also advance pointers to use post-increment instead of pre-increment + __ add(a1, a1, wordSize); + __ add(a2, a2, wordSize); + if (AvoidUnalignedAccesses) { + // both implementations (SIMD/nonSIMD) are using relatively large load + // instructions (ld1/ldp), which has huge penalty (up to x2 exec time) + // on some CPUs in case of address is not at least 16-byte aligned. + // Arrays are 8-byte aligned currently, so, we can make additional 8-byte + // load if needed at least for 1st address and make if 16-byte aligned. + Label ALIGNED16; + __ tbz(a1, 3, ALIGNED16); + __ ldr(tmp1, Address(__ post(a1, wordSize))); + __ ldr(tmp2, Address(__ post(a2, wordSize))); + __ sub(cnt1, cnt1, wordSize); + __ eor(tmp1, tmp1, tmp2); + __ cbnz(tmp1, NOT_EQUAL_NO_POP); + __ bind(ALIGNED16); + } + if (UseSIMDForArrayEquals) { + if (SoftwarePrefetchHintDistance >= 0) { + __ cmp(cnt1, prefetchLoopThreshold); + __ br(__ LE, NO_PREFETCH_LARGE_LOOP); + generate_large_array_equals_loop_simd(prefetchLoopThreshold, + /* prfm = */ true, NOT_EQUAL); + __ cmp(cnt1, nonPrefetchLoopThreshold); + __ br(__ LT, TAIL); + } + __ bind(NO_PREFETCH_LARGE_LOOP); + generate_large_array_equals_loop_simd(nonPrefetchLoopThreshold, + /* prfm = */ false, NOT_EQUAL); + } else { + __ push(spilled_regs, sp); + if (SoftwarePrefetchHintDistance >= 0) { + __ cmp(cnt1, prefetchLoopThreshold); + __ br(__ LE, NO_PREFETCH_LARGE_LOOP); + generate_large_array_equals_loop_nonsimd(prefetchLoopThreshold, + /* prfm = */ true, NOT_EQUAL); + __ cmp(cnt1, nonPrefetchLoopThreshold); + __ br(__ LT, TAIL); + } + __ bind(NO_PREFETCH_LARGE_LOOP); + generate_large_array_equals_loop_nonsimd(nonPrefetchLoopThreshold, + /* prfm = */ false, NOT_EQUAL); + } + __ bind(TAIL); + __ cbz(cnt1, EQUAL); + __ subs(cnt1, cnt1, wordSize); + __ br(__ LE, POST_LOOP); + __ bind(SMALL_LOOP); + __ ldr(tmp1, Address(__ post(a1, wordSize))); + __ ldr(tmp2, Address(__ post(a2, wordSize))); + __ subs(cnt1, cnt1, wordSize); + __ eor(tmp1, tmp1, tmp2); + __ cbnz(tmp1, NOT_EQUAL); + __ br(__ GT, SMALL_LOOP); + __ bind(POST_LOOP); + __ ldr(tmp1, Address(a1, cnt1)); + __ ldr(tmp2, Address(a2, cnt1)); + __ eor(tmp1, tmp1, tmp2); + __ cbnz(tmp1, NOT_EQUAL); + __ bind(EQUAL); + __ mov(result, true); + __ bind(NOT_EQUAL); + if (!UseSIMDForArrayEquals) { + __ pop(spilled_regs, sp); + } + __ bind(NOT_EQUAL_NO_POP); + __ leave(); + __ ret(lr); + return entry; + } + + /** * Arguments: * @@ -4970,6 +5146,11 @@ // has negatives stub for large arrays. StubRoutines::aarch64::_has_negatives = generate_has_negatives(StubRoutines::aarch64::_has_negatives_long); + // array equals stub for large arrays. + if (!UseSimpleArrayEquals) { + StubRoutines::aarch64::_large_array_equals = generate_large_array_equals(); + } + if (UseMultiplyToLenIntrinsic) { StubRoutines::_multiplyToLen = generate_multiplyToLen(); } --- old/src/hotspot/cpu/aarch64/stubRoutines_aarch64.cpp 2018-03-15 19:39:55.564351092 +0300 +++ new/src/hotspot/cpu/aarch64/stubRoutines_aarch64.cpp 2018-03-15 19:39:55.400345623 +0300 @@ -46,6 +46,7 @@ address StubRoutines::aarch64::_zero_blocks = NULL; address StubRoutines::aarch64::_has_negatives = NULL; address StubRoutines::aarch64::_has_negatives_long = NULL; +address StubRoutines::aarch64::_large_array_equals = NULL; bool StubRoutines::aarch64::_completed = false; /** --- old/src/hotspot/cpu/aarch64/stubRoutines_aarch64.hpp 2018-03-15 19:39:55.872361363 +0300 +++ new/src/hotspot/cpu/aarch64/stubRoutines_aarch64.hpp 2018-03-15 19:39:55.716356161 +0300 @@ -65,6 +65,7 @@ static address _has_negatives; static address _has_negatives_long; + static address _large_array_equals; static bool _completed; public: @@ -131,6 +132,10 @@ return _has_negatives_long; } + static address large_array_equals() { + return _large_array_equals; + } + static bool complete() { return _completed; } --- old/src/hotspot/cpu/aarch64/vm_version_aarch64.cpp 2018-03-15 19:39:56.192372016 +0300 +++ new/src/hotspot/cpu/aarch64/vm_version_aarch64.cpp 2018-03-15 19:39:56.028366564 +0300 @@ -203,7 +203,11 @@ if (FLAG_IS_DEFAULT(UseSIMDForMemoryOps)) { FLAG_SET_DEFAULT(UseSIMDForMemoryOps, (_variant > 0)); } + if (FLAG_IS_DEFAULT(UseSIMDForArrayEquals)) { + FLAG_SET_DEFAULT(UseSIMDForArrayEquals, false); + } } + // ThunderX2 if ((_cpu == CPU_CAVIUM && (_model == 0xAF)) || (_cpu == CPU_BROADCOM && (_model == 0x516))) { @@ -218,7 +222,25 @@ } } - if (_cpu == CPU_ARM && (_model == 0xd03 || _model2 == 0xd03)) _features |= CPU_A53MAC; + // Cortex A53 + if (_cpu == CPU_ARM && (_model == 0xd03 || _model2 == 0xd03)) { + _features |= CPU_A53MAC; + if (FLAG_IS_DEFAULT(UseSIMDForArrayEquals)) { + FLAG_SET_DEFAULT(UseSIMDForArrayEquals, false); + } + } + + // Cortex A73 + if (_cpu == CPU_ARM && (_model == 0xd09 || _model2 == 0xd09)) { + if (FLAG_IS_DEFAULT(SoftwarePrefetchHintDistance)) { + FLAG_SET_DEFAULT(SoftwarePrefetchHintDistance, -1); + } + // A73 is faster with short-and-easy-for-speculative-execution-loop + if (FLAG_IS_DEFAULT(UseSimpleArrayEquals)) { + FLAG_SET_DEFAULT(UseSimpleArrayEquals, true); + } + } + if (_cpu == CPU_ARM && (_model == 0xd07 || _model2 == 0xd07)) _features |= CPU_STXR_PREFETCH; // If an olde style /proc/cpuinfo (cpu_lines == 1) then if _model is an A57 (0xd07) // we assume the worst and assume we could be on a big little system and have