< prev index next >

src/cpu/ppc/vm/macroAssembler_ppc.cpp

Print this page
rev 10106 : 8149655: PPC64: Implement CompactString intrinsics
Reviewed-by:

*** 43,52 **** --- 43,55 ---- #if INCLUDE_ALL_GCS #include "gc/g1/g1CollectedHeap.inline.hpp" #include "gc/g1/g1SATBCardTableModRefBS.hpp" #include "gc/g1/heapRegion.hpp" #endif // INCLUDE_ALL_GCS + #ifdef COMPILER2 + #include "opto/intrinsicnode.hpp" + #endif #ifdef PRODUCT #define BLOCK_COMMENT(str) // nothing #else #define BLOCK_COMMENT(str) block_comment(str)
*** 3166,3175 **** --- 3169,3725 ---- bind(done); } /////////////////////////////////////////// String intrinsics //////////////////////////////////////////// + #ifdef COMPILER2 + // Intrinsics for CompactStrings + + // Compress char[] to byte[] by compressing 16 bytes at once. + void MacroAssembler::string_compress_16(Register src, Register dst, Register cnt, + Register tmp1, Register tmp2, Register tmp3, Register tmp4, Register tmp5, + Label& Lfailure) { + + const Register tmp0 = R0; + assert_different_registers(src, dst, cnt, tmp0, tmp1, tmp2, tmp3, tmp4, tmp5); + Label Lloop, Lslow; + + // Check if cnt >= 8 (= 16 bytes) + lis(tmp1, 0xFF); // tmp1 = 0x00FF00FF00FF00FF + srwi_(tmp2, cnt, 3); + beq(CCR0, Lslow); + ori(tmp1, tmp1, 0xFF); + rldimi(tmp1, tmp1, 32, 0); + mtctr(tmp2); + + // 2x unrolled loop + bind(Lloop); + ld(tmp2, 0, src); // _0_1_2_3 (Big Endian) + ld(tmp4, 8, src); // _4_5_6_7 + + orr(tmp0, tmp2, tmp4); + rldicl(tmp3, tmp2, 6*8, 64-24); // _____1_2 + rldimi(tmp2, tmp2, 2*8, 2*8); // _0_2_3_3 + rldicl(tmp5, tmp4, 6*8, 64-24); // _____5_6 + rldimi(tmp4, tmp4, 2*8, 2*8); // _4_6_7_7 + + andc_(tmp0, tmp0, tmp1); + bne(CCR0, Lfailure); // Not latin1. + addi(src, src, 16); + + rlwimi(tmp3, tmp2, 0*8, 24, 31);// _____1_3 + srdi(tmp2, tmp2, 3*8); // ____0_2_ + rlwimi(tmp5, tmp4, 0*8, 24, 31);// _____5_7 + srdi(tmp4, tmp4, 3*8); // ____4_6_ + + orr(tmp2, tmp2, tmp3); // ____0123 + orr(tmp4, tmp4, tmp5); // ____4567 + + stw(tmp2, 0, dst); + stw(tmp4, 4, dst); + addi(dst, dst, 8); + bdnz(Lloop); + + bind(Lslow); // Fallback to slow version + } + + // Compress char[] to byte[]. cnt must be positive int. + void MacroAssembler::string_compress(Register src, Register dst, Register cnt, Register tmp, Label& Lfailure) { + Label Lloop; + mtctr(cnt); + + bind(Lloop); + lhz(tmp, 0, src); + cmplwi(CCR0, tmp, 0xff); + bgt(CCR0, Lfailure); // Not latin1. + addi(src, src, 2); + stb(tmp, 0, dst); + addi(dst, dst, 1); + bdnz(Lloop); + } + + // Inflate byte[] to char[] by inflating 16 bytes at once. + void MacroAssembler::string_inflate_16(Register src, Register dst, Register cnt, + Register tmp1, Register tmp2, Register tmp3, Register tmp4, Register tmp5) { + const Register tmp0 = R0; + assert_different_registers(src, dst, cnt, tmp0, tmp1, tmp2, tmp3, tmp4, tmp5); + Label Lloop, Lslow; + + // Check if cnt >= 8 + srwi_(tmp2, cnt, 3); + beq(CCR0, Lslow); + lis(tmp1, 0xFF); // tmp1 = 0x00FF00FF + ori(tmp1, tmp1, 0xFF); + mtctr(tmp2); + + // 2x unrolled loop + bind(Lloop); + lwz(tmp2, 0, src); // ____0123 (Big Endian) + lwz(tmp4, 4, src); // ____4567 + addi(src, src, 8); + + rldicl(tmp3, tmp2, 7*8, 64-8); // _______2 + rlwimi(tmp2, tmp2, 3*8, 16, 23);// ____0113 + rldicl(tmp5, tmp4, 7*8, 64-8); // _______6 + rlwimi(tmp4, tmp4, 3*8, 16, 23);// ____4557 + + andc(tmp0, tmp2, tmp1); // ____0_1_ + rlwimi(tmp2, tmp3, 2*8, 0, 23); // _____2_3 + andc(tmp3, tmp4, tmp1); // ____4_5_ + rlwimi(tmp4, tmp5, 2*8, 0, 23); // _____6_7 + + rldimi(tmp2, tmp0, 3*8, 0*8); // _0_1_2_3 + rldimi(tmp4, tmp3, 3*8, 0*8); // _4_5_6_7 + + std(tmp2, 0, dst); + std(tmp4, 8, dst); + addi(dst, dst, 16); + bdnz(Lloop); + + bind(Lslow); // Fallback to slow version + } + + // Inflate byte[] to char[]. cnt must be positive int. + void MacroAssembler::string_inflate(Register src, Register dst, Register cnt, Register tmp) { + Label Lloop; + mtctr(cnt); + + bind(Lloop); + lbz(tmp, 0, src); + addi(src, src, 1); + sth(tmp, 0, dst); + addi(dst, dst, 2); + bdnz(Lloop); + } + + void MacroAssembler::string_compare(Register str1, Register str2, + Register cnt1, Register cnt2, + Register tmp1, Register result, int ae) { + const Register tmp0 = R0, + diff = tmp1; + + assert_different_registers(str1, str2, cnt1, cnt2, tmp0, tmp1, result); + Label Ldone, Lslow, Lloop, Lreturn_diff; + + // Note: Making use of the fact that compareTo(a, b) == -compareTo(b, a) + // we interchange str1 and str2 in the UL case and negate the result. + // Like this, str1 is always latin1 encoded, except for the UU case. + // In addition, we need 0 (or sign which is 0) extend. + + if (ae == StrIntrinsicNode::UU) { + srwi(cnt1, cnt1, 1); + } else { + clrldi(cnt1, cnt1, 32); + } + + if (ae != StrIntrinsicNode::LL) { + srwi(cnt2, cnt2, 1); + } else { + clrldi(cnt2, cnt2, 32); + } + + // See if the lengths are different, and calculate min in cnt1. + // Save diff in case we need it for a tie-breaker. + subf_(diff, cnt2, cnt1); // diff = cnt1 - cnt2 + // if (diff > 0) { cnt1 = cnt2; } + if (VM_Version::has_isel()) { + isel(cnt1, CCR0, Assembler::greater, /*invert*/ false, cnt2); + } else { + Label Lskip; + blt(CCR0, Lskip); + mr(cnt1, cnt2); + bind(Lskip); + } + + // Rename registers + Register chr1 = result; + Register chr2 = tmp0; + + // Compare multiple characters in fast loop (only implemented for same encoding). + int stride1 = 8, stride2 = 8; + if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) { + int log2_chars_per_iter = (ae == StrIntrinsicNode::LL) ? 3 : 2; + Label Lfastloop, Lskipfast; + + srwi_(tmp0, cnt1, log2_chars_per_iter); + beq(CCR0, Lskipfast); + rldicl(cnt2, cnt1, 0, 64 - log2_chars_per_iter); // Remaining characters. + li(cnt1, 1 << log2_chars_per_iter); // Initialize for failure case: Rescan characters from current iteration. + mtctr(tmp0); + + bind(Lfastloop); + ld(chr1, 0, str1); + ld(chr2, 0, str2); + cmpd(CCR0, chr1, chr2); + bne(CCR0, Lslow); + addi(str1, str1, stride1); + addi(str2, str2, stride2); + bdnz(Lfastloop); + mr(cnt1, cnt2); // Remaining characters. + bind(Lskipfast); + } + + // Loop which searches the first difference character by character. + cmpwi(CCR0, cnt1, 0); + beq(CCR0, Lreturn_diff); + bind(Lslow); + mtctr(cnt1); + + switch (ae) { + case StrIntrinsicNode::LL: stride1 = 1; stride2 = 1; break; + case StrIntrinsicNode::UL: // fallthru (see comment above) + case StrIntrinsicNode::LU: stride1 = 1; stride2 = 2; break; + case StrIntrinsicNode::UU: stride1 = 2; stride2 = 2; break; + default: ShouldNotReachHere(); break; + } + + bind(Lloop); + if (stride1 == 1) { lbz(chr1, 0, str1); } else { lhz(chr1, 0 ,str1); } + if (stride2 == 1) { lbz(chr2, 0, str2); } else { lhz(chr2, 0 ,str2); } + subf_(result, chr2, chr1); // result = chr1 - chr2 + bne(CCR0, Ldone); + addi(str1, str1, stride1); + addi(str2, str2, stride2); + bdnz(Lloop); + + // If strings are equal up to min length, return the length difference. + bind(Lreturn_diff); + mr(result, diff); + + // Otherwise, return the difference between the first mismatched chars. + bind(Ldone); + if (ae == StrIntrinsicNode::UL) { + neg(result, result); // Negate result (see note above). + } + } + + void MacroAssembler::array_equals(bool is_array_equ, Register ary1, Register ary2, + Register limit, Register tmp1, Register result, bool is_byte) { + const Register tmp0 = R0; + assert_different_registers(ary1, ary2, limit, tmp0, tmp1, result); + Label Ldone, Lskiploop, Lloop, Lfastloop, Lskipfast; + bool limit_needs_shift = false; + + if (is_array_equ) { + const int length_offset = arrayOopDesc::length_offset_in_bytes(); + const int base_offset = arrayOopDesc::base_offset_in_bytes(is_byte ? T_BYTE : T_CHAR); + + // Return true if the same array. + cmpd(CCR0, ary1, ary2); + beq(CCR0, Lskiploop); + + // Return false if one of them is NULL. + cmpdi(CCR0, ary1, 0); + cmpdi(CCR1, ary2, 0); + li(result, 0); + cror(CCR0, Assembler::equal, CCR1, Assembler::equal); + beq(CCR0, Ldone); + + // Load the lengths of arrays. + lwz(limit, length_offset, ary1); + lwz(tmp0, length_offset, ary2); + + // Return false if the two arrays are not equal length. + cmpw(CCR0, limit, tmp0); + bne(CCR0, Ldone); + + // Load array addresses. + addi(ary1, ary1, base_offset); + addi(ary2, ary2, base_offset); + } else { + limit_needs_shift = !is_byte; + li(result, 0); // Assume not equal. + } + + // Rename registers + Register chr1 = tmp0; + Register chr2 = tmp1; + + // Compare 8 bytes per iteration in fast loop. + const int log2_chars_per_iter = is_byte ? 3 : 2; + + srwi_(tmp0, limit, log2_chars_per_iter + (limit_needs_shift ? 1 : 0)); + beq(CCR0, Lskipfast); + mtctr(tmp0); + + bind(Lfastloop); + ld(chr1, 0, ary1); + ld(chr2, 0, ary2); + addi(ary1, ary1, 8); + addi(ary2, ary2, 8); + cmpd(CCR0, chr1, chr2); + bne(CCR0, Ldone); + bdnz(Lfastloop); + + bind(Lskipfast); + rldicl_(limit, limit, limit_needs_shift ? 64 - 1 : 0, 64 - log2_chars_per_iter); // Remaining characters. + beq(CCR0, Lskiploop); + mtctr(limit); + + // Character by character. + bind(Lloop); + if (is_byte) { + lbz(chr1, 0, ary1); + lbz(chr2, 0, ary2); + addi(ary1, ary1, 1); + addi(ary2, ary2, 1); + } else { + lhz(chr1, 0, ary1); + lhz(chr2, 0, ary2); + addi(ary1, ary1, 2); + addi(ary2, ary2, 2); + } + cmpw(CCR0, chr1, chr2); + bne(CCR0, Ldone); + bdnz(Lloop); + + bind(Lskiploop); + li(result, 1); // All characters are equal. + bind(Ldone); + } + + void MacroAssembler::string_indexof(Register result, Register haystack, Register haycnt, + Register needle, ciTypeArray* needle_values, Register needlecnt, int needlecntval, + Register tmp1, Register tmp2, Register tmp3, Register tmp4, int ae) { + + // Ensure 0<needlecnt<=haycnt in ideal graph as prerequisite! + Label L_TooShort, L_Found, L_NotFound, L_End; + Register last_addr = haycnt, // Kill haycnt at the beginning. + addr = tmp1, + n_start = tmp2, + ch1 = tmp3, + ch2 = R0; + + assert(ae != StrIntrinsicNode::LU, "Invalid encoding"); + const int h_csize = (ae == StrIntrinsicNode::LL) ? 1 : 2; + const int n_csize = (ae == StrIntrinsicNode::UU) ? 2 : 1; + + // ************************************************************************************************** + // Prepare for main loop: optimized for needle count >=2, bail out otherwise. + // ************************************************************************************************** + + // Compute last haystack addr to use if no match gets found. + clrldi(haycnt, haycnt, 32); // Ensure positive int is valid as 64 bit value. + addi(addr, haystack, -h_csize); // Accesses use pre-increment. + if (needlecntval == 0) { // variable needlecnt + cmpwi(CCR6, needlecnt, 2); + clrldi(needlecnt, needlecnt, 32); // Ensure positive int is valid as 64 bit value. + blt(CCR6, L_TooShort); // Variable needlecnt: handle short needle separately. + } + + if (n_csize == 2) { lwz(n_start, 0, needle); } else { lhz(n_start, 0, needle); } // Load first 2 characters of needle. + + if (needlecntval == 0) { // variable needlecnt + subf(ch1, needlecnt, haycnt); // Last character index to compare is haycnt-needlecnt. + addi(needlecnt, needlecnt, -2); // Rest of needle. + } else { // constant needlecnt + guarantee(needlecntval != 1, "IndexOf with single-character needle must be handled separately"); + assert((needlecntval & 0x7fff) == needlecntval, "wrong immediate"); + addi(ch1, haycnt, -needlecntval); // Last character index to compare is haycnt-needlecnt. + if (needlecntval > 3) { li(needlecnt, needlecntval - 2); } // Rest of needle. + } + + if (h_csize == 2) { slwi(ch1, ch1, 1); } // Scale to number of bytes. + + if (ae ==StrIntrinsicNode::UL) { + srwi(tmp4, n_start, 1*8); // ___0 + rlwimi(n_start, tmp4, 2*8, 0, 23); // _0_1 + } + + add(last_addr, haystack, ch1); // Point to last address to compare (haystack+2*(haycnt-needlecnt)). + + // Main Loop (now we have at least 2 characters). + Label L_OuterLoop, L_InnerLoop, L_FinalCheck, L_Comp1, L_Comp2; + bind(L_OuterLoop); // Search for 1st 2 characters. + Register addr_diff = tmp4; + subf(addr_diff, addr, last_addr); // Difference between already checked address and last address to check. + addi(addr, addr, h_csize); // This is the new address we want to use for comparing. + srdi_(ch2, addr_diff, h_csize); + beq(CCR0, L_FinalCheck); // 2 characters left? + mtctr(ch2); // num of characters / 2 + bind(L_InnerLoop); // Main work horse (2x unrolled search loop) + if (h_csize == 2) { // Load 2 characters of haystack (ignore alignment). + lwz(ch1, 0, addr); + lwz(ch2, 2, addr); + } else { + lhz(ch1, 0, addr); + lhz(ch2, 1, addr); + } + cmpw(CCR0, ch1, n_start); // Compare 2 characters (1 would be sufficient but try to reduce branches to CompLoop). + cmpw(CCR1, ch2, n_start); + beq(CCR0, L_Comp1); // Did we find the needle start? + beq(CCR1, L_Comp2); + addi(addr, addr, 2 * h_csize); + bdnz(L_InnerLoop); + bind(L_FinalCheck); + andi_(addr_diff, addr_diff, h_csize); // Remaining characters not covered by InnerLoop: (num of characters) & 1. + beq(CCR0, L_NotFound); + if (h_csize == 2) { lwz(ch1, 0, addr); } else { lhz(ch1, 0, addr); } // One position left at which we have to compare. + cmpw(CCR1, ch1, n_start); + beq(CCR1, L_Comp1); + bind(L_NotFound); + li(result, -1); // not found + b(L_End); + + // ************************************************************************************************** + // Special Case: unfortunately, the variable needle case can be called with needlecnt<2 + // ************************************************************************************************** + if (needlecntval == 0) { // We have to handle these cases separately. + Label L_OneCharLoop; + bind(L_TooShort); + mtctr(haycnt); + if (n_csize == 2) { lhz(n_start, 0, needle); } else { lbz(n_start, 0, needle); } // First character of needle + bind(L_OneCharLoop); + if (h_csize == 2) { lhzu(ch1, 2, addr); } else { lbzu(ch1, 1, addr); } + cmpw(CCR1, ch1, n_start); + beq(CCR1, L_Found); // Did we find the one character needle? + bdnz(L_OneCharLoop); + li(result, -1); // Not found. + b(L_End); + } + + // ************************************************************************************************** + // Regular Case Part II: compare rest of needle (first 2 characters have been compared already) + // ************************************************************************************************** + + // Compare the rest + bind(L_Comp2); + addi(addr, addr, h_csize); // First comparison has failed, 2nd one hit. + bind(L_Comp1); // Addr points to possible needle start. + if (needlecntval != 2) { // Const needlecnt==2? + if (needlecntval != 3) { + if (needlecntval == 0) { beq(CCR6, L_Found); } // Variable needlecnt==2? + Register n_ind = tmp4, + h_ind = n_ind; + li(n_ind, 2 * n_csize); // First 2 characters are already compared, use index 2. + mtctr(needlecnt); // Decremented by 2, still > 0. + Label L_CompLoop; + bind(L_CompLoop); + if (ae ==StrIntrinsicNode::UL) { + h_ind = ch1; + sldi(h_ind, n_ind, 1); + } + if (n_csize == 2) { lhzx(ch2, needle, n_ind); } else { lbzx(ch2, needle, n_ind); } + if (h_csize == 2) { lhzx(ch1, addr, h_ind); } else { lbzx(ch1, addr, h_ind); } + cmpw(CCR1, ch1, ch2); + bne(CCR1, L_OuterLoop); + addi(n_ind, n_ind, n_csize); + bdnz(L_CompLoop); + } else { // No loop required if there's only one needle character left. + if (n_csize == 2) { lhz(ch2, 2 * 2, needle); } else { lbz(ch2, 2 * 1, needle); } + if (h_csize == 2) { lhz(ch1, 2 * 2, addr); } else { lbz(ch1, 2 * 1, addr); } + cmpw(CCR1, ch1, ch2); + bne(CCR1, L_OuterLoop); + } + } + // Return index ... + bind(L_Found); + subf(result, haystack, addr); // relative to haystack, ... + if (h_csize == 2) { srdi(result, result, 1); } // in characters. + bind(L_End); + } // string_indexof + + void MacroAssembler::string_indexof_char(Register result, Register haystack, Register haycnt, + Register needle, jchar needleChar, Register tmp1, Register tmp2, bool is_byte) { + assert_different_registers(haystack, haycnt, needle, tmp1, tmp2); + + Label L_InnerLoop, L_FinalCheck, L_Found1, L_Found2, L_NotFound, L_End; + Register addr = tmp1, + ch1 = tmp2, + ch2 = R0; + + const int h_csize = is_byte ? 1 : 2; + + //4: + srwi_(tmp2, haycnt, 1); // Shift right by exact_log2(UNROLL_FACTOR). + mr(addr, haystack); + beq(CCR0, L_FinalCheck); + mtctr(tmp2); // Move to count register. + //8: + bind(L_InnerLoop); // Main work horse (2x unrolled search loop). + if (!is_byte) { + lhz(ch1, 0, addr); + lhz(ch2, 2, addr); + } else { + lbz(ch1, 0, addr); + lbz(ch2, 1, addr); + } + (needle != R0) ? cmpw(CCR0, ch1, needle) : cmplwi(CCR0, ch1, (unsigned int)needleChar); + (needle != R0) ? cmpw(CCR1, ch2, needle) : cmplwi(CCR1, ch2, (unsigned int)needleChar); + beq(CCR0, L_Found1); // Did we find the needle? + beq(CCR1, L_Found2); + addi(addr, addr, 2 * h_csize); + bdnz(L_InnerLoop); + //16: + bind(L_FinalCheck); + andi_(R0, haycnt, 1); + beq(CCR0, L_NotFound); + if (!is_byte) { lhz(ch1, 0, addr); } else { lbz(ch1, 0 ,addr); } // One position left at which we have to compare. + (needle != R0) ? cmpw(CCR1, ch1, needle) : cmplwi(CCR1, ch1, (unsigned int)needleChar); + beq(CCR1, L_Found1); + //21: + bind(L_NotFound); + li(result, -1); // Not found. + b(L_End); + + bind(L_Found2); + addi(addr, addr, h_csize); + //24: + bind(L_Found1); // Return index ... + subf(result, haystack, addr); // relative to haystack, ... + if (!is_byte) { srdi(result, result, 1); } // in characters. + bind(L_End); + } // string_indexof_char + + + void MacroAssembler::has_negatives(Register src, Register cnt, Register result, + Register tmp1, Register tmp2) { + const Register tmp0 = R0; + assert_different_registers(src, result, cnt, tmp0, tmp1, tmp2); + Label Lfastloop, Lslow, Lloop, Lnoneg, Ldone; + + // Check if cnt >= 8 (= 16 bytes) + lis(tmp1, (int)(short)0x8080); // tmp1 = 0x8080808080808080 + srwi_(tmp2, cnt, 4); + li(result, 1); // Assume there's a negative byte. + beq(CCR0, Lslow); + ori(tmp1, tmp1, 0x8080); + rldimi(tmp1, tmp1, 32, 0); + mtctr(tmp2); + + // 2x unrolled loop + bind(Lfastloop); + ld(tmp2, 0, src); + ld(tmp0, 8, src); + + orr(tmp0, tmp2, tmp0); + + and_(tmp0, tmp0, tmp1); + bne(CCR0, Ldone); // Found negative byte. + addi(src, src, 16); + + bdnz(Lfastloop); + + bind(Lslow); // Fallback to slow version + rldicl_(tmp0, cnt, 0, 64-4); + beq(CCR0, Lnoneg); + mtctr(tmp0); + bind(Lloop); + lbz(tmp0, 0, src); + addi(src, src, 1); + andi_(tmp0, tmp0, 0x80); + bne(CCR0, Ldone); // Found negative byte. + bdnz(Lloop); + bind(Lnoneg); + li(result, 0); + + bind(Ldone); + } + + + // Intrinsics for non-CompactStrings + // Search for a single jchar in an jchar[]. // // Assumes that result differs from all other registers. // // 'haystack' is the addresses of a jchar-array.
*** 3611,3620 **** --- 4161,4172 ---- } li(result_reg, 1); bind(Ldone_false); } + #endif // Compiler2 + // Helpers for Intrinsic Emitters // // Revert the byte order of a 32bit value in a register // src: 0x44556677 // dst: 0x77665544
< prev index next >