919 // Simply use distance from start of const section (should be patched in the end).
920 long disp = toc_distance();
921
922 RelocationHolder rspec = internal_word_Relocation::spec(pc() + disp);
923 relocate(rspec);
924 z_larl(Rtoc, RelAddr::pcrel_off32(disp)); // Offset is in halfwords.
925 }
926
927 // PCrelative TOC access.
928 // Load from anywhere pcrelative (with relocation of load instr)
929 void MacroAssembler::load_long_pcrelative(Register Rdst, address dataLocation) {
930 address pc = this->pc();
931 ptrdiff_t total_distance = dataLocation - pc;
932 RelocationHolder rspec = internal_word_Relocation::spec(dataLocation);
933
934 assert((total_distance & 0x01L) == 0, "halfword alignment is mandatory");
935 assert(total_distance != 0, "sanity");
936
937 // Some extra safety net.
938 if (!RelAddr::is_in_range_of_RelAddr32(total_distance)) {
939 guarantee(RelAddr::is_in_range_of_RelAddr32(total_distance), "too far away");
940 }
941
942 (this)->relocate(rspec, relocInfo::pcrel_addr_format);
943 z_lgrl(Rdst, RelAddr::pcrel_off32(total_distance));
944 }
945
946
947 // PCrelative TOC access.
948 // Load from anywhere pcrelative (with relocation of load instr)
949 // loaded addr has to be relocated when added to constant pool.
950 void MacroAssembler::load_addr_pcrelative(Register Rdst, address addrLocation) {
951 address pc = this->pc();
952 ptrdiff_t total_distance = addrLocation - pc;
953 RelocationHolder rspec = internal_word_Relocation::spec(addrLocation);
954
955 assert((total_distance & 0x01L) == 0, "halfword alignment is mandatory");
956
957 // Some extra safety net.
958 if (!RelAddr::is_in_range_of_RelAddr32(total_distance)) {
959 guarantee(RelAddr::is_in_range_of_RelAddr32(total_distance), "too far away");
960 }
961
962 (this)->relocate(rspec, relocInfo::pcrel_addr_format);
963 z_lgrl(Rdst, RelAddr::pcrel_off32(total_distance));
964 }
965
966 // Generic operation: load a value from memory and test.
967 // CondCode indicates the sign (<0, ==0, >0) of the loaded value.
968 void MacroAssembler::load_and_test_byte(Register dst, const Address &a) {
969 z_lb(dst, a);
970 z_ltr(dst, dst);
971 }
972
973 void MacroAssembler::load_and_test_short(Register dst, const Address &a) {
974 int64_t disp = a.disp20();
975 if (Displacement::is_shortDisp(disp)) {
976 z_lh(dst, a);
977 } else if (Displacement::is_longDisp(disp)) {
978 z_lhy(dst, a);
979 } else {
1008 } else {
1009 ShouldNotReachHere();
1010 }
1011 }
1012
1013 // Test a bit in a register. Result is reflected in CC.
1014 void MacroAssembler::testbit(Register r, unsigned int bitPos) {
1015 if (bitPos < 16) {
1016 z_tmll(r, 1U<<bitPos);
1017 } else if (bitPos < 32) {
1018 z_tmlh(r, 1U<<(bitPos-16));
1019 } else if (bitPos < 48) {
1020 z_tmhl(r, 1U<<(bitPos-32));
1021 } else if (bitPos < 64) {
1022 z_tmhh(r, 1U<<(bitPos-48));
1023 } else {
1024 ShouldNotReachHere();
1025 }
1026 }
1027
1028 // Clear a register, i.e. load const zero into reg.
1029 // Return len (in bytes) of generated instruction(s).
1030 // whole_reg: Clear 64 bits if true, 32 bits otherwise.
1031 // set_cc: Use instruction that sets the condition code, if true.
1032 int MacroAssembler::clear_reg(Register r, bool whole_reg, bool set_cc) {
1033 unsigned int start_off = offset();
1034 if (whole_reg) {
1035 set_cc ? z_xgr(r, r) : z_laz(r, 0, Z_R0);
1036 } else { // Only 32bit register.
1037 set_cc ? z_xr(r, r) : z_lhi(r, 0);
1038 }
1039 return offset() - start_off;
1040 }
1041
1042 #ifdef ASSERT
1043 int MacroAssembler::preset_reg(Register r, unsigned long pattern, int pattern_len) {
1044 switch (pattern_len) {
1045 case 1:
1046 pattern = (pattern & 0x000000ff) | ((pattern & 0x000000ff)<<8);
1047 case 2:
4879 if (VM_Version::has_ExecuteExtensions()) {
4880 z_exrl(Z_R1, MVC_template);
4881 } else {
4882 z_ex(tmp1_reg, 0, Z_R0, Z_R1);
4883 }
4884
4885 bind(done);
4886
4887 BLOCK_COMMENT("} CopyRawMemory_AlignedDisjoint");
4888
4889 int block_end = offset();
4890 return block_end - block_start;
4891 }
4892
4893 //------------------------------------------------------
4894 // Special String Intrinsics. Implementation
4895 //------------------------------------------------------
4896
4897 // Intrinsics for CompactStrings
4898
4899 // Compress char[] to byte[]. odd_reg contains cnt. Kills dst. Early clobber: result
4900 // The result is the number of characters copied before the first incompatible character was found.
4901 // If tmp2 is provided and the compression fails, the compression stops exactly at this point and the result is precise.
4902 //
4903 // Note: Does not behave exactly like package private StringUTF16 compress java implementation in case of failure:
4904 // - Different number of characters may have been written to dead array (if tmp2 not provided).
4905 // - Returns a number <cnt instead of 0. (Result gets compared with cnt.)
4906 unsigned int MacroAssembler::string_compress(Register result, Register src, Register dst, Register odd_reg,
4907 Register even_reg, Register tmp, Register tmp2) {
4908 int block_start = offset();
4909 Label Lloop1, Lloop2, Lslow, Ldone;
4910 const Register addr2 = dst, ind1 = result, mask = tmp;
4911 const bool precise = (tmp2 != noreg);
4912
4913 BLOCK_COMMENT("string_compress {");
4914
4915 z_sll(odd_reg, 1); // Number of bytes to read. (Must be a positive simm32.)
4916 clear_reg(ind1); // Index to read.
4917 z_llilf(mask, 0xFF00FF00);
4918 z_ahi(odd_reg, -16); // Last possible index for fast loop.
4919 z_brl(Lslow);
4920
4921 // ind1: index, even_reg: index increment, odd_reg: index limit
4922 z_iihf(mask, 0xFF00FF00);
4923 z_lhi(even_reg, 16);
4924
4925 bind(Lloop1); // 8 Characters per iteration.
4926 z_lg(Z_R0, Address(src, ind1));
4927 z_lg(Z_R1, Address(src, ind1, 8));
4928 if (precise) {
4929 if (VM_Version::has_DistinctOpnds()) {
4930 z_ogrk(tmp2, Z_R0, Z_R1);
4931 } else {
4932 z_lgr(tmp2, Z_R0);
4933 z_ogr(tmp2, Z_R1);
4934 }
4935 z_ngr(tmp2, mask);
4936 z_brne(Lslow); // Failed fast case, retry slowly.
4937 }
4938 z_stcmh(Z_R0, 5, 0, addr2);
4939 z_stcm(Z_R0, 5, 2, addr2);
4940 if (!precise) { z_ogr(Z_R0, Z_R1); }
4941 z_stcmh(Z_R1, 5, 4, addr2);
4942 z_stcm(Z_R1, 5, 6, addr2);
4943 if (!precise) {
4944 z_ngr(Z_R0, mask);
4945 z_brne(Ldone); // Failed (more than needed was written).
4946 }
4947 z_aghi(addr2, 8);
4948 z_brxle(ind1, even_reg, Lloop1);
4949
4950 bind(Lslow);
4951 // Compute index limit and skip if negative.
4952 z_ahi(odd_reg, 16-2); // Last possible index for slow loop.
4953 z_lhi(even_reg, 2);
4954 z_cr(ind1, odd_reg);
4955 z_brh(Ldone);
4956
4957 bind(Lloop2); // 1 Character per iteration.
4958 z_llh(Z_R0, Address(src, ind1));
4959 z_tmll(Z_R0, 0xFF00);
4960 z_brnaz(Ldone); // Failed slow case: Return number of written characters.
4961 z_stc(Z_R0, Address(addr2));
4962 z_aghi(addr2, 1);
4963 z_brxle(ind1, even_reg, Lloop2);
4964
4965 bind(Ldone); // result = ind1 = 2*cnt
4966 z_srl(ind1, 1);
4967
4968 BLOCK_COMMENT("} string_compress");
4969
4970 return offset() - block_start;
4971 }
4972
4973 // Inflate byte[] to char[].
4974 unsigned int MacroAssembler::string_inflate_trot(Register src, Register dst, Register cnt, Register tmp) {
4975 int block_start = offset();
4976
4977 BLOCK_COMMENT("string_inflate {");
4978
4979 Register stop_char = Z_R0;
4980 Register table = Z_R1;
4981 Register src_addr = tmp;
4982
4983 assert_different_registers(Z_R0, Z_R1, tmp, src, dst, cnt);
4984 assert(dst->encoding()%2 == 0, "must be even reg");
4985 assert(cnt->encoding()%2 == 1, "must be odd reg");
4986 assert(cnt->encoding() - dst->encoding() == 1, "must be even/odd pair");
4987
4988 StubRoutines::zarch::generate_load_trot_table_addr(this, table); // kills Z_R0 (if ASSERT)
4989 clear_reg(stop_char); // Stop character. Not used here, but initialized to have a defined value.
4990 lgr_if_needed(src_addr, src);
4991 z_llgfr(cnt, cnt); // # src characters, must be a positive simm32.
4992
4993 translate_ot(dst, src_addr, /* mask = */ 0x0001);
4994
4995 BLOCK_COMMENT("} string_inflate");
4996
4997 return offset() - block_start;
4998 }
4999
5000 // Inflate byte[] to char[]. odd_reg contains cnt. Kills src.
5001 unsigned int MacroAssembler::string_inflate(Register src, Register dst, Register odd_reg,
5002 Register even_reg, Register tmp) {
5003 int block_start = offset();
5004
5005 BLOCK_COMMENT("string_inflate {");
5006
5007 Label Lloop1, Lloop2, Lslow, Ldone;
5008 const Register addr1 = src, ind2 = tmp;
5009
5010 z_sll(odd_reg, 1); // Number of bytes to write. (Must be a positive simm32.)
5011 clear_reg(ind2); // Index to write.
5012 z_ahi(odd_reg, -16); // Last possible index for fast loop.
5013 z_brl(Lslow);
5014
5015 // ind2: index, even_reg: index increment, odd_reg: index limit
5016 clear_reg(Z_R0);
5017 clear_reg(Z_R1);
5018 z_lhi(even_reg, 16);
5019
5020 bind(Lloop1); // 8 Characters per iteration.
5021 z_icmh(Z_R0, 5, 0, addr1);
5022 z_icmh(Z_R1, 5, 4, addr1);
5023 z_icm(Z_R0, 5, 2, addr1);
5024 z_icm(Z_R1, 5, 6, addr1);
5025 z_aghi(addr1, 8);
5026 z_stg(Z_R0, Address(dst, ind2));
5027 z_stg(Z_R1, Address(dst, ind2, 8));
5028 z_brxle(ind2, even_reg, Lloop1);
5029
5030 bind(Lslow);
5031 // Compute index limit and skip if negative.
5032 z_ahi(odd_reg, 16-2); // Last possible index for slow loop.
5033 z_lhi(even_reg, 2);
5034 z_cr(ind2, odd_reg);
5035 z_brh(Ldone);
5036
5037 bind(Lloop2); // 1 Character per iteration.
5038 z_llc(Z_R0, Address(addr1));
5039 z_sth(Z_R0, Address(dst, ind2));
5040 z_aghi(addr1, 1);
5041 z_brxle(ind2, even_reg, Lloop2);
5042
5043 bind(Ldone);
5044
5045 BLOCK_COMMENT("} string_inflate");
5046
5047 return offset() - block_start;
5048 }
5049
5050 // Kills src.
5051 unsigned int MacroAssembler::has_negatives(Register result, Register src, Register cnt,
5052 Register odd_reg, Register even_reg, Register tmp) {
5053 int block_start = offset();
5054 Label Lloop1, Lloop2, Lslow, Lnotfound, Ldone;
5055 const Register addr = src, mask = tmp;
5056
5057 BLOCK_COMMENT("has_negatives {");
5058
5059 z_llgfr(Z_R1, cnt); // Number of bytes to read. (Must be a positive simm32.)
5060 z_llilf(mask, 0x80808080);
5061 z_lhi(result, 1); // Assume true.
5062 // Last possible addr for fast loop.
5063 z_lay(odd_reg, -16, Z_R1, src);
5064 z_chi(cnt, 16);
5065 z_brl(Lslow);
5066
|
919 // Simply use distance from start of const section (should be patched in the end).
920 long disp = toc_distance();
921
922 RelocationHolder rspec = internal_word_Relocation::spec(pc() + disp);
923 relocate(rspec);
924 z_larl(Rtoc, RelAddr::pcrel_off32(disp)); // Offset is in halfwords.
925 }
926
927 // PCrelative TOC access.
928 // Load from anywhere pcrelative (with relocation of load instr)
929 void MacroAssembler::load_long_pcrelative(Register Rdst, address dataLocation) {
930 address pc = this->pc();
931 ptrdiff_t total_distance = dataLocation - pc;
932 RelocationHolder rspec = internal_word_Relocation::spec(dataLocation);
933
934 assert((total_distance & 0x01L) == 0, "halfword alignment is mandatory");
935 assert(total_distance != 0, "sanity");
936
937 // Some extra safety net.
938 if (!RelAddr::is_in_range_of_RelAddr32(total_distance)) {
939 guarantee(RelAddr::is_in_range_of_RelAddr32(total_distance), "load_long_pcrelative can't handle distance " INTPTR_FORMAT, total_distance);
940 }
941
942 (this)->relocate(rspec, relocInfo::pcrel_addr_format);
943 z_lgrl(Rdst, RelAddr::pcrel_off32(total_distance));
944 }
945
946
947 // PCrelative TOC access.
948 // Load from anywhere pcrelative (with relocation of load instr)
949 // loaded addr has to be relocated when added to constant pool.
950 void MacroAssembler::load_addr_pcrelative(Register Rdst, address addrLocation) {
951 address pc = this->pc();
952 ptrdiff_t total_distance = addrLocation - pc;
953 RelocationHolder rspec = internal_word_Relocation::spec(addrLocation);
954
955 assert((total_distance & 0x01L) == 0, "halfword alignment is mandatory");
956
957 // Some extra safety net.
958 if (!RelAddr::is_in_range_of_RelAddr32(total_distance)) {
959 guarantee(RelAddr::is_in_range_of_RelAddr32(total_distance), "load_long_pcrelative can't handle distance " INTPTR_FORMAT, total_distance);
960 }
961
962 (this)->relocate(rspec, relocInfo::pcrel_addr_format);
963 z_lgrl(Rdst, RelAddr::pcrel_off32(total_distance));
964 }
965
966 // Generic operation: load a value from memory and test.
967 // CondCode indicates the sign (<0, ==0, >0) of the loaded value.
968 void MacroAssembler::load_and_test_byte(Register dst, const Address &a) {
969 z_lb(dst, a);
970 z_ltr(dst, dst);
971 }
972
973 void MacroAssembler::load_and_test_short(Register dst, const Address &a) {
974 int64_t disp = a.disp20();
975 if (Displacement::is_shortDisp(disp)) {
976 z_lh(dst, a);
977 } else if (Displacement::is_longDisp(disp)) {
978 z_lhy(dst, a);
979 } else {
1008 } else {
1009 ShouldNotReachHere();
1010 }
1011 }
1012
1013 // Test a bit in a register. Result is reflected in CC.
1014 void MacroAssembler::testbit(Register r, unsigned int bitPos) {
1015 if (bitPos < 16) {
1016 z_tmll(r, 1U<<bitPos);
1017 } else if (bitPos < 32) {
1018 z_tmlh(r, 1U<<(bitPos-16));
1019 } else if (bitPos < 48) {
1020 z_tmhl(r, 1U<<(bitPos-32));
1021 } else if (bitPos < 64) {
1022 z_tmhh(r, 1U<<(bitPos-48));
1023 } else {
1024 ShouldNotReachHere();
1025 }
1026 }
1027
1028 void MacroAssembler::prefetch_read(Address a) {
1029 z_pfd(1, a.disp20(), a.indexOrR0(), a.base());
1030 }
1031 void MacroAssembler::prefetch_update(Address a) {
1032 z_pfd(2, a.disp20(), a.indexOrR0(), a.base());
1033 }
1034
1035 // Clear a register, i.e. load const zero into reg.
1036 // Return len (in bytes) of generated instruction(s).
1037 // whole_reg: Clear 64 bits if true, 32 bits otherwise.
1038 // set_cc: Use instruction that sets the condition code, if true.
1039 int MacroAssembler::clear_reg(Register r, bool whole_reg, bool set_cc) {
1040 unsigned int start_off = offset();
1041 if (whole_reg) {
1042 set_cc ? z_xgr(r, r) : z_laz(r, 0, Z_R0);
1043 } else { // Only 32bit register.
1044 set_cc ? z_xr(r, r) : z_lhi(r, 0);
1045 }
1046 return offset() - start_off;
1047 }
1048
1049 #ifdef ASSERT
1050 int MacroAssembler::preset_reg(Register r, unsigned long pattern, int pattern_len) {
1051 switch (pattern_len) {
1052 case 1:
1053 pattern = (pattern & 0x000000ff) | ((pattern & 0x000000ff)<<8);
1054 case 2:
4886 if (VM_Version::has_ExecuteExtensions()) {
4887 z_exrl(Z_R1, MVC_template);
4888 } else {
4889 z_ex(tmp1_reg, 0, Z_R0, Z_R1);
4890 }
4891
4892 bind(done);
4893
4894 BLOCK_COMMENT("} CopyRawMemory_AlignedDisjoint");
4895
4896 int block_end = offset();
4897 return block_end - block_start;
4898 }
4899
4900 //------------------------------------------------------
4901 // Special String Intrinsics. Implementation
4902 //------------------------------------------------------
4903
4904 // Intrinsics for CompactStrings
4905
4906 // Compress char[] to byte[].
4907 // Restores: src, dst
4908 // Uses: cnt
4909 // Kills: tmp, Z_R0, Z_R1.
4910 // Early clobber: result.
4911 // Note:
4912 // cnt is signed int. Do not rely on high word!
4913 // counts # characters, not bytes.
4914 // The result is the number of characters copied before the first incompatible character was found.
4915 // If precise is true, the processing stops exactly at this point. Otherwise, the result may be off
4916 // by a few bytes. The result always indicates the number of copied characters.
4917 //
4918 // Note: Does not behave exactly like package private StringUTF16 compress java implementation in case of failure:
4919 // - Different number of characters may have been written to dead array (if precise is false).
4920 // - Returns a number <cnt instead of 0. (Result gets compared with cnt.)
4921 unsigned int MacroAssembler::string_compress(Register result, Register src, Register dst, Register cnt,
4922 Register tmp, bool precise) {
4923 assert_different_registers(Z_R0, Z_R1, src, dst, cnt, tmp);
4924
4925 if (precise) {
4926 BLOCK_COMMENT("encode_iso_array {");
4927 } else {
4928 BLOCK_COMMENT("string_compress {");
4929 }
4930 int block_start = offset();
4931
4932 Register Rsrc = src;
4933 Register Rdst = dst;
4934 Register Rix = tmp;
4935 Register Rcnt = cnt;
4936 Register Rmask = result; // holds incompatibility check mask until result value is stored.
4937 Label ScalarShortcut, AllDone;
4938
4939 if (VM_Version::has_Prefetch()) {
4940 prefetch_read(Address(Rsrc)); // make sure first cache line is prepared.
4941 prefetch_update(Address(Rdst));
4942 }
4943 z_iilf(Rmask, 0xFF00FF00);
4944 z_iihf(Rmask, 0xFF00FF00);
4945
4946 {
4947 //---< shortcuts for short strings (very frequent) >---
4948 // Strings with 4 and 8 characters were fond to occur very frequently.
4949 // Therefore, we handle them right away with minimal overhead.
4950 Label skipShortcut, skip4Shortcut;
4951 z_chi(Rcnt, 4);
4952 z_brne(skip4Shortcut); // 4 characters are very frequent
4953 z_lg(Z_R0, 0, Rsrc); // Treat exactly 4 characters specially.
4954 if (VM_Version::has_DistinctOpnds()) {
4955 z_ngrk(Rix, Z_R0, Rmask);
4956 } else {
4957 z_lgr(Rix, Z_R0);
4958 z_ngr(Rix, Rmask);
4959 }
4960 z_brnz(skipShortcut);
4961 z_stcmh(Z_R0, 5, 0, Rdst);
4962 z_stcm(Z_R0, 5, 2, Rdst);
4963 z_lgfr(result, Rcnt);
4964 z_bru(AllDone);
4965 bind(skip4Shortcut);
4966 z_chi(Rcnt, 8);
4967 z_bru(skipShortcut); // There's more to do...
4968 clear_reg(Z_R0, true, false); // #characters already processed (none). Precond for scalar loop.
4969 z_brl(ScalarShortcut); // Just a few characters
4970 z_lmg(Z_R0, Z_R1, 0, Rsrc); // Treat exactly 8 characters specially.
4971 if (VM_Version::has_DistinctOpnds()) {
4972 z_ogrk(Rix, Z_R0, Z_R1);
4973 } else {
4974 z_lgr(Rix, Z_R0);
4975 z_ogr(Rix, Z_R1);
4976 }
4977 z_ngr(Rix, Rmask);
4978 z_brnz(skipShortcut);
4979 z_stcmh(Z_R0, 5, 0, Rdst);
4980 z_stcm(Z_R0, 5, 2, Rdst);
4981 z_stcmh(Z_R1, 5, 4, Rdst);
4982 z_stcm(Z_R1, 5, 6, Rdst);
4983 z_lgfr(result, Rcnt);
4984 z_bru(AllDone);
4985 bind(skipShortcut);
4986 }
4987 clear_reg(Z_R0); // make sure register is properly initialized.
4988
4989 if (VM_Version::has_VectorFacility()) {
4990 const int min_vcnt = 32; // Minimum #characters required to use vector instructions.
4991 // Otherwise just do nothing in vector mode.
4992 // Must be multiple of 2*(vector register length in chars (8 HW = 128 bits)).
4993 const int log_min_vcnt = exact_log2(min_vcnt);
4994 Label VectorLoop, VectorDone, VectorBreak;
4995
4996 VectorRegister Vtmp1 = Z_V16;
4997 VectorRegister Vtmp2 = Z_V17;
4998 VectorRegister Vmask = Z_V18;
4999 VectorRegister Vzero = Z_V19;
5000 VectorRegister Vsrc_first = Z_V20;
5001 VectorRegister Vsrc_last = Z_V23;
5002
5003 assert((Vsrc_last->encoding() - Vsrc_first->encoding() + 1) == min_vcnt/8, "logic error");
5004 assert(VM_Version::has_DistinctOpnds(), "Assumption when has_VectorFacility()");
5005 z_srak(Rix, Rcnt, log_min_vcnt); // # vector loop iterations
5006 z_brz(VectorDone); // not enough data for vector loop
5007
5008 z_vzero(Vzero); // all zeroes
5009 z_vgmh(Vmask, 0, 7); // generate 0xff00 mask for all 2-byte elements
5010 z_sllg(Z_R0, Rix, log_min_vcnt); // remember #chars that will be processed by vector loop
5011
5012 bind(VectorLoop);
5013 z_vlm(Vsrc_first, Vsrc_last, 0, Rsrc);
5014 add2reg(Rsrc, min_vcnt*2);
5015
5016 //---< check for incompatible character >---
5017 z_vo(Vtmp1, Z_V20, Z_V21);
5018 z_vo(Vtmp2, Z_V22, Z_V23);
5019 z_vo(Vtmp1, Vtmp1, Vtmp2);
5020 z_vn(Vtmp1, Vtmp1, Vmask);
5021 z_vceqhs(Vtmp1, Vtmp1, Vzero); // high half of all chars must be zero for successful compress.
5022 z_brne(VectorBreak); // break vector loop, incompatible character found.
5023 // re-process data from current iteration in break handler.
5024
5025 //---< pack & store characters >---
5026 z_vpkh(Vtmp1, Z_V20, Z_V21); // pack (src1, src2) -> tmp1
5027 z_vpkh(Vtmp2, Z_V22, Z_V23); // pack (src3, src4) -> tmp2
5028 z_vstm(Vtmp1, Vtmp2, 0, Rdst); // store packed string
5029 add2reg(Rdst, min_vcnt);
5030
5031 z_brct(Rix, VectorLoop);
5032
5033 z_bru(VectorDone);
5034
5035 bind(VectorBreak);
5036 z_sll(Rix, log_min_vcnt); // # chars processed so far in VectorLoop, excl. current iteration.
5037 z_sr(Z_R0, Rix); // correct # chars processed in total.
5038
5039 bind(VectorDone);
5040 }
5041
5042 {
5043 const int min_cnt = 8; // Minimum #characters required to use unrolled loop.
5044 // Otherwise just do nothing in unrolled loop.
5045 // Must be multiple of 8.
5046 const int log_min_cnt = exact_log2(min_cnt);
5047 Label UnrolledLoop, UnrolledDone, UnrolledBreak;
5048
5049 if (VM_Version::has_DistinctOpnds()) {
5050 z_srk(Rix, Rcnt, Z_R0); // remaining # chars to compress in unrolled loop
5051 } else {
5052 z_lr(Rix, Rcnt);
5053 z_sr(Rix, Z_R0);
5054 }
5055 z_sra(Rix, log_min_cnt); // unrolled loop count
5056 z_brz(UnrolledDone);
5057
5058 bind(UnrolledLoop);
5059 z_lmg(Z_R0, Z_R1, 0, Rsrc);
5060 if (precise) {
5061 z_ogr(Z_R1, Z_R0); // check all 8 chars for incompatibility
5062 z_ngr(Z_R1, Rmask);
5063 z_brnz(UnrolledBreak);
5064
5065 z_lg(Z_R1, 8, Rsrc); // reload destroyed register
5066 z_stcmh(Z_R0, 5, 0, Rdst);
5067 z_stcm(Z_R0, 5, 2, Rdst);
5068 } else {
5069 z_stcmh(Z_R0, 5, 0, Rdst);
5070 z_stcm(Z_R0, 5, 2, Rdst);
5071
5072 z_ogr(Z_R0, Z_R1);
5073 z_ngr(Z_R0, Rmask);
5074 z_brnz(UnrolledBreak);
5075 }
5076 z_stcmh(Z_R1, 5, 4, Rdst);
5077 z_stcm(Z_R1, 5, 6, Rdst);
5078
5079 add2reg(Rsrc, min_cnt*2);
5080 add2reg(Rdst, min_cnt);
5081 z_brct(Rix, UnrolledLoop);
5082
5083 z_lgfr(Z_R0, Rcnt); // # chars processed in total after unrolled loop.
5084 z_nilf(Z_R0, ~(min_cnt-1));
5085 z_tmll(Rcnt, min_cnt-1);
5086 z_brnaz(ScalarShortcut); // if all bits zero, there is nothing left to do for scalar loop.
5087 // Rix == 0 in all cases.
5088 z_lgfr(result, Rcnt); // all characters processed.
5089 z_sgfr(Rdst, Rcnt); // restore ptr
5090 z_sgfr(Rsrc, Rcnt); // restore ptr, double the element count for Rsrc restore
5091 z_sgfr(Rsrc, Rcnt);
5092 z_bru(AllDone);
5093
5094 bind(UnrolledBreak);
5095 z_lgfr(Z_R0, Rcnt); // # chars processed in total after unrolled loop
5096 z_nilf(Z_R0, ~(min_cnt-1));
5097 z_sll(Rix, log_min_cnt); // # chars processed so far in UnrolledLoop, excl. current iteration.
5098 z_sr(Z_R0, Rix); // correct # chars processed in total.
5099 if (!precise) {
5100 z_lgfr(result, Z_R0);
5101 z_aghi(result, min_cnt/2); // min_cnt/2 characters have already been written
5102 // but ptrs were not updated yet.
5103 z_sgfr(Rdst, Z_R0); // restore ptr
5104 z_sgfr(Rsrc, Z_R0); // restore ptr, double the element count for Rsrc restore
5105 z_sgfr(Rsrc, Z_R0);
5106 z_bru(AllDone);
5107 }
5108 bind(UnrolledDone);
5109 }
5110
5111 {
5112 Label ScalarLoop, ScalarDone, ScalarBreak;
5113
5114 bind(ScalarShortcut);
5115
5116 {
5117 //---< Special treatment for very short strings (one or two characters) >---
5118 // For these strings, we are sure that the above code was skipped.
5119 // Thus, no registers were modified, register restore is not required.
5120 Label ScalarDoit, Scalar2Char;
5121 z_chi(Rcnt, 2);
5122 z_brh(ScalarDoit);
5123 z_llh(Z_R1, 0, Z_R0, Rsrc);
5124 z_bre(Scalar2Char);
5125 z_tmll(Z_R1, 0xff00);
5126 z_lghi(result, 0); // cnt == 1, first char invalid, no chars successfully processed
5127 z_brnaz(AllDone);
5128 z_stc(Z_R1, 0, Z_R0, Rdst);
5129 z_lghi(result, 1);
5130 z_bru(AllDone);
5131
5132 bind(Scalar2Char);
5133 z_llh(Z_R0, 2, Z_R0, Rsrc);
5134 z_tmll(Z_R1, 0xff00);
5135 z_lghi(result, 0); // cnt == 2, first char invalid, no chars successfully processed
5136 z_brnaz(AllDone);
5137 z_stc(Z_R1, 0, Z_R0, Rdst);
5138 z_tmll(Z_R0, 0xff00);
5139 z_lghi(result, 1); // cnt == 2, second char invalid, one char successfully processed
5140 z_brnaz(AllDone);
5141 z_stc(Z_R0, 1, Z_R0, Rdst);
5142 z_lghi(result, 2);
5143 z_bru(AllDone);
5144
5145 bind(ScalarDoit);
5146 }
5147
5148 if (VM_Version::has_DistinctOpnds()) {
5149 z_srk(Rix, Rcnt, Z_R0); // remaining # chars to compress in unrolled loop
5150 } else {
5151 z_lr(Rix, Rcnt);
5152 z_sr(Rix, Z_R0);
5153 }
5154 z_lgfr(result, Rcnt); // # processed characters (if all runs ok).
5155 z_brz(ScalarDone);
5156
5157 bind(ScalarLoop);
5158 z_llh(Z_R1, 0, Z_R0, Rsrc);
5159 z_tmll(Z_R1, 0xff00);
5160 z_brnaz(ScalarBreak);
5161 z_stc(Z_R1, 0, Z_R0, Rdst);
5162 add2reg(Rsrc, 2);
5163 add2reg(Rdst, 1);
5164 z_brct(Rix, ScalarLoop);
5165
5166 z_bru(ScalarDone);
5167
5168 bind(ScalarBreak);
5169 z_sr(result, Rix);
5170
5171 bind(ScalarDone);
5172 z_sgfr(Rdst, result); // restore ptr
5173 z_sgfr(Rsrc, result); // restore ptr, double the element count for Rsrc restore
5174 z_sgfr(Rsrc, result);
5175 }
5176 bind(AllDone);
5177
5178 if (precise) {
5179 BLOCK_COMMENT("} encode_iso_array");
5180 } else {
5181 BLOCK_COMMENT("} string_compress");
5182 }
5183 return offset() - block_start;
5184 }
5185
5186 // Inflate byte[] to char[].
5187 unsigned int MacroAssembler::string_inflate_trot(Register src, Register dst, Register cnt, Register tmp) {
5188 int block_start = offset();
5189
5190 BLOCK_COMMENT("string_inflate {");
5191
5192 Register stop_char = Z_R0;
5193 Register table = Z_R1;
5194 Register src_addr = tmp;
5195
5196 assert_different_registers(Z_R0, Z_R1, tmp, src, dst, cnt);
5197 assert(dst->encoding()%2 == 0, "must be even reg");
5198 assert(cnt->encoding()%2 == 1, "must be odd reg");
5199 assert(cnt->encoding() - dst->encoding() == 1, "must be even/odd pair");
5200
5201 StubRoutines::zarch::generate_load_trot_table_addr(this, table); // kills Z_R0 (if ASSERT)
5202 clear_reg(stop_char); // Stop character. Not used here, but initialized to have a defined value.
5203 lgr_if_needed(src_addr, src);
5204 z_llgfr(cnt, cnt); // # src characters, must be a positive simm32.
5205
5206 translate_ot(dst, src_addr, /* mask = */ 0x0001);
5207
5208 BLOCK_COMMENT("} string_inflate");
5209
5210 return offset() - block_start;
5211 }
5212
5213 // Inflate byte[] to char[].
5214 // Restores: src, dst
5215 // Uses: cnt
5216 // Kills: tmp, Z_R0, Z_R1.
5217 // Note:
5218 // cnt is signed int. Do not rely on high word!
5219 // counts # characters, not bytes.
5220 unsigned int MacroAssembler::string_inflate(Register src, Register dst, Register cnt, Register tmp) {
5221 assert_different_registers(Z_R0, Z_R1, src, dst, cnt, tmp);
5222
5223 BLOCK_COMMENT("string_inflate {");
5224 int block_start = offset();
5225
5226 Register Rcnt = cnt; // # characters (src: bytes, dst: char (2-byte)), remaining after current loop.
5227 Register Rix = tmp; // loop index
5228 Register Rsrc = src; // addr(src array)
5229 Register Rdst = dst; // addr(dst array)
5230 Label ScalarShortcut, AllDone;
5231
5232 if (VM_Version::has_Prefetch()) {
5233 prefetch_read(Address(Rsrc)); // make sure first cache line is prepared.
5234 prefetch_update(Address(Rdst));
5235 }
5236
5237 {
5238 //---< shortcuts for short strings (very frequent) >---
5239 Label skipShortcut, skip4Shortcut;
5240 clear_reg(Z_R0); // make sure register is properly initialized.
5241 z_chi(Rcnt, 4);
5242 z_brne(skip4Shortcut); // 4 characters are very frequent
5243 z_icmh(Z_R0, 5, 0, Rsrc); // Treat exactly 8 characters specially.
5244 z_icm(Z_R0, 5, 2, Rsrc);
5245 z_stg(Z_R0, 0, Rdst);
5246 z_bru(AllDone);
5247 bind(skip4Shortcut);
5248
5249 z_chi(Rcnt, 8);
5250 z_brh(skipShortcut); // There's a lot to do...
5251 z_lgfr(Z_R0, Rcnt); // remaining #characters (<= 8). Precond for scalar loop.
5252 // This does not destroy the "register cleared" state of Z_R0.
5253 z_brl(ScalarShortcut); // Just a few characters
5254 clear_reg(Z_R1);
5255 z_icmh(Z_R0, 5, 0, Rsrc); // Treat exactly 8 characters specially.
5256 z_icmh(Z_R1, 5, 4, Rsrc);
5257 z_icm(Z_R0, 5, 2, Rsrc);
5258 z_icm(Z_R1, 5, 6, Rsrc);
5259 z_stmg(Z_R0, Z_R1, 0, Rdst);
5260 z_bru(AllDone);
5261 bind(skipShortcut);
5262 }
5263 clear_reg(Z_R0); // make sure register is properly initialized.
5264
5265 if (VM_Version::has_VectorFacility()) {
5266 const int min_vcnt = 32; // Minimum #characters required to use vector instructions.
5267 // Otherwise just do nothing in vector mode.
5268 // Must be multiple of vector register length (16 bytes = 128 bits).
5269 const int log_min_vcnt = exact_log2(min_vcnt);
5270 Label VectorLoop, VectorDone;
5271
5272 assert(VM_Version::has_DistinctOpnds(), "Assumption when has_VectorFacility()");
5273 z_srak(Rix, Rcnt, log_min_vcnt); // calculate # vector loop iterations
5274 z_brz(VectorDone); // skip if none
5275
5276 z_sllg(Z_R0, Rix, log_min_vcnt); // remember #chars that will be processed by vector loop
5277
5278 bind(VectorLoop);
5279 z_vlm(Z_V20, Z_V21, 0, Rsrc); // get next 32 characters (single-byte)
5280 add2reg(Rsrc, min_vcnt);
5281
5282 z_vuplhb(Z_V22, Z_V20); // V2 <- (expand) V0(high)
5283 z_vupllb(Z_V23, Z_V20); // V3 <- (expand) V0(low)
5284 z_vuplhb(Z_V24, Z_V21); // V4 <- (expand) V1(high)
5285 z_vupllb(Z_V25, Z_V21); // V5 <- (expand) V1(low)
5286 z_vstm(Z_V22, Z_V25, 0, Rdst); // store next 32 bytes
5287 add2reg(Rdst, min_vcnt*2);
5288
5289 z_brct(Rix, VectorLoop);
5290
5291 bind(VectorDone);
5292 }
5293
5294 const int min_cnt = 8; // Minimum #characters required to use unrolled scalar loop.
5295 // Otherwise just do nothing in unrolled scalar mode.
5296 // Must be multiple of 8.
5297 {
5298 const int log_min_cnt = exact_log2(min_cnt);
5299 Label UnrolledLoop, UnrolledDone;
5300
5301
5302 if (VM_Version::has_DistinctOpnds()) {
5303 z_srk(Rix, Rcnt, Z_R0); // remaining # chars to process in unrolled loop
5304 } else {
5305 z_lr(Rix, Rcnt);
5306 z_sr(Rix, Z_R0);
5307 }
5308 z_sra(Rix, log_min_cnt); // unrolled loop count
5309 z_brz(UnrolledDone);
5310
5311 clear_reg(Z_R0);
5312 clear_reg(Z_R1);
5313
5314 bind(UnrolledLoop);
5315 z_icmh(Z_R0, 5, 0, Rsrc);
5316 z_icmh(Z_R1, 5, 4, Rsrc);
5317 z_icm(Z_R0, 5, 2, Rsrc);
5318 z_icm(Z_R1, 5, 6, Rsrc);
5319 add2reg(Rsrc, min_cnt);
5320
5321 z_stmg(Z_R0, Z_R1, 0, Rdst);
5322
5323 add2reg(Rdst, min_cnt*2);
5324 z_brct(Rix, UnrolledLoop);
5325
5326 bind(UnrolledDone);
5327 z_lgfr(Z_R0, Rcnt); // # chars left over after unrolled loop.
5328 z_nilf(Z_R0, min_cnt-1);
5329 z_brnz(ScalarShortcut); // if zero, there is nothing left to do for scalar loop.
5330 // Rix == 0 in all cases.
5331 z_sgfr(Z_R0, Rcnt); // negative # characters the ptrs have been advanced previously.
5332 z_agr(Rdst, Z_R0); // restore ptr, double the element count for Rdst restore.
5333 z_agr(Rdst, Z_R0);
5334 z_agr(Rsrc, Z_R0); // restore ptr.
5335 z_bru(AllDone);
5336 }
5337
5338 {
5339 bind(ScalarShortcut);
5340 // Z_R0 must contain remaining # characters as 64-bit signed int here.
5341 // register contents is preserved over scalar processing (for register fixup).
5342
5343 {
5344 Label ScalarDefault;
5345 z_chi(Rcnt, 2);
5346 z_brh(ScalarDefault);
5347 z_llc(Z_R0, 0, Z_R0, Rsrc); // 6 bytes
5348 z_sth(Z_R0, 0, Z_R0, Rdst); // 4 bytes
5349 z_brl(AllDone);
5350 z_llc(Z_R0, 1, Z_R0, Rsrc); // 6 bytes
5351 z_sth(Z_R0, 2, Z_R0, Rdst); // 4 bytes
5352 z_bru(AllDone);
5353 bind(ScalarDefault);
5354 }
5355
5356 Label CodeTable;
5357 // Some comments on Rix calculation:
5358 // - Rcnt is small, therefore no bits shifted out of low word (sll(g) instructions).
5359 // - high word of both Rix and Rcnt may contain garbage
5360 // - the final lngfr takes care of that garbage, extending the sign to high word
5361 z_sllg(Rix, Z_R0, 2); // calculate 10*Rix = (4*Rix + Rix)*2
5362 z_ar(Rix, Z_R0);
5363 z_larl(Z_R1, CodeTable);
5364 z_sll(Rix, 1);
5365 z_lngfr(Rix, Rix); // ix range: [0..7], after inversion & mult: [-(7*12)..(0*12)].
5366 z_bc(Assembler::bcondAlways, 0, Rix, Z_R1);
5367
5368 z_llc(Z_R1, 6, Z_R0, Rsrc); // 6 bytes
5369 z_sth(Z_R1, 12, Z_R0, Rdst); // 4 bytes
5370
5371 z_llc(Z_R1, 5, Z_R0, Rsrc);
5372 z_sth(Z_R1, 10, Z_R0, Rdst);
5373
5374 z_llc(Z_R1, 4, Z_R0, Rsrc);
5375 z_sth(Z_R1, 8, Z_R0, Rdst);
5376
5377 z_llc(Z_R1, 3, Z_R0, Rsrc);
5378 z_sth(Z_R1, 6, Z_R0, Rdst);
5379
5380 z_llc(Z_R1, 2, Z_R0, Rsrc);
5381 z_sth(Z_R1, 4, Z_R0, Rdst);
5382
5383 z_llc(Z_R1, 1, Z_R0, Rsrc);
5384 z_sth(Z_R1, 2, Z_R0, Rdst);
5385
5386 z_llc(Z_R1, 0, Z_R0, Rsrc);
5387 z_sth(Z_R1, 0, Z_R0, Rdst);
5388 bind(CodeTable);
5389
5390 z_chi(Rcnt, 8); // no fixup for small strings. Rdst, Rsrc were not modified.
5391 z_brl(AllDone);
5392
5393 z_sgfr(Z_R0, Rcnt); // # characters the ptrs have been advanced previously.
5394 z_agr(Rdst, Z_R0); // restore ptr, double the element count for Rdst restore.
5395 z_agr(Rdst, Z_R0);
5396 z_agr(Rsrc, Z_R0); // restore ptr.
5397 }
5398 bind(AllDone);
5399
5400 BLOCK_COMMENT("} string_inflate");
5401 return offset() - block_start;
5402 }
5403
5404 // Inflate byte[] to char[], length known at compile time.
5405 // Restores: src, dst
5406 // Kills: tmp, Z_R0, Z_R1.
5407 // Note:
5408 // len is signed int. Counts # characters, not bytes.
5409 unsigned int MacroAssembler::string_inflate_const(Register src, Register dst, Register tmp, int len) {
5410 assert_different_registers(Z_R0, Z_R1, src, dst, tmp);
5411
5412 BLOCK_COMMENT("string_inflate_const {");
5413 int block_start = offset();
5414
5415 Register Rix = tmp; // loop index
5416 Register Rsrc = src; // addr(src array)
5417 Register Rdst = dst; // addr(dst array)
5418 Label ScalarShortcut, AllDone;
5419 int nprocessed = 0;
5420 int src_off = 0; // compensate for saved (optimized away) ptr advancement.
5421 int dst_off = 0; // compensate for saved (optimized away) ptr advancement.
5422 bool restore_inputs = false;
5423 bool workreg_clear = false;
5424
5425 if (VM_Version::has_Prefetch()) {
5426 prefetch_read(Address(Rsrc)); // make sure first cache line is prepared.
5427 prefetch_update(Address(Rdst));
5428 }
5429
5430 if ((len >= 32) && VM_Version::has_VectorFacility()) {
5431 const int min_vcnt = 32; // Minimum #characters required to use vector instructions.
5432 // Otherwise just do nothing in vector mode.
5433 // Must be multiple of vector register length (16 bytes = 128 bits).
5434 const int log_min_vcnt = exact_log2(min_vcnt);
5435 const int iterations = (len - nprocessed) >> log_min_vcnt;
5436 nprocessed += iterations << log_min_vcnt;
5437 Label VectorLoop;
5438
5439 if (iterations == 1) {
5440 z_vlm(Z_V20, Z_V21, 0+src_off, Rsrc); // get next 32 characters (single-byte)
5441 z_vuplhb(Z_V22, Z_V20); // V2 <- (expand) V0(high)
5442 z_vupllb(Z_V23, Z_V20); // V3 <- (expand) V0(low)
5443 z_vuplhb(Z_V24, Z_V21); // V4 <- (expand) V1(high)
5444 z_vupllb(Z_V25, Z_V21); // V5 <- (expand) V1(low)
5445 z_vstm(Z_V22, Z_V25, 0+dst_off, Rdst); // store next 32 bytes
5446
5447 src_off += min_vcnt;
5448 dst_off += min_vcnt*2;
5449 } else {
5450 restore_inputs = true;
5451
5452 z_lgfi(Rix, len>>log_min_vcnt);
5453 bind(VectorLoop);
5454 z_vlm(Z_V20, Z_V21, 0, Rsrc); // get next 32 characters (single-byte)
5455 add2reg(Rsrc, min_vcnt);
5456
5457 z_vuplhb(Z_V22, Z_V20); // V2 <- (expand) V0(high)
5458 z_vupllb(Z_V23, Z_V20); // V3 <- (expand) V0(low)
5459 z_vuplhb(Z_V24, Z_V21); // V4 <- (expand) V1(high)
5460 z_vupllb(Z_V25, Z_V21); // V5 <- (expand) V1(low)
5461 z_vstm(Z_V22, Z_V25, 0, Rdst); // store next 32 bytes
5462 add2reg(Rdst, min_vcnt*2);
5463
5464 z_brct(Rix, VectorLoop);
5465 }
5466 }
5467
5468 if (((len-nprocessed) >= 16) && VM_Version::has_VectorFacility()) {
5469 const int min_vcnt = 16; // Minimum #characters required to use vector instructions.
5470 // Otherwise just do nothing in vector mode.
5471 // Must be multiple of vector register length (16 bytes = 128 bits).
5472 const int log_min_vcnt = exact_log2(min_vcnt);
5473 const int iterations = (len - nprocessed) >> log_min_vcnt;
5474 nprocessed += iterations << log_min_vcnt;
5475 assert(iterations == 1, "must be!");
5476
5477 z_vl(Z_V20, 0+src_off, Z_R0, Rsrc); // get next 16 characters (single-byte)
5478 z_vuplhb(Z_V22, Z_V20); // V2 <- (expand) V0(high)
5479 z_vupllb(Z_V23, Z_V20); // V3 <- (expand) V0(low)
5480 z_vstm(Z_V22, Z_V23, 0+dst_off, Rdst); // store next 32 bytes
5481
5482 src_off += min_vcnt;
5483 dst_off += min_vcnt*2;
5484 }
5485
5486 if ((len-nprocessed) > 8) {
5487 const int min_cnt = 8; // Minimum #characters required to use unrolled scalar loop.
5488 // Otherwise just do nothing in unrolled scalar mode.
5489 // Must be multiple of 8.
5490 const int log_min_cnt = exact_log2(min_cnt);
5491 const int iterations = (len - nprocessed) >> log_min_cnt;
5492 nprocessed += iterations << log_min_cnt;
5493
5494 //---< avoid loop overhead/ptr increment for small # iterations >---
5495 if (iterations <= 2) {
5496 clear_reg(Z_R0);
5497 clear_reg(Z_R1);
5498 workreg_clear = true;
5499
5500 z_icmh(Z_R0, 5, 0+src_off, Rsrc);
5501 z_icmh(Z_R1, 5, 4+src_off, Rsrc);
5502 z_icm(Z_R0, 5, 2+src_off, Rsrc);
5503 z_icm(Z_R1, 5, 6+src_off, Rsrc);
5504 z_stmg(Z_R0, Z_R1, 0+dst_off, Rdst);
5505
5506 src_off += min_cnt;
5507 dst_off += min_cnt*2;
5508 }
5509
5510 if (iterations == 2) {
5511 z_icmh(Z_R0, 5, 0+src_off, Rsrc);
5512 z_icmh(Z_R1, 5, 4+src_off, Rsrc);
5513 z_icm(Z_R0, 5, 2+src_off, Rsrc);
5514 z_icm(Z_R1, 5, 6+src_off, Rsrc);
5515 z_stmg(Z_R0, Z_R1, 0+dst_off, Rdst);
5516
5517 src_off += min_cnt;
5518 dst_off += min_cnt*2;
5519 }
5520
5521 if (iterations > 2) {
5522 Label UnrolledLoop;
5523 restore_inputs = true;
5524
5525 clear_reg(Z_R0);
5526 clear_reg(Z_R1);
5527 workreg_clear = true;
5528
5529 z_lgfi(Rix, iterations);
5530 bind(UnrolledLoop);
5531 z_icmh(Z_R0, 5, 0, Rsrc);
5532 z_icmh(Z_R1, 5, 4, Rsrc);
5533 z_icm(Z_R0, 5, 2, Rsrc);
5534 z_icm(Z_R1, 5, 6, Rsrc);
5535 add2reg(Rsrc, min_cnt);
5536
5537 z_stmg(Z_R0, Z_R1, 0, Rdst);
5538 add2reg(Rdst, min_cnt*2);
5539
5540 z_brct(Rix, UnrolledLoop);
5541 }
5542 }
5543
5544 if ((len-nprocessed) > 0) {
5545 switch (len-nprocessed) {
5546 case 8:
5547 if (!workreg_clear) {
5548 clear_reg(Z_R0);
5549 clear_reg(Z_R1);
5550 }
5551 z_icmh(Z_R0, 5, 0+src_off, Rsrc);
5552 z_icmh(Z_R1, 5, 4+src_off, Rsrc);
5553 z_icm(Z_R0, 5, 2+src_off, Rsrc);
5554 z_icm(Z_R1, 5, 6+src_off, Rsrc);
5555 z_stmg(Z_R0, Z_R1, 0+dst_off, Rdst);
5556 break;
5557 case 7:
5558 if (!workreg_clear) {
5559 clear_reg(Z_R0);
5560 clear_reg(Z_R1);
5561 }
5562 clear_reg(Rix);
5563 z_icm(Z_R0, 5, 0+src_off, Rsrc);
5564 z_icm(Z_R1, 5, 2+src_off, Rsrc);
5565 z_icm(Rix, 5, 4+src_off, Rsrc);
5566 z_stm(Z_R0, Z_R1, 0+dst_off, Rdst);
5567 z_llc(Z_R0, 6+src_off, Z_R0, Rsrc);
5568 z_st(Rix, 8+dst_off, Z_R0, Rdst);
5569 z_sth(Z_R0, 12+dst_off, Z_R0, Rdst);
5570 break;
5571 case 6:
5572 if (!workreg_clear) {
5573 clear_reg(Z_R0);
5574 clear_reg(Z_R1);
5575 }
5576 clear_reg(Rix);
5577 z_icm(Z_R0, 5, 0+src_off, Rsrc);
5578 z_icm(Z_R1, 5, 2+src_off, Rsrc);
5579 z_icm(Rix, 5, 4+src_off, Rsrc);
5580 z_stm(Z_R0, Z_R1, 0+dst_off, Rdst);
5581 z_st(Rix, 8+dst_off, Z_R0, Rdst);
5582 break;
5583 case 5:
5584 if (!workreg_clear) {
5585 clear_reg(Z_R0);
5586 clear_reg(Z_R1);
5587 }
5588 z_icm(Z_R0, 5, 0+src_off, Rsrc);
5589 z_icm(Z_R1, 5, 2+src_off, Rsrc);
5590 z_llc(Rix, 4+src_off, Z_R0, Rsrc);
5591 z_stm(Z_R0, Z_R1, 0+dst_off, Rdst);
5592 z_sth(Rix, 8+dst_off, Z_R0, Rdst);
5593 break;
5594 case 4:
5595 if (!workreg_clear) {
5596 clear_reg(Z_R0);
5597 clear_reg(Z_R1);
5598 }
5599 z_icm(Z_R0, 5, 0+src_off, Rsrc);
5600 z_icm(Z_R1, 5, 2+src_off, Rsrc);
5601 z_stm(Z_R0, Z_R1, 0+dst_off, Rdst);
5602 break;
5603 case 3:
5604 if (!workreg_clear) {
5605 clear_reg(Z_R0);
5606 }
5607 z_llc(Z_R1, 2+src_off, Z_R0, Rsrc);
5608 z_icm(Z_R0, 5, 0+src_off, Rsrc);
5609 z_sth(Z_R1, 4+dst_off, Z_R0, Rdst);
5610 z_st(Z_R0, 0+dst_off, Rdst);
5611 break;
5612 case 2:
5613 z_llc(Z_R0, 0+src_off, Z_R0, Rsrc);
5614 z_llc(Z_R1, 1+src_off, Z_R0, Rsrc);
5615 z_sth(Z_R0, 0+dst_off, Z_R0, Rdst);
5616 z_sth(Z_R1, 2+dst_off, Z_R0, Rdst);
5617 break;
5618 case 1:
5619 z_llc(Z_R0, 0+src_off, Z_R0, Rsrc);
5620 z_sth(Z_R0, 0+dst_off, Z_R0, Rdst);
5621 break;
5622 default:
5623 guarantee(false, "Impossible");
5624 break;
5625 }
5626 src_off += len-nprocessed;
5627 dst_off += (len-nprocessed)*2;
5628 nprocessed = len;
5629 }
5630
5631 //---< restore modified input registers >---
5632 if ((nprocessed > 0) && restore_inputs) {
5633 z_agfi(Rsrc, -(nprocessed-src_off));
5634 if (nprocessed < 1000000000) { // avoid int overflow
5635 z_agfi(Rdst, -(nprocessed*2-dst_off));
5636 } else {
5637 z_agfi(Rdst, -(nprocessed-dst_off));
5638 z_agfi(Rdst, -nprocessed);
5639 }
5640 }
5641
5642 BLOCK_COMMENT("} string_inflate_const");
5643 return offset() - block_start;
5644 }
5645
5646 // Kills src.
5647 unsigned int MacroAssembler::has_negatives(Register result, Register src, Register cnt,
5648 Register odd_reg, Register even_reg, Register tmp) {
5649 int block_start = offset();
5650 Label Lloop1, Lloop2, Lslow, Lnotfound, Ldone;
5651 const Register addr = src, mask = tmp;
5652
5653 BLOCK_COMMENT("has_negatives {");
5654
5655 z_llgfr(Z_R1, cnt); // Number of bytes to read. (Must be a positive simm32.)
5656 z_llilf(mask, 0x80808080);
5657 z_lhi(result, 1); // Assume true.
5658 // Last possible addr for fast loop.
5659 z_lay(odd_reg, -16, Z_R1, src);
5660 z_chi(cnt, 16);
5661 z_brl(Lslow);
5662
|