919 // Simply use distance from start of const section (should be patched in the end).
920 long disp = toc_distance();
921
922 RelocationHolder rspec = internal_word_Relocation::spec(pc() + disp);
923 relocate(rspec);
924 z_larl(Rtoc, RelAddr::pcrel_off32(disp)); // Offset is in halfwords.
925 }
926
927 // PCrelative TOC access.
928 // Load from anywhere pcrelative (with relocation of load instr)
929 void MacroAssembler::load_long_pcrelative(Register Rdst, address dataLocation) {
930 address pc = this->pc();
931 ptrdiff_t total_distance = dataLocation - pc;
932 RelocationHolder rspec = internal_word_Relocation::spec(dataLocation);
933
934 assert((total_distance & 0x01L) == 0, "halfword alignment is mandatory");
935 assert(total_distance != 0, "sanity");
936
937 // Some extra safety net.
938 if (!RelAddr::is_in_range_of_RelAddr32(total_distance)) {
939 guarantee(RelAddr::is_in_range_of_RelAddr32(total_distance), "too far away");
940 }
941
942 (this)->relocate(rspec, relocInfo::pcrel_addr_format);
943 z_lgrl(Rdst, RelAddr::pcrel_off32(total_distance));
944 }
945
946
947 // PCrelative TOC access.
948 // Load from anywhere pcrelative (with relocation of load instr)
949 // loaded addr has to be relocated when added to constant pool.
950 void MacroAssembler::load_addr_pcrelative(Register Rdst, address addrLocation) {
951 address pc = this->pc();
952 ptrdiff_t total_distance = addrLocation - pc;
953 RelocationHolder rspec = internal_word_Relocation::spec(addrLocation);
954
955 assert((total_distance & 0x01L) == 0, "halfword alignment is mandatory");
956
957 // Some extra safety net.
958 if (!RelAddr::is_in_range_of_RelAddr32(total_distance)) {
959 guarantee(RelAddr::is_in_range_of_RelAddr32(total_distance), "too far away");
960 }
961
962 (this)->relocate(rspec, relocInfo::pcrel_addr_format);
963 z_lgrl(Rdst, RelAddr::pcrel_off32(total_distance));
964 }
965
966 // Generic operation: load a value from memory and test.
967 // CondCode indicates the sign (<0, ==0, >0) of the loaded value.
968 void MacroAssembler::load_and_test_byte(Register dst, const Address &a) {
969 z_lb(dst, a);
970 z_ltr(dst, dst);
971 }
972
973 void MacroAssembler::load_and_test_short(Register dst, const Address &a) {
974 int64_t disp = a.disp20();
975 if (Displacement::is_shortDisp(disp)) {
976 z_lh(dst, a);
977 } else if (Displacement::is_longDisp(disp)) {
978 z_lhy(dst, a);
979 } else {
1008 } else {
1009 ShouldNotReachHere();
1010 }
1011 }
1012
1013 // Test a bit in a register. Result is reflected in CC.
1014 void MacroAssembler::testbit(Register r, unsigned int bitPos) {
1015 if (bitPos < 16) {
1016 z_tmll(r, 1U<<bitPos);
1017 } else if (bitPos < 32) {
1018 z_tmlh(r, 1U<<(bitPos-16));
1019 } else if (bitPos < 48) {
1020 z_tmhl(r, 1U<<(bitPos-32));
1021 } else if (bitPos < 64) {
1022 z_tmhh(r, 1U<<(bitPos-48));
1023 } else {
1024 ShouldNotReachHere();
1025 }
1026 }
1027
1028 // Clear a register, i.e. load const zero into reg.
1029 // Return len (in bytes) of generated instruction(s).
1030 // whole_reg: Clear 64 bits if true, 32 bits otherwise.
1031 // set_cc: Use instruction that sets the condition code, if true.
1032 int MacroAssembler::clear_reg(Register r, bool whole_reg, bool set_cc) {
1033 unsigned int start_off = offset();
1034 if (whole_reg) {
1035 set_cc ? z_xgr(r, r) : z_laz(r, 0, Z_R0);
1036 } else { // Only 32bit register.
1037 set_cc ? z_xr(r, r) : z_lhi(r, 0);
1038 }
1039 return offset() - start_off;
1040 }
1041
1042 #ifdef ASSERT
1043 int MacroAssembler::preset_reg(Register r, unsigned long pattern, int pattern_len) {
1044 switch (pattern_len) {
1045 case 1:
1046 pattern = (pattern & 0x000000ff) | ((pattern & 0x000000ff)<<8);
1047 case 2:
4879 if (VM_Version::has_ExecuteExtensions()) {
4880 z_exrl(Z_R1, MVC_template);
4881 } else {
4882 z_ex(tmp1_reg, 0, Z_R0, Z_R1);
4883 }
4884
4885 bind(done);
4886
4887 BLOCK_COMMENT("} CopyRawMemory_AlignedDisjoint");
4888
4889 int block_end = offset();
4890 return block_end - block_start;
4891 }
4892
4893 //------------------------------------------------------
4894 // Special String Intrinsics. Implementation
4895 //------------------------------------------------------
4896
4897 // Intrinsics for CompactStrings
4898
4899 // Compress char[] to byte[]. odd_reg contains cnt. Kills dst. Early clobber: result
4900 // The result is the number of characters copied before the first incompatible character was found.
4901 // If tmp2 is provided and the compression fails, the compression stops exactly at this point and the result is precise.
4902 //
4903 // Note: Does not behave exactly like package private StringUTF16 compress java implementation in case of failure:
4904 // - Different number of characters may have been written to dead array (if tmp2 not provided).
4905 // - Returns a number <cnt instead of 0. (Result gets compared with cnt.)
4906 unsigned int MacroAssembler::string_compress(Register result, Register src, Register dst, Register odd_reg,
4907 Register even_reg, Register tmp, Register tmp2) {
4908 int block_start = offset();
4909 Label Lloop1, Lloop2, Lslow, Ldone;
4910 const Register addr2 = dst, ind1 = result, mask = tmp;
4911 const bool precise = (tmp2 != noreg);
4912
4913 BLOCK_COMMENT("string_compress {");
4914
4915 z_sll(odd_reg, 1); // Number of bytes to read. (Must be a positive simm32.)
4916 clear_reg(ind1); // Index to read.
4917 z_llilf(mask, 0xFF00FF00);
4918 z_ahi(odd_reg, -16); // Last possible index for fast loop.
4919 z_brl(Lslow);
4920
4921 // ind1: index, even_reg: index increment, odd_reg: index limit
4922 z_iihf(mask, 0xFF00FF00);
4923 z_lhi(even_reg, 16);
4924
4925 bind(Lloop1); // 8 Characters per iteration.
4926 z_lg(Z_R0, Address(src, ind1));
4927 z_lg(Z_R1, Address(src, ind1, 8));
4928 if (precise) {
4929 if (VM_Version::has_DistinctOpnds()) {
4930 z_ogrk(tmp2, Z_R0, Z_R1);
4931 } else {
4932 z_lgr(tmp2, Z_R0);
4933 z_ogr(tmp2, Z_R1);
4934 }
4935 z_ngr(tmp2, mask);
4936 z_brne(Lslow); // Failed fast case, retry slowly.
4937 }
4938 z_stcmh(Z_R0, 5, 0, addr2);
4939 z_stcm(Z_R0, 5, 2, addr2);
4940 if (!precise) { z_ogr(Z_R0, Z_R1); }
4941 z_stcmh(Z_R1, 5, 4, addr2);
4942 z_stcm(Z_R1, 5, 6, addr2);
4943 if (!precise) {
4944 z_ngr(Z_R0, mask);
4945 z_brne(Ldone); // Failed (more than needed was written).
4946 }
4947 z_aghi(addr2, 8);
4948 z_brxle(ind1, even_reg, Lloop1);
4949
4950 bind(Lslow);
4951 // Compute index limit and skip if negative.
4952 z_ahi(odd_reg, 16-2); // Last possible index for slow loop.
4953 z_lhi(even_reg, 2);
4954 z_cr(ind1, odd_reg);
4955 z_brh(Ldone);
4956
4957 bind(Lloop2); // 1 Character per iteration.
4958 z_llh(Z_R0, Address(src, ind1));
4959 z_tmll(Z_R0, 0xFF00);
4960 z_brnaz(Ldone); // Failed slow case: Return number of written characters.
4961 z_stc(Z_R0, Address(addr2));
4962 z_aghi(addr2, 1);
4963 z_brxle(ind1, even_reg, Lloop2);
4964
4965 bind(Ldone); // result = ind1 = 2*cnt
4966 z_srl(ind1, 1);
4967
4968 BLOCK_COMMENT("} string_compress");
4969
4970 return offset() - block_start;
4971 }
4972
4973 // Inflate byte[] to char[].
4974 unsigned int MacroAssembler::string_inflate_trot(Register src, Register dst, Register cnt, Register tmp) {
4975 int block_start = offset();
4976
4977 BLOCK_COMMENT("string_inflate {");
4978
4979 Register stop_char = Z_R0;
4980 Register table = Z_R1;
4981 Register src_addr = tmp;
4982
4983 assert_different_registers(Z_R0, Z_R1, tmp, src, dst, cnt);
4984 assert(dst->encoding()%2 == 0, "must be even reg");
4985 assert(cnt->encoding()%2 == 1, "must be odd reg");
4986 assert(cnt->encoding() - dst->encoding() == 1, "must be even/odd pair");
4987
4988 StubRoutines::zarch::generate_load_trot_table_addr(this, table); // kills Z_R0 (if ASSERT)
4989 clear_reg(stop_char); // Stop character. Not used here, but initialized to have a defined value.
4990 lgr_if_needed(src_addr, src);
4991 z_llgfr(cnt, cnt); // # src characters, must be a positive simm32.
4992
4993 translate_ot(dst, src_addr, /* mask = */ 0x0001);
4994
4995 BLOCK_COMMENT("} string_inflate");
4996
4997 return offset() - block_start;
4998 }
4999
5000 // Inflate byte[] to char[]. odd_reg contains cnt. Kills src.
5001 unsigned int MacroAssembler::string_inflate(Register src, Register dst, Register odd_reg,
5002 Register even_reg, Register tmp) {
5003 int block_start = offset();
5004
5005 BLOCK_COMMENT("string_inflate {");
5006
5007 Label Lloop1, Lloop2, Lslow, Ldone;
5008 const Register addr1 = src, ind2 = tmp;
5009
5010 z_sll(odd_reg, 1); // Number of bytes to write. (Must be a positive simm32.)
5011 clear_reg(ind2); // Index to write.
5012 z_ahi(odd_reg, -16); // Last possible index for fast loop.
5013 z_brl(Lslow);
5014
5015 // ind2: index, even_reg: index increment, odd_reg: index limit
5016 clear_reg(Z_R0);
5017 clear_reg(Z_R1);
5018 z_lhi(even_reg, 16);
5019
5020 bind(Lloop1); // 8 Characters per iteration.
5021 z_icmh(Z_R0, 5, 0, addr1);
5022 z_icmh(Z_R1, 5, 4, addr1);
5023 z_icm(Z_R0, 5, 2, addr1);
5024 z_icm(Z_R1, 5, 6, addr1);
5025 z_aghi(addr1, 8);
5026 z_stg(Z_R0, Address(dst, ind2));
5027 z_stg(Z_R1, Address(dst, ind2, 8));
5028 z_brxle(ind2, even_reg, Lloop1);
5029
5030 bind(Lslow);
5031 // Compute index limit and skip if negative.
5032 z_ahi(odd_reg, 16-2); // Last possible index for slow loop.
5033 z_lhi(even_reg, 2);
5034 z_cr(ind2, odd_reg);
5035 z_brh(Ldone);
5036
5037 bind(Lloop2); // 1 Character per iteration.
5038 z_llc(Z_R0, Address(addr1));
5039 z_sth(Z_R0, Address(dst, ind2));
5040 z_aghi(addr1, 1);
5041 z_brxle(ind2, even_reg, Lloop2);
5042
5043 bind(Ldone);
5044
5045 BLOCK_COMMENT("} string_inflate");
5046
5047 return offset() - block_start;
5048 }
5049
5050 // Kills src.
5051 unsigned int MacroAssembler::has_negatives(Register result, Register src, Register cnt,
5052 Register odd_reg, Register even_reg, Register tmp) {
5053 int block_start = offset();
5054 Label Lloop1, Lloop2, Lslow, Lnotfound, Ldone;
5055 const Register addr = src, mask = tmp;
5056
5057 BLOCK_COMMENT("has_negatives {");
5058
5059 z_llgfr(Z_R1, cnt); // Number of bytes to read. (Must be a positive simm32.)
5060 z_llilf(mask, 0x80808080);
5061 z_lhi(result, 1); // Assume true.
5062 // Last possible addr for fast loop.
5063 z_lay(odd_reg, -16, Z_R1, src);
5064 z_chi(cnt, 16);
5065 z_brl(Lslow);
5066
|
919 // Simply use distance from start of const section (should be patched in the end).
920 long disp = toc_distance();
921
922 RelocationHolder rspec = internal_word_Relocation::spec(pc() + disp);
923 relocate(rspec);
924 z_larl(Rtoc, RelAddr::pcrel_off32(disp)); // Offset is in halfwords.
925 }
926
927 // PCrelative TOC access.
928 // Load from anywhere pcrelative (with relocation of load instr)
929 void MacroAssembler::load_long_pcrelative(Register Rdst, address dataLocation) {
930 address pc = this->pc();
931 ptrdiff_t total_distance = dataLocation - pc;
932 RelocationHolder rspec = internal_word_Relocation::spec(dataLocation);
933
934 assert((total_distance & 0x01L) == 0, "halfword alignment is mandatory");
935 assert(total_distance != 0, "sanity");
936
937 // Some extra safety net.
938 if (!RelAddr::is_in_range_of_RelAddr32(total_distance)) {
939 guarantee(RelAddr::is_in_range_of_RelAddr32(total_distance), "load_long_pcrelative can't handle distance " INTPTR_FORMAT, total_distance);
940 }
941
942 (this)->relocate(rspec, relocInfo::pcrel_addr_format);
943 z_lgrl(Rdst, RelAddr::pcrel_off32(total_distance));
944 }
945
946
947 // PCrelative TOC access.
948 // Load from anywhere pcrelative (with relocation of load instr)
949 // loaded addr has to be relocated when added to constant pool.
950 void MacroAssembler::load_addr_pcrelative(Register Rdst, address addrLocation) {
951 address pc = this->pc();
952 ptrdiff_t total_distance = addrLocation - pc;
953 RelocationHolder rspec = internal_word_Relocation::spec(addrLocation);
954
955 assert((total_distance & 0x01L) == 0, "halfword alignment is mandatory");
956
957 // Some extra safety net.
958 if (!RelAddr::is_in_range_of_RelAddr32(total_distance)) {
959 guarantee(RelAddr::is_in_range_of_RelAddr32(total_distance), "load_long_pcrelative can't handle distance " INTPTR_FORMAT, total_distance);
960 }
961
962 (this)->relocate(rspec, relocInfo::pcrel_addr_format);
963 z_lgrl(Rdst, RelAddr::pcrel_off32(total_distance));
964 }
965
966 // Generic operation: load a value from memory and test.
967 // CondCode indicates the sign (<0, ==0, >0) of the loaded value.
968 void MacroAssembler::load_and_test_byte(Register dst, const Address &a) {
969 z_lb(dst, a);
970 z_ltr(dst, dst);
971 }
972
973 void MacroAssembler::load_and_test_short(Register dst, const Address &a) {
974 int64_t disp = a.disp20();
975 if (Displacement::is_shortDisp(disp)) {
976 z_lh(dst, a);
977 } else if (Displacement::is_longDisp(disp)) {
978 z_lhy(dst, a);
979 } else {
1008 } else {
1009 ShouldNotReachHere();
1010 }
1011 }
1012
1013 // Test a bit in a register. Result is reflected in CC.
1014 void MacroAssembler::testbit(Register r, unsigned int bitPos) {
1015 if (bitPos < 16) {
1016 z_tmll(r, 1U<<bitPos);
1017 } else if (bitPos < 32) {
1018 z_tmlh(r, 1U<<(bitPos-16));
1019 } else if (bitPos < 48) {
1020 z_tmhl(r, 1U<<(bitPos-32));
1021 } else if (bitPos < 64) {
1022 z_tmhh(r, 1U<<(bitPos-48));
1023 } else {
1024 ShouldNotReachHere();
1025 }
1026 }
1027
1028 void MacroAssembler::prefetch_read(Address a) {
1029 z_pfd(1, a.disp20(), a.indexOrR0(), a.base());
1030 }
1031 void MacroAssembler::prefetch_update(Address a) {
1032 z_pfd(2, a.disp20(), a.indexOrR0(), a.base());
1033 }
1034
1035 // Clear a register, i.e. load const zero into reg.
1036 // Return len (in bytes) of generated instruction(s).
1037 // whole_reg: Clear 64 bits if true, 32 bits otherwise.
1038 // set_cc: Use instruction that sets the condition code, if true.
1039 int MacroAssembler::clear_reg(Register r, bool whole_reg, bool set_cc) {
1040 unsigned int start_off = offset();
1041 if (whole_reg) {
1042 set_cc ? z_xgr(r, r) : z_laz(r, 0, Z_R0);
1043 } else { // Only 32bit register.
1044 set_cc ? z_xr(r, r) : z_lhi(r, 0);
1045 }
1046 return offset() - start_off;
1047 }
1048
1049 #ifdef ASSERT
1050 int MacroAssembler::preset_reg(Register r, unsigned long pattern, int pattern_len) {
1051 switch (pattern_len) {
1052 case 1:
1053 pattern = (pattern & 0x000000ff) | ((pattern & 0x000000ff)<<8);
1054 case 2:
4886 if (VM_Version::has_ExecuteExtensions()) {
4887 z_exrl(Z_R1, MVC_template);
4888 } else {
4889 z_ex(tmp1_reg, 0, Z_R0, Z_R1);
4890 }
4891
4892 bind(done);
4893
4894 BLOCK_COMMENT("} CopyRawMemory_AlignedDisjoint");
4895
4896 int block_end = offset();
4897 return block_end - block_start;
4898 }
4899
4900 //------------------------------------------------------
4901 // Special String Intrinsics. Implementation
4902 //------------------------------------------------------
4903
4904 // Intrinsics for CompactStrings
4905
4906 // Compress char[] to byte[].
4907 // Restores: src, dst
4908 // Uses: cnt
4909 // Kills: tmp, Z_R0, Z_R1.
4910 // Early clobber: result.
4911 // Note:
4912 // cnt is signed int. Do not rely on high word!
4913 // counts # characters, not bytes.
4914 // The result is the number of characters copied before the first incompatible character was found.
4915 // If precise is true, the processing stops exactly at this point. Otherwise, the result may be off
4916 // by a few bytes. The result always indicates the number of copied characters.
4917 //
4918 // Note: Does not behave exactly like package private StringUTF16 compress java implementation in case of failure:
4919 // - Different number of characters may have been written to dead array (if precise is false).
4920 // - Returns a number <cnt instead of 0. (Result gets compared with cnt.)
4921 unsigned int MacroAssembler::string_compress(Register result, Register src, Register dst, Register cnt,
4922 Register tmp, bool precise) {
4923 assert_different_registers(Z_R0, Z_R1, src, dst, cnt, tmp);
4924
4925 if (precise) {
4926 BLOCK_COMMENT("encode_iso_array {");
4927 } else {
4928 BLOCK_COMMENT("string_compress {");
4929 }
4930 int block_start = offset();
4931
4932 Register Rsrc = src;
4933 Register Rdst = dst;
4934 Register Rix = tmp;
4935 Register Rcnt = cnt;
4936 Register Rmask = result; // holds incompatibility check mask until result value is stored.
4937 Label ScalarShortcut, AllDone;
4938
4939 z_iilf(Rmask, 0xFF00FF00);
4940 z_iihf(Rmask, 0xFF00FF00);
4941
4942 {
4943 //---< shortcuts for short strings (very frequent) >---
4944 // Strings with 4 and 8 characters were fond to occur very frequently.
4945 // Therefore, we handle them right away with minimal overhead.
4946 Label skipShortcut, skip4Shortcut, skip8Shortcut;
4947 Register Rout = Z_R0;
4948 z_chi(Rcnt, 4);
4949 z_brne(skip4Shortcut); // 4 characters are very frequent
4950 z_lg(Z_R0, 0, Rsrc); // Treat exactly 4 characters specially.
4951 if (VM_Version::has_DistinctOpnds()) {
4952 Rout = Z_R0;
4953 z_ngrk(Rix, Z_R0, Rmask);
4954 } else {
4955 Rout = Rix;
4956 z_lgr(Rix, Z_R0);
4957 z_ngr(Z_R0, Rmask);
4958 }
4959 z_brnz(skipShortcut);
4960 z_stcmh(Rout, 5, 0, Rdst);
4961 z_stcm(Rout, 5, 2, Rdst);
4962 z_lgfr(result, Rcnt);
4963 z_bru(AllDone);
4964 bind(skip4Shortcut);
4965
4966 z_chi(Rcnt, 8);
4967 z_brne(skip8Shortcut); // There's more to do...
4968 z_lmg(Z_R0, Z_R1, 0, Rsrc); // Treat exactly 8 characters specially.
4969 if (VM_Version::has_DistinctOpnds()) {
4970 Rout = Z_R0;
4971 z_ogrk(Rix, Z_R0, Z_R1);
4972 z_ngr(Rix, Rmask);
4973 } else {
4974 Rout = Rix;
4975 z_lgr(Rix, Z_R0);
4976 z_ogr(Z_R0, Z_R1);
4977 z_ngr(Z_R0, Rmask);
4978 }
4979 z_brnz(skipShortcut);
4980 z_stcmh(Rout, 5, 0, Rdst);
4981 z_stcm(Rout, 5, 2, Rdst);
4982 z_stcmh(Z_R1, 5, 4, Rdst);
4983 z_stcm(Z_R1, 5, 6, Rdst);
4984 z_lgfr(result, Rcnt);
4985 z_bru(AllDone);
4986
4987 bind(skip8Shortcut);
4988 clear_reg(Z_R0, true, false); // #characters already processed (none). Precond for scalar loop.
4989 z_brl(ScalarShortcut); // Just a few characters
4990
4991 bind(skipShortcut);
4992 }
4993 clear_reg(Z_R0); // make sure register is properly initialized.
4994
4995 if (VM_Version::has_VectorFacility()) {
4996 const int min_vcnt = 32; // Minimum #characters required to use vector instructions.
4997 // Otherwise just do nothing in vector mode.
4998 // Must be multiple of 2*(vector register length in chars (8 HW = 128 bits)).
4999 const int log_min_vcnt = exact_log2(min_vcnt);
5000 Label VectorLoop, VectorDone, VectorBreak;
5001
5002 VectorRegister Vtmp1 = Z_V16;
5003 VectorRegister Vtmp2 = Z_V17;
5004 VectorRegister Vmask = Z_V18;
5005 VectorRegister Vzero = Z_V19;
5006 VectorRegister Vsrc_first = Z_V20;
5007 VectorRegister Vsrc_last = Z_V23;
5008
5009 assert((Vsrc_last->encoding() - Vsrc_first->encoding() + 1) == min_vcnt/8, "logic error");
5010 assert(VM_Version::has_DistinctOpnds(), "Assumption when has_VectorFacility()");
5011 z_srak(Rix, Rcnt, log_min_vcnt); // # vector loop iterations
5012 z_brz(VectorDone); // not enough data for vector loop
5013
5014 z_vzero(Vzero); // all zeroes
5015 z_vgmh(Vmask, 0, 7); // generate 0xff00 mask for all 2-byte elements
5016 z_sllg(Z_R0, Rix, log_min_vcnt); // remember #chars that will be processed by vector loop
5017
5018 bind(VectorLoop);
5019 z_vlm(Vsrc_first, Vsrc_last, 0, Rsrc);
5020 add2reg(Rsrc, min_vcnt*2);
5021
5022 //---< check for incompatible character >---
5023 z_vo(Vtmp1, Z_V20, Z_V21);
5024 z_vo(Vtmp2, Z_V22, Z_V23);
5025 z_vo(Vtmp1, Vtmp1, Vtmp2);
5026 z_vn(Vtmp1, Vtmp1, Vmask);
5027 z_vceqhs(Vtmp1, Vtmp1, Vzero); // high half of all chars must be zero for successful compress.
5028 z_brne(VectorBreak); // break vector loop, incompatible character found.
5029 // re-process data from current iteration in break handler.
5030
5031 //---< pack & store characters >---
5032 z_vpkh(Vtmp1, Z_V20, Z_V21); // pack (src1, src2) -> tmp1
5033 z_vpkh(Vtmp2, Z_V22, Z_V23); // pack (src3, src4) -> tmp2
5034 z_vstm(Vtmp1, Vtmp2, 0, Rdst); // store packed string
5035 add2reg(Rdst, min_vcnt);
5036
5037 z_brct(Rix, VectorLoop);
5038
5039 z_bru(VectorDone);
5040
5041 bind(VectorBreak);
5042 z_sll(Rix, log_min_vcnt); // # chars processed so far in VectorLoop, excl. current iteration.
5043 z_sr(Z_R0, Rix); // correct # chars processed in total.
5044
5045 bind(VectorDone);
5046 }
5047
5048 {
5049 const int min_cnt = 8; // Minimum #characters required to use unrolled loop.
5050 // Otherwise just do nothing in unrolled loop.
5051 // Must be multiple of 8.
5052 const int log_min_cnt = exact_log2(min_cnt);
5053 Label UnrolledLoop, UnrolledDone, UnrolledBreak;
5054
5055 if (VM_Version::has_DistinctOpnds()) {
5056 z_srk(Rix, Rcnt, Z_R0); // remaining # chars to compress in unrolled loop
5057 } else {
5058 z_lr(Rix, Rcnt);
5059 z_sr(Rix, Z_R0);
5060 }
5061 z_sra(Rix, log_min_cnt); // unrolled loop count
5062 z_brz(UnrolledDone);
5063
5064 bind(UnrolledLoop);
5065 z_lmg(Z_R0, Z_R1, 0, Rsrc);
5066 if (precise) {
5067 z_ogr(Z_R1, Z_R0); // check all 8 chars for incompatibility
5068 z_ngr(Z_R1, Rmask);
5069 z_brnz(UnrolledBreak);
5070
5071 z_lg(Z_R1, 8, Rsrc); // reload destroyed register
5072 z_stcmh(Z_R0, 5, 0, Rdst);
5073 z_stcm(Z_R0, 5, 2, Rdst);
5074 } else {
5075 z_stcmh(Z_R0, 5, 0, Rdst);
5076 z_stcm(Z_R0, 5, 2, Rdst);
5077
5078 z_ogr(Z_R0, Z_R1);
5079 z_ngr(Z_R0, Rmask);
5080 z_brnz(UnrolledBreak);
5081 }
5082 z_stcmh(Z_R1, 5, 4, Rdst);
5083 z_stcm(Z_R1, 5, 6, Rdst);
5084
5085 add2reg(Rsrc, min_cnt*2);
5086 add2reg(Rdst, min_cnt);
5087 z_brct(Rix, UnrolledLoop);
5088
5089 z_lgfr(Z_R0, Rcnt); // # chars processed in total after unrolled loop.
5090 z_nilf(Z_R0, ~(min_cnt-1));
5091 z_tmll(Rcnt, min_cnt-1);
5092 z_brnaz(ScalarShortcut); // if all bits zero, there is nothing left to do for scalar loop.
5093 // Rix == 0 in all cases.
5094 z_lgfr(result, Rcnt); // all characters processed.
5095 z_sgfr(Rdst, Rcnt); // restore ptr
5096 z_sgfr(Rsrc, Rcnt); // restore ptr, double the element count for Rsrc restore
5097 z_sgfr(Rsrc, Rcnt);
5098 z_bru(AllDone);
5099
5100 bind(UnrolledBreak);
5101 z_lgfr(Z_R0, Rcnt); // # chars processed in total after unrolled loop
5102 z_nilf(Z_R0, ~(min_cnt-1));
5103 z_sll(Rix, log_min_cnt); // # chars processed so far in UnrolledLoop, excl. current iteration.
5104 z_sr(Z_R0, Rix); // correct # chars processed in total.
5105 if (!precise) {
5106 z_lgfr(result, Z_R0);
5107 z_aghi(result, min_cnt/2); // min_cnt/2 characters have already been written
5108 // but ptrs were not updated yet.
5109 z_sgfr(Rdst, Z_R0); // restore ptr
5110 z_sgfr(Rsrc, Z_R0); // restore ptr, double the element count for Rsrc restore
5111 z_sgfr(Rsrc, Z_R0);
5112 z_bru(AllDone);
5113 }
5114 bind(UnrolledDone);
5115 }
5116
5117 {
5118 Label ScalarLoop, ScalarDone, ScalarBreak;
5119
5120 bind(ScalarShortcut);
5121 z_ltgfr(result, Rcnt);
5122 z_brz(AllDone);
5123
5124 {
5125 //---< Special treatment for very short strings (one or two characters) >---
5126 // For these strings, we are sure that the above code was skipped.
5127 // Thus, no registers were modified, register restore is not required.
5128 Label ScalarDoit, Scalar2Char;
5129 z_chi(Rcnt, 2);
5130 z_brh(ScalarDoit);
5131 z_llh(Z_R1, 0, Z_R0, Rsrc);
5132 z_bre(Scalar2Char);
5133 z_tmll(Z_R1, 0xff00);
5134 z_lghi(result, 0); // cnt == 1, first char invalid, no chars successfully processed
5135 z_brnaz(AllDone);
5136 z_stc(Z_R1, 0, Z_R0, Rdst);
5137 z_lghi(result, 1);
5138 z_bru(AllDone);
5139
5140 bind(Scalar2Char);
5141 z_llh(Z_R0, 2, Z_R0, Rsrc);
5142 z_tmll(Z_R1, 0xff00);
5143 z_lghi(result, 0); // cnt == 2, first char invalid, no chars successfully processed
5144 z_brnaz(AllDone);
5145 z_stc(Z_R1, 0, Z_R0, Rdst);
5146 z_tmll(Z_R0, 0xff00);
5147 z_lghi(result, 1); // cnt == 2, second char invalid, one char successfully processed
5148 z_brnaz(AllDone);
5149 z_stc(Z_R0, 1, Z_R0, Rdst);
5150 z_lghi(result, 2);
5151 z_bru(AllDone);
5152
5153 bind(ScalarDoit);
5154 }
5155
5156 if (VM_Version::has_DistinctOpnds()) {
5157 z_srk(Rix, Rcnt, Z_R0); // remaining # chars to compress in unrolled loop
5158 } else {
5159 z_lr(Rix, Rcnt);
5160 z_sr(Rix, Z_R0);
5161 }
5162 z_lgfr(result, Rcnt); // # processed characters (if all runs ok).
5163 z_brz(ScalarDone);
5164
5165 bind(ScalarLoop);
5166 z_llh(Z_R1, 0, Z_R0, Rsrc);
5167 z_tmll(Z_R1, 0xff00);
5168 z_brnaz(ScalarBreak);
5169 z_stc(Z_R1, 0, Z_R0, Rdst);
5170 add2reg(Rsrc, 2);
5171 add2reg(Rdst, 1);
5172 z_brct(Rix, ScalarLoop);
5173
5174 z_bru(ScalarDone);
5175
5176 bind(ScalarBreak);
5177 z_sr(result, Rix);
5178
5179 bind(ScalarDone);
5180 z_sgfr(Rdst, result); // restore ptr
5181 z_sgfr(Rsrc, result); // restore ptr, double the element count for Rsrc restore
5182 z_sgfr(Rsrc, result);
5183 }
5184 bind(AllDone);
5185
5186 if (precise) {
5187 BLOCK_COMMENT("} encode_iso_array");
5188 } else {
5189 BLOCK_COMMENT("} string_compress");
5190 }
5191 return offset() - block_start;
5192 }
5193
5194 // Inflate byte[] to char[].
5195 unsigned int MacroAssembler::string_inflate_trot(Register src, Register dst, Register cnt, Register tmp) {
5196 int block_start = offset();
5197
5198 BLOCK_COMMENT("string_inflate {");
5199
5200 Register stop_char = Z_R0;
5201 Register table = Z_R1;
5202 Register src_addr = tmp;
5203
5204 assert_different_registers(Z_R0, Z_R1, tmp, src, dst, cnt);
5205 assert(dst->encoding()%2 == 0, "must be even reg");
5206 assert(cnt->encoding()%2 == 1, "must be odd reg");
5207 assert(cnt->encoding() - dst->encoding() == 1, "must be even/odd pair");
5208
5209 StubRoutines::zarch::generate_load_trot_table_addr(this, table); // kills Z_R0 (if ASSERT)
5210 clear_reg(stop_char); // Stop character. Not used here, but initialized to have a defined value.
5211 lgr_if_needed(src_addr, src);
5212 z_llgfr(cnt, cnt); // # src characters, must be a positive simm32.
5213
5214 translate_ot(dst, src_addr, /* mask = */ 0x0001);
5215
5216 BLOCK_COMMENT("} string_inflate");
5217
5218 return offset() - block_start;
5219 }
5220
5221 // Inflate byte[] to char[].
5222 // Restores: src, dst
5223 // Uses: cnt
5224 // Kills: tmp, Z_R0, Z_R1.
5225 // Note:
5226 // cnt is signed int. Do not rely on high word!
5227 // counts # characters, not bytes.
5228 unsigned int MacroAssembler::string_inflate(Register src, Register dst, Register cnt, Register tmp) {
5229 assert_different_registers(Z_R0, Z_R1, src, dst, cnt, tmp);
5230
5231 BLOCK_COMMENT("string_inflate {");
5232 int block_start = offset();
5233
5234 Register Rcnt = cnt; // # characters (src: bytes, dst: char (2-byte)), remaining after current loop.
5235 Register Rix = tmp; // loop index
5236 Register Rsrc = src; // addr(src array)
5237 Register Rdst = dst; // addr(dst array)
5238 Label ScalarShortcut, AllDone;
5239
5240 {
5241 //---< shortcuts for short strings (very frequent) >---
5242 Label skipShortcut, skip4Shortcut;
5243 z_ltr(Rcnt, Rcnt); // absolutely nothing to do for strings of len == 0.
5244 z_brz(AllDone);
5245 clear_reg(Z_R0); // make sure registers are properly initialized.
5246 clear_reg(Z_R1);
5247 z_chi(Rcnt, 4);
5248 z_brne(skip4Shortcut); // 4 characters are very frequent
5249 z_icm(Z_R0, 5, 0, Rsrc); // Treat exactly 4 characters specially.
5250 z_icm(Z_R1, 5, 2, Rsrc);
5251 z_stm(Z_R0, Z_R1, 0, Rdst);
5252 z_bru(AllDone);
5253 bind(skip4Shortcut);
5254
5255 z_chi(Rcnt, 8);
5256 z_brh(skipShortcut); // There's a lot to do...
5257 z_lgfr(Z_R0, Rcnt); // remaining #characters (<= 8). Precond for scalar loop.
5258 // This does not destroy the "register cleared" state of Z_R0.
5259 z_brl(ScalarShortcut); // Just a few characters
5260 z_icmh(Z_R0, 5, 0, Rsrc); // Treat exactly 8 characters specially.
5261 z_icmh(Z_R1, 5, 4, Rsrc);
5262 z_icm(Z_R0, 5, 2, Rsrc);
5263 z_icm(Z_R1, 5, 6, Rsrc);
5264 z_stmg(Z_R0, Z_R1, 0, Rdst);
5265 z_bru(AllDone);
5266 bind(skipShortcut);
5267 }
5268 clear_reg(Z_R0); // make sure register is properly initialized.
5269
5270 if (VM_Version::has_VectorFacility()) {
5271 const int min_vcnt = 32; // Minimum #characters required to use vector instructions.
5272 // Otherwise just do nothing in vector mode.
5273 // Must be multiple of vector register length (16 bytes = 128 bits).
5274 const int log_min_vcnt = exact_log2(min_vcnt);
5275 Label VectorLoop, VectorDone;
5276
5277 assert(VM_Version::has_DistinctOpnds(), "Assumption when has_VectorFacility()");
5278 z_srak(Rix, Rcnt, log_min_vcnt); // calculate # vector loop iterations
5279 z_brz(VectorDone); // skip if none
5280
5281 z_sllg(Z_R0, Rix, log_min_vcnt); // remember #chars that will be processed by vector loop
5282
5283 bind(VectorLoop);
5284 z_vlm(Z_V20, Z_V21, 0, Rsrc); // get next 32 characters (single-byte)
5285 add2reg(Rsrc, min_vcnt);
5286
5287 z_vuplhb(Z_V22, Z_V20); // V2 <- (expand) V0(high)
5288 z_vupllb(Z_V23, Z_V20); // V3 <- (expand) V0(low)
5289 z_vuplhb(Z_V24, Z_V21); // V4 <- (expand) V1(high)
5290 z_vupllb(Z_V25, Z_V21); // V5 <- (expand) V1(low)
5291 z_vstm(Z_V22, Z_V25, 0, Rdst); // store next 32 bytes
5292 add2reg(Rdst, min_vcnt*2);
5293
5294 z_brct(Rix, VectorLoop);
5295
5296 bind(VectorDone);
5297 }
5298
5299 const int min_cnt = 8; // Minimum #characters required to use unrolled scalar loop.
5300 // Otherwise just do nothing in unrolled scalar mode.
5301 // Must be multiple of 8.
5302 {
5303 const int log_min_cnt = exact_log2(min_cnt);
5304 Label UnrolledLoop, UnrolledDone;
5305
5306
5307 if (VM_Version::has_DistinctOpnds()) {
5308 z_srk(Rix, Rcnt, Z_R0); // remaining # chars to process in unrolled loop
5309 } else {
5310 z_lr(Rix, Rcnt);
5311 z_sr(Rix, Z_R0);
5312 }
5313 z_sra(Rix, log_min_cnt); // unrolled loop count
5314 z_brz(UnrolledDone);
5315
5316 clear_reg(Z_R0);
5317 clear_reg(Z_R1);
5318
5319 bind(UnrolledLoop);
5320 z_icmh(Z_R0, 5, 0, Rsrc);
5321 z_icmh(Z_R1, 5, 4, Rsrc);
5322 z_icm(Z_R0, 5, 2, Rsrc);
5323 z_icm(Z_R1, 5, 6, Rsrc);
5324 add2reg(Rsrc, min_cnt);
5325
5326 z_stmg(Z_R0, Z_R1, 0, Rdst);
5327
5328 add2reg(Rdst, min_cnt*2);
5329 z_brct(Rix, UnrolledLoop);
5330
5331 bind(UnrolledDone);
5332 z_lgfr(Z_R0, Rcnt); // # chars left over after unrolled loop.
5333 z_nilf(Z_R0, min_cnt-1);
5334 z_brnz(ScalarShortcut); // if zero, there is nothing left to do for scalar loop.
5335 // Rix == 0 in all cases.
5336 z_sgfr(Z_R0, Rcnt); // negative # characters the ptrs have been advanced previously.
5337 z_agr(Rdst, Z_R0); // restore ptr, double the element count for Rdst restore.
5338 z_agr(Rdst, Z_R0);
5339 z_agr(Rsrc, Z_R0); // restore ptr.
5340 z_bru(AllDone);
5341 }
5342
5343 {
5344 bind(ScalarShortcut);
5345 // Z_R0 must contain remaining # characters as 64-bit signed int here.
5346 // register contents is preserved over scalar processing (for register fixup).
5347
5348 {
5349 Label ScalarDefault;
5350 z_chi(Rcnt, 2);
5351 z_brh(ScalarDefault);
5352 z_llc(Z_R0, 0, Z_R0, Rsrc); // 6 bytes
5353 z_sth(Z_R0, 0, Z_R0, Rdst); // 4 bytes
5354 z_brl(AllDone);
5355 z_llc(Z_R0, 1, Z_R0, Rsrc); // 6 bytes
5356 z_sth(Z_R0, 2, Z_R0, Rdst); // 4 bytes
5357 z_bru(AllDone);
5358 bind(ScalarDefault);
5359 }
5360
5361 Label CodeTable;
5362 // Some comments on Rix calculation:
5363 // - Rcnt is small, therefore no bits shifted out of low word (sll(g) instructions).
5364 // - high word of both Rix and Rcnt may contain garbage
5365 // - the final lngfr takes care of that garbage, extending the sign to high word
5366 z_sllg(Rix, Z_R0, 2); // calculate 10*Rix = (4*Rix + Rix)*2
5367 z_ar(Rix, Z_R0);
5368 z_larl(Z_R1, CodeTable);
5369 z_sll(Rix, 1);
5370 z_lngfr(Rix, Rix); // ix range: [0..7], after inversion & mult: [-(7*12)..(0*12)].
5371 z_bc(Assembler::bcondAlways, 0, Rix, Z_R1);
5372
5373 z_llc(Z_R1, 6, Z_R0, Rsrc); // 6 bytes
5374 z_sth(Z_R1, 12, Z_R0, Rdst); // 4 bytes
5375
5376 z_llc(Z_R1, 5, Z_R0, Rsrc);
5377 z_sth(Z_R1, 10, Z_R0, Rdst);
5378
5379 z_llc(Z_R1, 4, Z_R0, Rsrc);
5380 z_sth(Z_R1, 8, Z_R0, Rdst);
5381
5382 z_llc(Z_R1, 3, Z_R0, Rsrc);
5383 z_sth(Z_R1, 6, Z_R0, Rdst);
5384
5385 z_llc(Z_R1, 2, Z_R0, Rsrc);
5386 z_sth(Z_R1, 4, Z_R0, Rdst);
5387
5388 z_llc(Z_R1, 1, Z_R0, Rsrc);
5389 z_sth(Z_R1, 2, Z_R0, Rdst);
5390
5391 z_llc(Z_R1, 0, Z_R0, Rsrc);
5392 z_sth(Z_R1, 0, Z_R0, Rdst);
5393 bind(CodeTable);
5394
5395 z_chi(Rcnt, 8); // no fixup for small strings. Rdst, Rsrc were not modified.
5396 z_brl(AllDone);
5397
5398 z_sgfr(Z_R0, Rcnt); // # characters the ptrs have been advanced previously.
5399 z_agr(Rdst, Z_R0); // restore ptr, double the element count for Rdst restore.
5400 z_agr(Rdst, Z_R0);
5401 z_agr(Rsrc, Z_R0); // restore ptr.
5402 }
5403 bind(AllDone);
5404
5405 BLOCK_COMMENT("} string_inflate");
5406 return offset() - block_start;
5407 }
5408
5409 // Inflate byte[] to char[], length known at compile time.
5410 // Restores: src, dst
5411 // Kills: tmp, Z_R0, Z_R1.
5412 // Note:
5413 // len is signed int. Counts # characters, not bytes.
5414 unsigned int MacroAssembler::string_inflate_const(Register src, Register dst, Register tmp, int len) {
5415 assert_different_registers(Z_R0, Z_R1, src, dst, tmp);
5416
5417 BLOCK_COMMENT("string_inflate_const {");
5418 int block_start = offset();
5419
5420 Register Rix = tmp; // loop index
5421 Register Rsrc = src; // addr(src array)
5422 Register Rdst = dst; // addr(dst array)
5423 Label ScalarShortcut, AllDone;
5424 int nprocessed = 0;
5425 int src_off = 0; // compensate for saved (optimized away) ptr advancement.
5426 int dst_off = 0; // compensate for saved (optimized away) ptr advancement.
5427 bool restore_inputs = false;
5428 bool workreg_clear = false;
5429
5430 if ((len >= 32) && VM_Version::has_VectorFacility()) {
5431 const int min_vcnt = 32; // Minimum #characters required to use vector instructions.
5432 // Otherwise just do nothing in vector mode.
5433 // Must be multiple of vector register length (16 bytes = 128 bits).
5434 const int log_min_vcnt = exact_log2(min_vcnt);
5435 const int iterations = (len - nprocessed) >> log_min_vcnt;
5436 nprocessed += iterations << log_min_vcnt;
5437 Label VectorLoop;
5438
5439 if (iterations == 1) {
5440 z_vlm(Z_V20, Z_V21, 0+src_off, Rsrc); // get next 32 characters (single-byte)
5441 z_vuplhb(Z_V22, Z_V20); // V2 <- (expand) V0(high)
5442 z_vupllb(Z_V23, Z_V20); // V3 <- (expand) V0(low)
5443 z_vuplhb(Z_V24, Z_V21); // V4 <- (expand) V1(high)
5444 z_vupllb(Z_V25, Z_V21); // V5 <- (expand) V1(low)
5445 z_vstm(Z_V22, Z_V25, 0+dst_off, Rdst); // store next 32 bytes
5446
5447 src_off += min_vcnt;
5448 dst_off += min_vcnt*2;
5449 } else {
5450 restore_inputs = true;
5451
5452 z_lgfi(Rix, len>>log_min_vcnt);
5453 bind(VectorLoop);
5454 z_vlm(Z_V20, Z_V21, 0, Rsrc); // get next 32 characters (single-byte)
5455 add2reg(Rsrc, min_vcnt);
5456
5457 z_vuplhb(Z_V22, Z_V20); // V2 <- (expand) V0(high)
5458 z_vupllb(Z_V23, Z_V20); // V3 <- (expand) V0(low)
5459 z_vuplhb(Z_V24, Z_V21); // V4 <- (expand) V1(high)
5460 z_vupllb(Z_V25, Z_V21); // V5 <- (expand) V1(low)
5461 z_vstm(Z_V22, Z_V25, 0, Rdst); // store next 32 bytes
5462 add2reg(Rdst, min_vcnt*2);
5463
5464 z_brct(Rix, VectorLoop);
5465 }
5466 }
5467
5468 if (((len-nprocessed) >= 16) && VM_Version::has_VectorFacility()) {
5469 const int min_vcnt = 16; // Minimum #characters required to use vector instructions.
5470 // Otherwise just do nothing in vector mode.
5471 // Must be multiple of vector register length (16 bytes = 128 bits).
5472 const int log_min_vcnt = exact_log2(min_vcnt);
5473 const int iterations = (len - nprocessed) >> log_min_vcnt;
5474 nprocessed += iterations << log_min_vcnt;
5475 assert(iterations == 1, "must be!");
5476
5477 z_vl(Z_V20, 0+src_off, Z_R0, Rsrc); // get next 16 characters (single-byte)
5478 z_vuplhb(Z_V22, Z_V20); // V2 <- (expand) V0(high)
5479 z_vupllb(Z_V23, Z_V20); // V3 <- (expand) V0(low)
5480 z_vstm(Z_V22, Z_V23, 0+dst_off, Rdst); // store next 32 bytes
5481
5482 src_off += min_vcnt;
5483 dst_off += min_vcnt*2;
5484 }
5485
5486 if ((len-nprocessed) > 8) {
5487 const int min_cnt = 8; // Minimum #characters required to use unrolled scalar loop.
5488 // Otherwise just do nothing in unrolled scalar mode.
5489 // Must be multiple of 8.
5490 const int log_min_cnt = exact_log2(min_cnt);
5491 const int iterations = (len - nprocessed) >> log_min_cnt;
5492 nprocessed += iterations << log_min_cnt;
5493
5494 //---< avoid loop overhead/ptr increment for small # iterations >---
5495 if (iterations <= 2) {
5496 clear_reg(Z_R0);
5497 clear_reg(Z_R1);
5498 workreg_clear = true;
5499
5500 z_icmh(Z_R0, 5, 0+src_off, Rsrc);
5501 z_icmh(Z_R1, 5, 4+src_off, Rsrc);
5502 z_icm(Z_R0, 5, 2+src_off, Rsrc);
5503 z_icm(Z_R1, 5, 6+src_off, Rsrc);
5504 z_stmg(Z_R0, Z_R1, 0+dst_off, Rdst);
5505
5506 src_off += min_cnt;
5507 dst_off += min_cnt*2;
5508 }
5509
5510 if (iterations == 2) {
5511 z_icmh(Z_R0, 5, 0+src_off, Rsrc);
5512 z_icmh(Z_R1, 5, 4+src_off, Rsrc);
5513 z_icm(Z_R0, 5, 2+src_off, Rsrc);
5514 z_icm(Z_R1, 5, 6+src_off, Rsrc);
5515 z_stmg(Z_R0, Z_R1, 0+dst_off, Rdst);
5516
5517 src_off += min_cnt;
5518 dst_off += min_cnt*2;
5519 }
5520
5521 if (iterations > 2) {
5522 Label UnrolledLoop;
5523 restore_inputs = true;
5524
5525 clear_reg(Z_R0);
5526 clear_reg(Z_R1);
5527 workreg_clear = true;
5528
5529 z_lgfi(Rix, iterations);
5530 bind(UnrolledLoop);
5531 z_icmh(Z_R0, 5, 0, Rsrc);
5532 z_icmh(Z_R1, 5, 4, Rsrc);
5533 z_icm(Z_R0, 5, 2, Rsrc);
5534 z_icm(Z_R1, 5, 6, Rsrc);
5535 add2reg(Rsrc, min_cnt);
5536
5537 z_stmg(Z_R0, Z_R1, 0, Rdst);
5538 add2reg(Rdst, min_cnt*2);
5539
5540 z_brct(Rix, UnrolledLoop);
5541 }
5542 }
5543
5544 if ((len-nprocessed) > 0) {
5545 switch (len-nprocessed) {
5546 case 8:
5547 if (!workreg_clear) {
5548 clear_reg(Z_R0);
5549 clear_reg(Z_R1);
5550 }
5551 z_icmh(Z_R0, 5, 0+src_off, Rsrc);
5552 z_icmh(Z_R1, 5, 4+src_off, Rsrc);
5553 z_icm(Z_R0, 5, 2+src_off, Rsrc);
5554 z_icm(Z_R1, 5, 6+src_off, Rsrc);
5555 z_stmg(Z_R0, Z_R1, 0+dst_off, Rdst);
5556 break;
5557 case 7:
5558 if (!workreg_clear) {
5559 clear_reg(Z_R0);
5560 clear_reg(Z_R1);
5561 }
5562 clear_reg(Rix);
5563 z_icm(Z_R0, 5, 0+src_off, Rsrc);
5564 z_icm(Z_R1, 5, 2+src_off, Rsrc);
5565 z_icm(Rix, 5, 4+src_off, Rsrc);
5566 z_stm(Z_R0, Z_R1, 0+dst_off, Rdst);
5567 z_llc(Z_R0, 6+src_off, Z_R0, Rsrc);
5568 z_st(Rix, 8+dst_off, Z_R0, Rdst);
5569 z_sth(Z_R0, 12+dst_off, Z_R0, Rdst);
5570 break;
5571 case 6:
5572 if (!workreg_clear) {
5573 clear_reg(Z_R0);
5574 clear_reg(Z_R1);
5575 }
5576 clear_reg(Rix);
5577 z_icm(Z_R0, 5, 0+src_off, Rsrc);
5578 z_icm(Z_R1, 5, 2+src_off, Rsrc);
5579 z_icm(Rix, 5, 4+src_off, Rsrc);
5580 z_stm(Z_R0, Z_R1, 0+dst_off, Rdst);
5581 z_st(Rix, 8+dst_off, Z_R0, Rdst);
5582 break;
5583 case 5:
5584 if (!workreg_clear) {
5585 clear_reg(Z_R0);
5586 clear_reg(Z_R1);
5587 }
5588 z_icm(Z_R0, 5, 0+src_off, Rsrc);
5589 z_icm(Z_R1, 5, 2+src_off, Rsrc);
5590 z_llc(Rix, 4+src_off, Z_R0, Rsrc);
5591 z_stm(Z_R0, Z_R1, 0+dst_off, Rdst);
5592 z_sth(Rix, 8+dst_off, Z_R0, Rdst);
5593 break;
5594 case 4:
5595 if (!workreg_clear) {
5596 clear_reg(Z_R0);
5597 clear_reg(Z_R1);
5598 }
5599 z_icm(Z_R0, 5, 0+src_off, Rsrc);
5600 z_icm(Z_R1, 5, 2+src_off, Rsrc);
5601 z_stm(Z_R0, Z_R1, 0+dst_off, Rdst);
5602 break;
5603 case 3:
5604 if (!workreg_clear) {
5605 clear_reg(Z_R0);
5606 }
5607 z_llc(Z_R1, 2+src_off, Z_R0, Rsrc);
5608 z_icm(Z_R0, 5, 0+src_off, Rsrc);
5609 z_sth(Z_R1, 4+dst_off, Z_R0, Rdst);
5610 z_st(Z_R0, 0+dst_off, Rdst);
5611 break;
5612 case 2:
5613 z_llc(Z_R0, 0+src_off, Z_R0, Rsrc);
5614 z_llc(Z_R1, 1+src_off, Z_R0, Rsrc);
5615 z_sth(Z_R0, 0+dst_off, Z_R0, Rdst);
5616 z_sth(Z_R1, 2+dst_off, Z_R0, Rdst);
5617 break;
5618 case 1:
5619 z_llc(Z_R0, 0+src_off, Z_R0, Rsrc);
5620 z_sth(Z_R0, 0+dst_off, Z_R0, Rdst);
5621 break;
5622 default:
5623 guarantee(false, "Impossible");
5624 break;
5625 }
5626 src_off += len-nprocessed;
5627 dst_off += (len-nprocessed)*2;
5628 nprocessed = len;
5629 }
5630
5631 //---< restore modified input registers >---
5632 if ((nprocessed > 0) && restore_inputs) {
5633 z_agfi(Rsrc, -(nprocessed-src_off));
5634 if (nprocessed < 1000000000) { // avoid int overflow
5635 z_agfi(Rdst, -(nprocessed*2-dst_off));
5636 } else {
5637 z_agfi(Rdst, -(nprocessed-dst_off));
5638 z_agfi(Rdst, -nprocessed);
5639 }
5640 }
5641
5642 BLOCK_COMMENT("} string_inflate_const");
5643 return offset() - block_start;
5644 }
5645
5646 // Kills src.
5647 unsigned int MacroAssembler::has_negatives(Register result, Register src, Register cnt,
5648 Register odd_reg, Register even_reg, Register tmp) {
5649 int block_start = offset();
5650 Label Lloop1, Lloop2, Lslow, Lnotfound, Ldone;
5651 const Register addr = src, mask = tmp;
5652
5653 BLOCK_COMMENT("has_negatives {");
5654
5655 z_llgfr(Z_R1, cnt); // Number of bytes to read. (Must be a positive simm32.)
5656 z_llilf(mask, 0x80808080);
5657 z_lhi(result, 1); // Assume true.
5658 // Last possible addr for fast loop.
5659 z_lay(odd_reg, -16, Z_R1, src);
5660 z_chi(cnt, 16);
5661 z_brl(Lslow);
5662
|