1 /*
2 * Copyright (c) 2018, 2019, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23
24
25 package org.graalvm.compiler.lir.amd64;
26
27 import static jdk.vm.ci.amd64.AMD64.k1;
28 import static jdk.vm.ci.amd64.AMD64.k2;
29 import static jdk.vm.ci.amd64.AMD64.rdi;
30 import static jdk.vm.ci.amd64.AMD64.rdx;
31 import static jdk.vm.ci.amd64.AMD64.rsi;
32 import static jdk.vm.ci.code.ValueUtil.asRegister;
33 import static org.graalvm.compiler.lir.LIRInstruction.OperandFlag.REG;
34
35 import org.graalvm.compiler.asm.Label;
36 import org.graalvm.compiler.asm.amd64.AMD64Address;
37 import org.graalvm.compiler.asm.amd64.AMD64Assembler;
38 import org.graalvm.compiler.asm.amd64.AMD64MacroAssembler;
39 import org.graalvm.compiler.core.common.LIRKind;
40 import org.graalvm.compiler.lir.LIRInstructionClass;
41 import org.graalvm.compiler.lir.Opcode;
42 import org.graalvm.compiler.lir.asm.CompilationResultBuilder;
43 import org.graalvm.compiler.lir.gen.LIRGeneratorTool;
44
45 import jdk.vm.ci.amd64.AMD64;
46 import jdk.vm.ci.amd64.AMD64Kind;
47 import jdk.vm.ci.code.Register;
48 import jdk.vm.ci.meta.Value;
49
50 @Opcode("AMD64_STRING_INFLATE")
51 public final class AMD64StringLatin1InflateOp extends AMD64LIRInstruction {
52 public static final LIRInstructionClass<AMD64StringLatin1InflateOp> TYPE = LIRInstructionClass.create(AMD64StringLatin1InflateOp.class);
53
54 @Use({REG}) private Value rsrc;
55 @Use({REG}) private Value rdst;
56 @Use({REG}) private Value rlen;
57
58 @Temp({REG}) private Value rsrcTemp;
59 @Temp({REG}) private Value rdstTemp;
60 @Temp({REG}) private Value rlenTemp;
61
62 @Temp({REG}) private Value vtmp1;
63 @Temp({REG}) private Value rtmp2;
64
65 public AMD64StringLatin1InflateOp(LIRGeneratorTool tool, Value src, Value dst, Value len) {
66 super(TYPE);
67
68 assert asRegister(src).equals(rsi);
69 assert asRegister(dst).equals(rdi);
70 assert asRegister(len).equals(rdx);
71
72 rsrcTemp = rsrc = src;
73 rdstTemp = rdst = dst;
74 rlenTemp = rlen = len;
75
76 vtmp1 = tool.newVariable(LIRKind.value(AMD64Kind.V512_BYTE));
77 rtmp2 = tool.newVariable(LIRKind.value(AMD64Kind.DWORD));
78 }
79
80 @Override
81 public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) {
82 Register src = asRegister(rsrc);
83 Register dst = asRegister(rdst);
84 Register len = asRegister(rlen);
85
86 Register tmp1 = asRegister(vtmp1);
87 Register tmp2 = asRegister(rtmp2);
88
89 byteArrayInflate(masm, src, dst, len, tmp1, tmp2);
90 }
91
92 /**
93 * Inflate a Latin1 string using a byte[] array representation into a UTF16 string using a
94 * char[] array representation.
95 *
96 * @param masm the assembler
97 * @param src (rsi) the start address of source byte[] to be inflated
98 * @param dst (rdi) the start address of destination char[] array
99 * @param len (rdx) the length
100 * @param vtmp (xmm) temporary xmm register
101 * @param tmp (gpr) temporary gpr register
102 */
103 private static void byteArrayInflate(AMD64MacroAssembler masm, Register src, Register dst, Register len, Register vtmp, Register tmp) {
104 assert vtmp.getRegisterCategory().equals(AMD64.XMM);
105
106 Label labelDone = new Label();
107 Label labelBelowThreshold = new Label();
108
109 assert src.number != dst.number && src.number != len.number && src.number != tmp.number;
110 assert dst.number != len.number && dst.number != tmp.number;
111 assert len.number != tmp.number;
112
113 if (masm.supports(AMD64.CPUFeature.AVX512BW) &&
114 masm.supports(AMD64.CPUFeature.AVX512VL) &&
115 masm.supports(AMD64.CPUFeature.BMI2)) {
116
117 // If the length of the string is less than 16, we chose not to use the
118 // AVX512 instructions.
119 masm.testl(len, -16);
120 masm.jcc(AMD64Assembler.ConditionFlag.Zero, labelBelowThreshold);
121
122 Label labelAvx512Tail = new Label();
123 // Test for suitable number chunks with respect to the size of the vector
124 // operation, mask off remaining number of chars (bytes) to inflate (such
125 // that 'len' will always hold the number of bytes left to inflate) after
126 // committing to the vector loop.
127 // Adjust vector pointers to upper address bounds and inverse loop index.
128 // This will keep the loop condition simple.
129 //
130 // NOTE: The above idiom/pattern is used in all the loops below.
131
132 masm.movl(tmp, len);
133 masm.andl(tmp, -32); // The vector count (in chars).
134 masm.jccb(AMD64Assembler.ConditionFlag.Zero, labelAvx512Tail);
135 masm.andl(len, 32 - 1); // The tail count (in chars).
136
137 masm.leaq(src, new AMD64Address(src, tmp, AMD64Address.Scale.Times1));
138 masm.leaq(dst, new AMD64Address(dst, tmp, AMD64Address.Scale.Times2));
139 masm.negq(tmp);
140
141 Label labelAvx512Loop = new Label();
142 // Inflate 32 chars per iteration, reading 256-bit compact vectors
143 // and writing 512-bit inflated ditto.
144 masm.bind(labelAvx512Loop);
145 masm.evpmovzxbw(vtmp, new AMD64Address(src, tmp, AMD64Address.Scale.Times1));
146 masm.evmovdqu16(new AMD64Address(dst, tmp, AMD64Address.Scale.Times2), vtmp);
147 masm.addq(tmp, 32);
148 masm.jcc(AMD64Assembler.ConditionFlag.NotZero, labelAvx512Loop);
149
150 masm.bind(labelAvx512Tail);
151 // All done if the tail count is zero.
152 masm.testl(len, len);
153 masm.jcc(AMD64Assembler.ConditionFlag.Zero, labelDone);
154
155 masm.kmovq(k2, k1); // Save k1
156
157 // Compute (1 << N) - 1 = ~(~0 << N), where N is the remaining number
158 // of characters to process.
159 masm.movl(tmp, -1);
160 masm.shlxl(tmp, tmp, len);
161 masm.notl(tmp);
162
163 masm.kmovd(k1, tmp);
164 masm.evpmovzxbw(vtmp, k1, new AMD64Address(src));
165 masm.evmovdqu16(new AMD64Address(dst), k1, vtmp);
166 masm.kmovq(k1, k2); // Restore k1
167 masm.jmp(labelDone);
168 }
169
170 if (masm.supports(AMD64.CPUFeature.SSE4_1)) {
171
172 Label labelSSETail = new Label();
173
174 if (masm.supports(AMD64.CPUFeature.AVX2)) {
175
176 Label labelAvx2Tail = new Label();
177
178 masm.movl(tmp, len);
179 masm.andl(tmp, -16);
180 masm.jccb(AMD64Assembler.ConditionFlag.Zero, labelAvx2Tail);
181 masm.andl(len, 16 - 1);
182
183 masm.leaq(src, new AMD64Address(src, tmp, AMD64Address.Scale.Times1));
184 masm.leaq(dst, new AMD64Address(dst, tmp, AMD64Address.Scale.Times2));
185 masm.negq(tmp);
186
187 Label labelAvx2Loop = new Label();
188 // Inflate 16 bytes (chars) per iteration, reading 128-bit compact vectors
189 // and writing 256-bit inflated ditto.
190 masm.bind(labelAvx2Loop);
191 masm.vpmovzxbw(vtmp, new AMD64Address(src, tmp, AMD64Address.Scale.Times1));
192 masm.vmovdqu(new AMD64Address(dst, tmp, AMD64Address.Scale.Times2), vtmp);
193 masm.addq(tmp, 16);
194 masm.jcc(AMD64Assembler.ConditionFlag.NotZero, labelAvx2Loop);
195
196 masm.bind(labelBelowThreshold);
197 masm.bind(labelAvx2Tail);
198
199 masm.movl(tmp, len);
200 masm.andl(tmp, -8);
201 masm.jccb(AMD64Assembler.ConditionFlag.Zero, labelSSETail);
202 masm.andl(len, 8 - 1);
203
204 // Inflate another 8 bytes before final tail copy.
205 masm.pmovzxbw(vtmp, new AMD64Address(src));
206 masm.movdqu(new AMD64Address(dst), vtmp);
207 masm.addq(src, 8);
208 masm.addq(dst, 16);
209
210 // Fall-through to labelSSETail.
211 } else {
212 // When there is no AVX2 support available, we use AVX/SSE support to
213 // inflate into maximum 128-bits per operation.
214
215 masm.movl(tmp, len);
216 masm.andl(tmp, -8);
217 masm.jccb(AMD64Assembler.ConditionFlag.Zero, labelSSETail);
218 masm.andl(len, 8 - 1);
219
220 masm.leaq(src, new AMD64Address(src, tmp, AMD64Address.Scale.Times1));
221 masm.leaq(dst, new AMD64Address(dst, tmp, AMD64Address.Scale.Times2));
222 masm.negq(tmp);
223
224 Label labelSSECopy8Loop = new Label();
225 // Inflate 8 bytes (chars) per iteration, reading 64-bit compact vectors
226 // and writing 128-bit inflated ditto.
227 masm.bind(labelSSECopy8Loop);
228 masm.pmovzxbw(vtmp, new AMD64Address(src, tmp, AMD64Address.Scale.Times1));
229 masm.movdqu(new AMD64Address(dst, tmp, AMD64Address.Scale.Times2), vtmp);
230 masm.addq(tmp, 8);
231 masm.jcc(AMD64Assembler.ConditionFlag.NotZero, labelSSECopy8Loop);
232
233 // Fall-through to labelSSETail.
234 }
235
236 Label labelCopyChars = new Label();
237
238 masm.bind(labelSSETail);
239 masm.cmpl(len, 4);
240 masm.jccb(AMD64Assembler.ConditionFlag.Less, labelCopyChars);
241
242 masm.movdl(vtmp, new AMD64Address(src));
243 masm.pmovzxbw(vtmp, vtmp);
244 masm.movq(new AMD64Address(dst), vtmp);
245 masm.subq(len, 4);
246 masm.addq(src, 4);
247 masm.addq(dst, 8);
248
249 masm.bind(labelCopyChars);
250 }
251
252 // Inflate any remaining characters (bytes) using a vanilla implementation.
253 masm.testl(len, len);
254 masm.jccb(AMD64Assembler.ConditionFlag.Zero, labelDone);
255 masm.leaq(src, new AMD64Address(src, len, AMD64Address.Scale.Times1));
256 masm.leaq(dst, new AMD64Address(dst, len, AMD64Address.Scale.Times2));
257 masm.negq(len);
258
259 Label labelCopyCharsLoop = new Label();
260 // Inflate a single byte (char) per iteration.
261 masm.bind(labelCopyCharsLoop);
262 masm.movzbl(tmp, new AMD64Address(src, len, AMD64Address.Scale.Times1));
263 masm.movw(new AMD64Address(dst, len, AMD64Address.Scale.Times2), tmp);
264 masm.incrementq(len, 1);
265 masm.jcc(AMD64Assembler.ConditionFlag.NotZero, labelCopyCharsLoop);
266
267 masm.bind(labelDone);
268 }
269
270 }