1 /*
2 * Copyright (c) 2013, 2019, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23
24
25 package org.graalvm.compiler.lir.amd64.vector;
26
27 import static jdk.vm.ci.code.ValueUtil.asRegister;
28 import static jdk.vm.ci.code.ValueUtil.isRegister;
29 import static jdk.vm.ci.code.ValueUtil.isStackSlot;
30 import static org.graalvm.compiler.asm.amd64.AMD64Assembler.VexMoveOp.VMOVD;
31 import static org.graalvm.compiler.asm.amd64.AMD64Assembler.VexMoveOp.VMOVDQU32;
32 import static org.graalvm.compiler.asm.amd64.AMD64Assembler.VexMoveOp.VMOVQ;
33 import static org.graalvm.compiler.asm.amd64.AMD64Assembler.VexMoveOp.VMOVSD;
34 import static org.graalvm.compiler.asm.amd64.AMD64Assembler.VexMoveOp.VMOVSS;
35 import static org.graalvm.compiler.asm.amd64.AMD64Assembler.VexMoveOp.VMOVUPD;
36 import static org.graalvm.compiler.asm.amd64.AMD64Assembler.VexMoveOp.VMOVUPS;
37 import static org.graalvm.compiler.asm.amd64.AMD64Assembler.VexRVMOp.VXORPD;
38 import static org.graalvm.compiler.lir.LIRInstruction.OperandFlag.COMPOSITE;
39 import static org.graalvm.compiler.lir.LIRInstruction.OperandFlag.HINT;
40 import static org.graalvm.compiler.lir.LIRInstruction.OperandFlag.REG;
41 import static org.graalvm.compiler.lir.LIRInstruction.OperandFlag.STACK;
42 import static org.graalvm.compiler.lir.LIRInstruction.OperandFlag.UNINITIALIZED;
43
44 import org.graalvm.compiler.asm.amd64.AMD64Address;
45 import org.graalvm.compiler.asm.amd64.AMD64Assembler.VexMoveOp;
46 import org.graalvm.compiler.asm.amd64.AMD64MacroAssembler;
47 import org.graalvm.compiler.asm.amd64.AVXKind;
48 import org.graalvm.compiler.asm.amd64.AVXKind.AVXSize;
49 import org.graalvm.compiler.debug.GraalError;
50 import org.graalvm.compiler.lir.LIRFrameState;
51 import org.graalvm.compiler.lir.LIRInstructionClass;
52 import org.graalvm.compiler.lir.Opcode;
53 import org.graalvm.compiler.lir.StandardOp.LoadConstantOp;
54 import org.graalvm.compiler.lir.StandardOp.ValueMoveOp;
55 import org.graalvm.compiler.lir.amd64.AMD64AddressValue;
56 import org.graalvm.compiler.lir.amd64.AMD64LIRInstruction;
57 import org.graalvm.compiler.lir.amd64.AMD64Move;
58 import org.graalvm.compiler.lir.amd64.AMD64RestoreRegistersOp;
59 import org.graalvm.compiler.lir.amd64.AMD64SaveRegistersOp;
60 import org.graalvm.compiler.lir.asm.CompilationResultBuilder;
61
62 import jdk.vm.ci.amd64.AMD64Kind;
63 import jdk.vm.ci.code.Register;
64 import jdk.vm.ci.code.RegisterValue;
65 import jdk.vm.ci.code.StackSlot;
66 import jdk.vm.ci.meta.AllocatableValue;
67 import jdk.vm.ci.meta.Constant;
68 import jdk.vm.ci.meta.JavaConstant;
69 import jdk.vm.ci.meta.Value;
70
71 public class AMD64VectorMove {
72
73 @Opcode("VMOVE")
74 public static final class MoveToRegOp extends AMD64LIRInstruction implements ValueMoveOp {
75 public static final LIRInstructionClass<MoveToRegOp> TYPE = LIRInstructionClass.create(MoveToRegOp.class);
76
77 @Def({REG, HINT}) protected AllocatableValue result;
78 @Use({REG, STACK}) protected AllocatableValue input;
79
80 public MoveToRegOp(AllocatableValue result, AllocatableValue input) {
81 super(TYPE);
82 this.result = result;
83 this.input = input;
84 }
85
86 @Override
87 public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) {
88 move(crb, masm, result, input);
89 }
90
91 @Override
92 public AllocatableValue getInput() {
93 return input;
94 }
95
96 @Override
97 public AllocatableValue getResult() {
98 return result;
99 }
100 }
101
102 @Opcode("VMOVE")
103 public static final class MoveFromRegOp extends AMD64LIRInstruction implements ValueMoveOp {
104 public static final LIRInstructionClass<MoveFromRegOp> TYPE = LIRInstructionClass.create(MoveFromRegOp.class);
105
106 @Def({REG, STACK}) protected AllocatableValue result;
107 @Use({REG, HINT}) protected AllocatableValue input;
108
109 public MoveFromRegOp(AllocatableValue result, AllocatableValue input) {
110 super(TYPE);
111 this.result = result;
112 this.input = input;
113 }
114
115 @Override
116 public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) {
117 move(crb, masm, result, input);
118 }
119
120 @Override
121 public AllocatableValue getInput() {
122 return input;
123 }
124
125 @Override
126 public AllocatableValue getResult() {
127 return result;
128 }
129 }
130
131 @Opcode("VMOVE")
132 public static class MoveFromConstOp extends AMD64LIRInstruction implements LoadConstantOp {
133 public static final LIRInstructionClass<MoveFromConstOp> TYPE = LIRInstructionClass.create(MoveFromConstOp.class);
134
135 @Def({REG, STACK}) protected AllocatableValue result;
136 private final JavaConstant input;
137
138 public MoveFromConstOp(AllocatableValue result, JavaConstant input) {
139 super(TYPE);
140 this.result = result;
141 this.input = input;
142 }
143
144 @Override
145 public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) {
146 if (isRegister(result)) {
147 const2reg(crb, masm, (RegisterValue) result, input);
148 } else {
149 assert isStackSlot(result);
150 AMD64Move.const2stack(crb, masm, result, input);
151 }
152 }
153
154 @Override
155 public Constant getConstant() {
156 return input;
157 }
158
159 @Override
160 public AllocatableValue getResult() {
161 return result;
162 }
163 }
164
165 @Opcode("VSTACKMOVE")
166 public static final class StackMoveOp extends AMD64LIRInstruction implements ValueMoveOp {
167 public static final LIRInstructionClass<StackMoveOp> TYPE = LIRInstructionClass.create(StackMoveOp.class);
168
169 @Def({STACK}) protected AllocatableValue result;
170 @Use({STACK, HINT}) protected AllocatableValue input;
171 @Alive({STACK, UNINITIALIZED}) private AllocatableValue backupSlot;
172
173 private Register scratch;
174
175 public StackMoveOp(AllocatableValue result, AllocatableValue input, Register scratch, AllocatableValue backupSlot) {
176 super(TYPE);
177 this.result = result;
178 this.input = input;
179 this.backupSlot = backupSlot;
180 this.scratch = scratch;
181 }
182
183 @Override
184 public AllocatableValue getInput() {
185 return input;
186 }
187
188 @Override
189 public AllocatableValue getResult() {
190 return result;
191 }
192
193 @Override
194 public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) {
195 // backup scratch register
196 move(crb, masm, backupSlot, scratch.asValue(backupSlot.getValueKind()));
197 // move stack slot
198 move(crb, masm, scratch.asValue(getInput().getValueKind()), getInput());
199 move(crb, masm, getResult(), scratch.asValue(getResult().getValueKind()));
200 // restore scratch register
201 move(crb, masm, scratch.asValue(backupSlot.getValueKind()), backupSlot);
202
203 }
204 }
205
206 public abstract static class VectorMemOp extends AMD64LIRInstruction {
207
208 protected final AVXSize size;
209 protected final VexMoveOp op;
210
211 @Use({COMPOSITE}) protected AMD64AddressValue address;
212 @State protected LIRFrameState state;
213
214 protected VectorMemOp(LIRInstructionClass<? extends VectorMemOp> c, AVXSize size, VexMoveOp op, AMD64AddressValue address, LIRFrameState state) {
215 super(c);
216 this.size = size;
217 this.op = op;
218 this.address = address;
219 this.state = state;
220 }
221
222 protected abstract void emitMemAccess(AMD64MacroAssembler masm);
223
224 @Override
225 public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) {
226 if (state != null) {
227 crb.recordImplicitException(masm.position(), state);
228 }
229 emitMemAccess(masm);
230 }
231 }
232
233 public static final class VectorLoadOp extends VectorMemOp {
234 public static final LIRInstructionClass<VectorLoadOp> TYPE = LIRInstructionClass.create(VectorLoadOp.class);
235
236 @Def({REG}) protected AllocatableValue result;
237
238 public VectorLoadOp(AVXSize size, VexMoveOp op, AllocatableValue result, AMD64AddressValue address, LIRFrameState state) {
239 super(TYPE, size, op, address, state);
240 this.result = result;
241 }
242
243 @Override
244 public void emitMemAccess(AMD64MacroAssembler masm) {
245 op.emit(masm, size, asRegister(result), address.toAddress());
246 }
247 }
248
249 public static class VectorStoreOp extends VectorMemOp {
250 public static final LIRInstructionClass<VectorStoreOp> TYPE = LIRInstructionClass.create(VectorStoreOp.class);
251
252 @Use({REG}) protected AllocatableValue input;
253
254 public VectorStoreOp(AVXSize size, VexMoveOp op, AMD64AddressValue address, AllocatableValue input, LIRFrameState state) {
255 super(TYPE, size, op, address, state);
256 this.input = input;
257 }
258
259 @Override
260 public void emitMemAccess(AMD64MacroAssembler masm) {
261 op.emit(masm, size, address.toAddress(), asRegister(input));
262 }
263 }
264
265 @Opcode("SAVE_REGISTER")
266 public static class SaveRegistersOp extends AMD64SaveRegistersOp {
267 public static final LIRInstructionClass<SaveRegistersOp> TYPE = LIRInstructionClass.create(SaveRegistersOp.class);
268
269 public SaveRegistersOp(Register[] savedRegisters, AllocatableValue[] slots) {
270 super(TYPE, savedRegisters, slots);
271 }
272
273 @Override
274 protected void saveRegister(CompilationResultBuilder crb, AMD64MacroAssembler masm, StackSlot result, Register register) {
275 AMD64Kind kind = (AMD64Kind) result.getPlatformKind();
276 if (kind.isXMM()) {
277 VexMoveOp op;
278 if (kind.getVectorLength() > 1) {
279 op = getVectorMoveOp(kind.getScalar());
280 } else {
281 op = getScalarMoveOp(kind);
282 }
283
284 AMD64Address addr = (AMD64Address) crb.asAddress(result);
285 op.emit(masm, AVXKind.getRegisterSize(kind), addr, register);
286 } else {
287 super.saveRegister(crb, masm, result, register);
288 }
289 }
290 }
291
292 @Opcode("RESTORE_REGISTER")
293 public static final class RestoreRegistersOp extends AMD64RestoreRegistersOp {
294 public static final LIRInstructionClass<RestoreRegistersOp> TYPE = LIRInstructionClass.create(RestoreRegistersOp.class);
295
296 public RestoreRegistersOp(AllocatableValue[] source, AMD64SaveRegistersOp save) {
297 super(TYPE, source, save);
298 }
299
300 @Override
301 protected void restoreRegister(CompilationResultBuilder crb, AMD64MacroAssembler masm, Register register, StackSlot input) {
302 AMD64Kind kind = (AMD64Kind) input.getPlatformKind();
303 if (kind.isXMM()) {
304 VexMoveOp op;
305 if (kind.getVectorLength() > 1) {
306 op = getVectorMoveOp(kind.getScalar());
307 } else {
308 op = getScalarMoveOp(kind);
309 }
310
311 AMD64Address addr = (AMD64Address) crb.asAddress(input);
312 op.emit(masm, AVXKind.getRegisterSize(kind), register, addr);
313 } else {
314 super.restoreRegister(crb, masm, register, input);
315 }
316 }
317 }
318
319 private static VexMoveOp getScalarMoveOp(AMD64Kind kind) {
320 switch (kind) {
321 case SINGLE:
322 return VMOVSS;
323 case DOUBLE:
324 return VMOVSD;
325 default:
326 throw GraalError.shouldNotReachHere();
327 }
328 }
329
330 private static VexMoveOp getVectorMoveOp(AMD64Kind kind) {
331 switch (kind) {
332 case SINGLE:
333 return VMOVUPS;
334 case DOUBLE:
335 return VMOVUPD;
336 default:
337 return VMOVDQU32;
338 }
339 }
340
341 private static VexMoveOp getVectorMemMoveOp(AMD64Kind kind) {
342 switch (AVXKind.getDataSize(kind)) {
343 case DWORD:
344 return VMOVD;
345 case QWORD:
346 return VMOVQ;
347 default:
348 return getVectorMoveOp(kind.getScalar());
349 }
350 }
351
352 private static void move(CompilationResultBuilder crb, AMD64MacroAssembler masm, AllocatableValue result, Value input) {
353 VexMoveOp op;
354 AVXSize size;
355 AMD64Kind kind = (AMD64Kind) result.getPlatformKind();
356 if (kind.getVectorLength() > 1) {
357 size = AVXKind.getRegisterSize(kind);
358 if (isRegister(input) && isRegister(result)) {
359 op = getVectorMoveOp(kind.getScalar());
360 } else {
361 op = getVectorMemMoveOp(kind);
362 }
363 } else {
364 size = AVXSize.XMM;
365 if (isRegister(input) && isRegister(result)) {
366 op = getVectorMoveOp(kind);
367 } else {
368 op = getScalarMoveOp(kind);
369 }
370 }
371
372 if (isRegister(input)) {
373 if (isRegister(result)) {
374 if (!asRegister(input).equals(asRegister(result))) {
375 op.emit(masm, size, asRegister(result), asRegister(input));
376 }
377 } else {
378 assert isStackSlot(result);
379 op.emit(masm, size, (AMD64Address) crb.asAddress(result), asRegister(input));
380 }
381 } else {
382 assert isStackSlot(input) && isRegister(result);
383 op.emit(masm, size, asRegister(result), (AMD64Address) crb.asAddress(input));
384 }
385 }
386
387 private static void const2reg(CompilationResultBuilder crb, AMD64MacroAssembler masm, RegisterValue result, JavaConstant input) {
388 if (input.isDefaultForKind()) {
389 AMD64Kind kind = (AMD64Kind) result.getPlatformKind();
390 Register register = result.getRegister();
391 VXORPD.emit(masm, AVXKind.getRegisterSize(kind), register, register, register);
392 return;
393 }
394
395 AMD64Address address;
396 switch (input.getJavaKind()) {
397 case Float:
398 address = (AMD64Address) crb.asFloatConstRef(input);
399 break;
400
401 case Double:
402 address = (AMD64Address) crb.asDoubleConstRef(input);
403 break;
404
405 default:
406 throw GraalError.shouldNotReachHere();
407 }
408 VexMoveOp op = getScalarMoveOp((AMD64Kind) result.getPlatformKind());
409 op.emit(masm, AVXSize.XMM, asRegister(result), address);
410 }
411
412 public static final class AVXMoveToIntOp extends AMD64LIRInstruction {
413 public static final LIRInstructionClass<AVXMoveToIntOp> TYPE = LIRInstructionClass.create(AVXMoveToIntOp.class);
414
415 @Opcode private final VexMoveOp opcode;
416
417 @Def({REG, STACK}) protected AllocatableValue result;
418 @Use({REG}) protected AllocatableValue input;
419
420 public AVXMoveToIntOp(VexMoveOp opcode, AllocatableValue result, AllocatableValue input) {
421 super(TYPE);
422 this.opcode = opcode;
423 this.result = result;
424 this.input = input;
425 }
426
427 @Override
428 public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) {
429 if (isRegister(result)) {
430 opcode.emitReverse(masm, AVXSize.XMM, asRegister(result), asRegister(input));
431 } else {
432 opcode.emit(masm, AVXSize.XMM, (AMD64Address) crb.asAddress(result), asRegister(input));
433 }
434 }
435 }
436 }