1 /*
2 * Copyright (c) 2013, 2018, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23
24
25 package org.graalvm.compiler.core.aarch64;
26
27 import static org.graalvm.compiler.lir.LIRValueUtil.asJavaConstant;
28 import static org.graalvm.compiler.lir.LIRValueUtil.isJavaConstant;
29
30 import java.util.function.Function;
31
32 import org.graalvm.compiler.asm.aarch64.AArch64Address.AddressingMode;
33 import org.graalvm.compiler.asm.aarch64.AArch64Assembler.ConditionFlag;
34 import org.graalvm.compiler.asm.aarch64.AArch64MacroAssembler;
35 import org.graalvm.compiler.core.common.LIRKind;
36 import org.graalvm.compiler.core.common.calc.Condition;
37 import org.graalvm.compiler.core.common.spi.LIRKindTool;
38 import org.graalvm.compiler.debug.GraalError;
39 import org.graalvm.compiler.lir.LIRFrameState;
40 import org.graalvm.compiler.lir.LIRValueUtil;
41 import org.graalvm.compiler.lir.LabelRef;
42 import org.graalvm.compiler.lir.StandardOp;
43 import org.graalvm.compiler.lir.SwitchStrategy;
44 import org.graalvm.compiler.lir.Variable;
45 import org.graalvm.compiler.lir.aarch64.AArch64AddressValue;
46 import org.graalvm.compiler.lir.aarch64.AArch64ArithmeticOp;
47 import org.graalvm.compiler.lir.aarch64.AArch64ArrayCompareToOp;
48 import org.graalvm.compiler.lir.aarch64.AArch64ArrayEqualsOp;
49 import org.graalvm.compiler.lir.aarch64.AArch64ByteSwapOp;
50 import org.graalvm.compiler.lir.aarch64.AArch64Compare;
51 import org.graalvm.compiler.lir.aarch64.AArch64ControlFlow;
52 import org.graalvm.compiler.lir.aarch64.AArch64ControlFlow.BranchOp;
53 import org.graalvm.compiler.lir.aarch64.AArch64ControlFlow.CondMoveOp;
54 import org.graalvm.compiler.lir.aarch64.AArch64ControlFlow.StrategySwitchOp;
55 import org.graalvm.compiler.lir.aarch64.AArch64ControlFlow.TableSwitchOp;
56 import org.graalvm.compiler.lir.aarch64.AArch64LIRFlagsVersioned;
57 import org.graalvm.compiler.lir.aarch64.AArch64Move;
58 import org.graalvm.compiler.lir.aarch64.AArch64AtomicMove.AtomicReadAndAddOp;
59 import org.graalvm.compiler.lir.aarch64.AArch64AtomicMove.AtomicReadAndAddLSEOp;
60 import org.graalvm.compiler.lir.aarch64.AArch64AtomicMove.CompareAndSwapOp;
61 import org.graalvm.compiler.lir.aarch64.AArch64AtomicMove.AtomicReadAndWriteOp;
62 import org.graalvm.compiler.lir.aarch64.AArch64Move.MembarOp;
63 import org.graalvm.compiler.lir.aarch64.AArch64PauseOp;
64 import org.graalvm.compiler.lir.gen.LIRGenerationResult;
65 import org.graalvm.compiler.lir.gen.LIRGenerator;
66 import org.graalvm.compiler.phases.util.Providers;
67
68 import jdk.vm.ci.aarch64.AArch64;
69 import jdk.vm.ci.aarch64.AArch64Kind;
70 import jdk.vm.ci.code.CallingConvention;
71 import jdk.vm.ci.code.RegisterValue;
72 import jdk.vm.ci.meta.AllocatableValue;
73 import jdk.vm.ci.meta.JavaConstant;
74 import jdk.vm.ci.meta.JavaKind;
75 import jdk.vm.ci.meta.PlatformKind;
76 import jdk.vm.ci.meta.PrimitiveConstant;
77 import jdk.vm.ci.meta.Value;
78 import jdk.vm.ci.meta.ValueKind;
79
80 public abstract class AArch64LIRGenerator extends LIRGenerator {
81
82 public AArch64LIRGenerator(LIRKindTool lirKindTool, AArch64ArithmeticLIRGenerator arithmeticLIRGen, MoveFactory moveFactory, Providers providers, LIRGenerationResult lirGenRes) {
83 super(lirKindTool, arithmeticLIRGen, moveFactory, providers, lirGenRes);
84 }
85
86 /**
87 * Checks whether the supplied constant can be used without loading it into a register for store
88 * operations, i.e., on the right hand side of a memory access.
89 *
90 * @param c The constant to check.
91 * @return True if the constant can be used directly, false if the constant needs to be in a
92 * register.
93 */
94 protected static final boolean canStoreConstant(JavaConstant c) {
95 // Our own code never calls this since we can't make a definite statement about whether or
96 // not we can inline a constant without knowing what kind of operation we execute. Let's be
97 // optimistic here and fix up mistakes later.
98 return true;
99 }
100
101 /**
102 * AArch64 cannot use anything smaller than a word in any instruction other than load and store.
103 */
104 @Override
105 public <K extends ValueKind<K>> K toRegisterKind(K kind) {
106 switch ((AArch64Kind) kind.getPlatformKind()) {
107 case BYTE:
108 case WORD:
109 return kind.changeType(AArch64Kind.DWORD);
110 default:
111 return kind;
112 }
113 }
114
115 @Override
116 public void emitNullCheck(Value address, LIRFrameState state) {
117 append(new AArch64Move.NullCheckOp(asAddressValue(address), state));
118 }
119
120 @Override
121 public Variable emitAddress(AllocatableValue stackslot) {
122 Variable result = newVariable(LIRKind.value(target().arch.getWordKind()));
123 append(new AArch64Move.StackLoadAddressOp(result, stackslot));
124 return result;
125 }
126
127 public AArch64AddressValue asAddressValue(Value address) {
128 if (address instanceof AArch64AddressValue) {
129 return (AArch64AddressValue) address;
130 } else {
131 return new AArch64AddressValue(address.getValueKind(), asAllocatable(address), Value.ILLEGAL, 0, 1, AddressingMode.BASE_REGISTER_ONLY);
132 }
133 }
134
135 @Override
136 public Variable emitLogicCompareAndSwap(LIRKind accessKind, Value address, Value expectedValue, Value newValue, Value trueValue, Value falseValue) {
137 Variable prevValue = newVariable(expectedValue.getValueKind());
138 Variable scratch = newVariable(LIRKind.value(AArch64Kind.DWORD));
139 append(new CompareAndSwapOp(prevValue, loadReg(expectedValue), loadReg(newValue), asAllocatable(address), scratch));
140 assert trueValue.getValueKind().equals(falseValue.getValueKind());
141 Variable result = newVariable(trueValue.getValueKind());
142 append(new CondMoveOp(result, ConditionFlag.EQ, asAllocatable(trueValue), asAllocatable(falseValue)));
143 return result;
144 }
145
146 @Override
147 public Variable emitValueCompareAndSwap(LIRKind accessKind, Value address, Value expectedValue, Value newValue) {
148 Variable result = newVariable(newValue.getValueKind());
149 Variable scratch = newVariable(LIRKind.value(AArch64Kind.WORD));
150 append(new CompareAndSwapOp(result, loadNonCompareConst(expectedValue), loadReg(newValue), asAllocatable(address), scratch));
151 return result;
152 }
153
154 @Override
155 public Value emitAtomicReadAndWrite(Value address, ValueKind<?> kind, Value newValue) {
156 Variable result = newVariable(kind);
157 Variable scratch = newVariable(kind);
158 append(new AtomicReadAndWriteOp((AArch64Kind) kind.getPlatformKind(), asAllocatable(result), asAllocatable(address), asAllocatable(newValue), asAllocatable(scratch)));
159 return result;
160 }
161
162 @Override
163 public Value emitAtomicReadAndAdd(Value address, ValueKind<?> kind, Value delta) {
164 Variable result = newVariable(kind);
165 if (AArch64LIRFlagsVersioned.useLSE(target().arch)) {
166 append(new AtomicReadAndAddLSEOp((AArch64Kind) kind.getPlatformKind(), asAllocatable(result), asAllocatable(address), asAllocatable(delta)));
167 } else {
168 append(new AtomicReadAndAddOp((AArch64Kind) kind.getPlatformKind(), asAllocatable(result), asAllocatable(address), delta));
169 }
170 return result;
171 }
172
173 @Override
174 public void emitMembar(int barriers) {
175 int necessaryBarriers = target().arch.requiredBarriers(barriers);
176 if (target().isMP && necessaryBarriers != 0) {
177 append(new MembarOp(necessaryBarriers));
178 }
179 }
180
181 @Override
182 public void emitJump(LabelRef label) {
183 assert label != null;
184 append(new StandardOp.JumpOp(label));
185 }
186
187 @Override
188 public void emitOverflowCheckBranch(LabelRef overflow, LabelRef noOverflow, LIRKind cmpKind, double overflowProbability) {
189 append(new AArch64ControlFlow.BranchOp(ConditionFlag.VS, overflow, noOverflow, overflowProbability));
190 }
191
192 /**
193 * Branches to label if (left & right) == 0. If negated is true branchse on non-zero instead.
194 *
195 * @param left Integer kind. Non null.
196 * @param right Integer kind. Non null.
197 * @param trueDestination destination if left & right == 0. Non null.
198 * @param falseDestination destination if left & right != 0. Non null
199 * @param trueSuccessorProbability hoistoric probability that comparison is true
200 */
201 @Override
202 public void emitIntegerTestBranch(Value left, Value right, LabelRef trueDestination, LabelRef falseDestination, double trueSuccessorProbability) {
203 assert ((AArch64Kind) left.getPlatformKind()).isInteger() && left.getPlatformKind() == right.getPlatformKind();
204 ((AArch64ArithmeticLIRGenerator) getArithmetic()).emitBinary(LIRKind.combine(left, right), AArch64ArithmeticOp.ANDS, true, left, right);
205 append(new AArch64ControlFlow.BranchOp(ConditionFlag.EQ, trueDestination, falseDestination, trueSuccessorProbability));
206 }
207
208 /**
209 * Conditionally move trueValue into new variable if cond + unorderedIsTrue is true, else
210 * falseValue.
211 *
212 * @param left Arbitrary value. Has to have same type as right. Non null.
213 * @param right Arbitrary value. Has to have same type as left. Non null.
214 * @param cond condition that decides whether to move trueValue or falseValue into result. Non
215 * null.
216 * @param unorderedIsTrue defines whether floating-point comparisons consider unordered true or
217 * not. Ignored for integer comparisons.
218 * @param trueValue arbitrary value same type as falseValue. Non null.
219 * @param falseValue arbitrary value same type as trueValue. Non null.
220 * @return value containing trueValue if cond + unorderedIsTrue is true, else falseValue. Non
221 * null.
222 */
223 @Override
224 public Variable emitConditionalMove(PlatformKind cmpKind, Value left, Value right, Condition cond, boolean unorderedIsTrue, Value trueValue, Value falseValue) {
225 boolean mirrored = emitCompare(cmpKind, left, right, cond, unorderedIsTrue);
226 Condition finalCondition = mirrored ? cond.mirror() : cond;
227 boolean finalUnorderedIsTrue = mirrored ? !unorderedIsTrue : unorderedIsTrue;
228 ConditionFlag cmpCondition = toConditionFlag(((AArch64Kind) cmpKind).isInteger(), finalCondition, finalUnorderedIsTrue);
229 Variable result = newVariable(trueValue.getValueKind());
230 append(new CondMoveOp(result, cmpCondition, loadReg(trueValue), loadReg(falseValue)));
231 return result;
232 }
233
234 @Override
235 public void emitCompareBranch(PlatformKind cmpKind, Value left, Value right, Condition cond, boolean unorderedIsTrue, LabelRef trueDestination, LabelRef falseDestination,
236 double trueDestinationProbability) {
237 boolean mirrored = emitCompare(cmpKind, left, right, cond, unorderedIsTrue);
238 Condition finalCondition = mirrored ? cond.mirror() : cond;
239 boolean finalUnorderedIsTrue = mirrored ? !unorderedIsTrue : unorderedIsTrue;
240 ConditionFlag cmpCondition = toConditionFlag(((AArch64Kind) cmpKind).isInteger(), finalCondition, finalUnorderedIsTrue);
241 append(new BranchOp(cmpCondition, trueDestination, falseDestination, trueDestinationProbability));
242 }
243
244 private static ConditionFlag toConditionFlag(boolean isInt, Condition cond, boolean unorderedIsTrue) {
245 return isInt ? toIntConditionFlag(cond) : toFloatConditionFlag(cond, unorderedIsTrue);
246 }
247
248 /**
249 * Takes a Condition and unorderedIsTrue flag and returns the correct Aarch64 specific
250 * ConditionFlag. Note: This is only correct if the emitCompare code for floats has correctly
251 * handled the case of 'EQ && unorderedIsTrue', respectively 'NE && !unorderedIsTrue'!
252 */
253 private static ConditionFlag toFloatConditionFlag(Condition cond, boolean unorderedIsTrue) {
254 switch (cond) {
255 case LT:
256 return unorderedIsTrue ? ConditionFlag.LT : ConditionFlag.LO;
257 case LE:
258 return unorderedIsTrue ? ConditionFlag.LE : ConditionFlag.LS;
259 case GE:
260 return unorderedIsTrue ? ConditionFlag.PL : ConditionFlag.GE;
261 case GT:
262 return unorderedIsTrue ? ConditionFlag.HI : ConditionFlag.GT;
263 case EQ:
264 return ConditionFlag.EQ;
265 case NE:
266 return ConditionFlag.NE;
267 default:
268 throw GraalError.shouldNotReachHere();
269 }
270 }
271
272 /**
273 * Takes a Condition and returns the correct Aarch64 specific ConditionFlag.
274 */
275 private static ConditionFlag toIntConditionFlag(Condition cond) {
276 switch (cond) {
277 case EQ:
278 return ConditionFlag.EQ;
279 case NE:
280 return ConditionFlag.NE;
281 case LT:
282 return ConditionFlag.LT;
283 case LE:
284 return ConditionFlag.LE;
285 case GT:
286 return ConditionFlag.GT;
287 case GE:
288 return ConditionFlag.GE;
289 case AE:
290 return ConditionFlag.HS;
291 case BE:
292 return ConditionFlag.LS;
293 case AT:
294 return ConditionFlag.HI;
295 case BT:
296 return ConditionFlag.LO;
297 default:
298 throw GraalError.shouldNotReachHere();
299 }
300 }
301
302 /**
303 * This method emits the compare instruction, and may reorder the operands. It returns true if
304 * it did so.
305 *
306 * @param a the left operand of the comparison. Has to have same type as b. Non null.
307 * @param b the right operand of the comparison. Has to have same type as a. Non null.
308 * @return true if mirrored (i.e. "b cmp a" instead of "a cmp b" was done).
309 */
310 protected boolean emitCompare(PlatformKind cmpKind, Value a, Value b, Condition condition, boolean unorderedIsTrue) {
311 Value left;
312 Value right;
313 boolean mirrored;
314 AArch64Kind kind = (AArch64Kind) cmpKind;
315 if (kind.isInteger()) {
316 Value aExt = a;
317 Value bExt = b;
318
319 int compareBytes = cmpKind.getSizeInBytes();
320 // AArch64 compares 32 or 64 bits: sign extend a and b as required.
321 if (compareBytes < a.getPlatformKind().getSizeInBytes()) {
322 aExt = arithmeticLIRGen.emitSignExtend(a, compareBytes * 8, 64);
323 }
324 if (compareBytes < b.getPlatformKind().getSizeInBytes()) {
325 bExt = arithmeticLIRGen.emitSignExtend(b, compareBytes * 8, 64);
326 }
327
328 if (LIRValueUtil.isVariable(bExt)) {
329 left = load(bExt);
330 right = loadNonConst(aExt);
331 mirrored = true;
332 } else {
333 left = load(aExt);
334 right = loadNonConst(bExt);
335 mirrored = false;
336 }
337 append(new AArch64Compare.CompareOp(left, loadNonCompareConst(right)));
338 } else if (kind.isSIMD()) {
339 if (AArch64Compare.FloatCompareOp.isFloatCmpConstant(a, condition, unorderedIsTrue)) {
340 left = load(b);
341 right = a;
342 mirrored = true;
343 } else if (AArch64Compare.FloatCompareOp.isFloatCmpConstant(b, condition, unorderedIsTrue)) {
344 left = load(a);
345 right = b;
346 mirrored = false;
347 } else {
348 left = load(a);
349 right = loadReg(b);
350 mirrored = false;
351 }
352 append(new AArch64Compare.FloatCompareOp(left, asAllocatable(right), condition, unorderedIsTrue));
353 } else {
354 throw GraalError.shouldNotReachHere();
355 }
356 return mirrored;
357 }
358
359 /**
360 * If value is a constant that cannot be used directly with a gpCompare instruction load it into
361 * a register and return the register, otherwise return constant value unchanged.
362 */
363 protected Value loadNonCompareConst(Value value) {
364 if (!isCompareConstant(value)) {
365 return loadReg(value);
366 }
367 return value;
368 }
369
370 /**
371 * Checks whether value can be used directly with a gpCompare instruction. This is <b>not</b>
372 * the same as {@link AArch64ArithmeticLIRGenerator#isArithmeticConstant(JavaConstant)}, because
373 * 0.0 is a valid compare constant for floats, while there are no arithmetic constants for
374 * floats.
375 *
376 * @param value any type. Non null.
377 * @return true if value can be used directly in comparison instruction, false otherwise.
378 */
379 public boolean isCompareConstant(Value value) {
380 if (isJavaConstant(value)) {
381 JavaConstant constant = asJavaConstant(value);
382 if (constant instanceof PrimitiveConstant) {
383 final long longValue = constant.asLong();
384 long maskedValue;
385 switch (constant.getJavaKind()) {
386 case Boolean:
387 case Byte:
388 maskedValue = longValue & 0xFF;
389 break;
390 case Char:
391 case Short:
392 maskedValue = longValue & 0xFFFF;
393 break;
394 case Int:
395 maskedValue = longValue & 0xFFFF_FFFF;
396 break;
397 case Long:
398 maskedValue = longValue;
399 break;
400 default:
401 throw GraalError.shouldNotReachHere();
402 }
403 return AArch64MacroAssembler.isArithmeticImmediate(maskedValue);
404 } else {
405 return constant.isDefaultForKind();
406 }
407 }
408 return false;
409 }
410
411 /**
412 * Moves trueValue into result if (left & right) == 0, else falseValue.
413 *
414 * @param left Integer kind. Non null.
415 * @param right Integer kind. Non null.
416 * @param trueValue Integer kind. Non null.
417 * @param falseValue Integer kind. Non null.
418 * @return virtual register containing trueValue if (left & right) == 0, else falseValue.
419 */
420 @Override
421 public Variable emitIntegerTestMove(Value left, Value right, Value trueValue, Value falseValue) {
422 assert ((AArch64Kind) left.getPlatformKind()).isInteger() && ((AArch64Kind) right.getPlatformKind()).isInteger();
423 assert ((AArch64Kind) trueValue.getPlatformKind()).isInteger() && ((AArch64Kind) falseValue.getPlatformKind()).isInteger();
424 ((AArch64ArithmeticLIRGenerator) getArithmetic()).emitBinary(left.getValueKind(), AArch64ArithmeticOp.ANDS, true, left, right);
425 Variable result = newVariable(trueValue.getValueKind());
426 append(new CondMoveOp(result, ConditionFlag.EQ, load(trueValue), load(falseValue)));
427 return result;
428 }
429
430 @Override
431 public void emitStrategySwitch(SwitchStrategy strategy, Variable key, LabelRef[] keyTargets, LabelRef defaultTarget) {
432 append(createStrategySwitchOp(strategy, keyTargets, defaultTarget, key, newVariable(key.getValueKind()), AArch64LIRGenerator::toIntConditionFlag));
433 }
434
435 protected StrategySwitchOp createStrategySwitchOp(SwitchStrategy strategy, LabelRef[] keyTargets, LabelRef defaultTarget, Variable key, AllocatableValue scratchValue,
436 Function<Condition, ConditionFlag> converter) {
437 return new StrategySwitchOp(strategy, keyTargets, defaultTarget, key, scratchValue, converter);
438 }
439
440 @Override
441 protected void emitTableSwitch(int lowKey, LabelRef defaultTarget, LabelRef[] targets, Value key) {
442 append(new TableSwitchOp(lowKey, defaultTarget, targets, key, newVariable(LIRKind.value(target().arch.getWordKind())), newVariable(key.getValueKind())));
443 }
444
445 @Override
446 public Variable emitByteSwap(Value input) {
447 Variable result = newVariable(LIRKind.combine(input));
448 append(new AArch64ByteSwapOp(result, input));
449 return result;
450 }
451
452 @Override
453 public Variable emitArrayCompareTo(JavaKind kind1, JavaKind kind2, Value array1, Value array2, Value length1, Value length2) {
454 LIRKind resultKind = LIRKind.value(AArch64Kind.DWORD);
455 // DMS TODO: check calling conversion and registers used
456 RegisterValue res = AArch64.r0.asValue(resultKind);
457 RegisterValue cnt1 = AArch64.r1.asValue(length1.getValueKind());
458 RegisterValue cnt2 = AArch64.r2.asValue(length2.getValueKind());
459 emitMove(cnt1, length1);
460 emitMove(cnt2, length2);
461 append(new AArch64ArrayCompareToOp(this, kind1, kind2, res, array1, array2, cnt1, cnt2));
462 Variable result = newVariable(resultKind);
463 emitMove(result, res);
464 return result;
465 }
466
467 @Override
468 public Variable emitArrayEquals(JavaKind kind, Value array1, Value array2, Value length) {
469 Variable result = newVariable(LIRKind.value(AArch64Kind.DWORD));
470 append(new AArch64ArrayEqualsOp(this, kind, result, array1, array2, asAllocatable(length)));
471 return result;
472 }
473
474 @Override
475 protected JavaConstant zapValueForKind(PlatformKind kind) {
476 long dead = 0xDEADDEADDEADDEADL;
477 switch ((AArch64Kind) kind) {
478 case BYTE:
479 return JavaConstant.forByte((byte) dead);
480 case WORD:
481 return JavaConstant.forShort((short) dead);
482 case DWORD:
483 return JavaConstant.forInt((int) dead);
484 case QWORD:
485 return JavaConstant.forLong(dead);
486 case SINGLE:
487 return JavaConstant.forFloat(Float.intBitsToFloat((int) dead));
488 case DOUBLE:
489 return JavaConstant.forDouble(Double.longBitsToDouble(dead));
490 default:
491 throw GraalError.shouldNotReachHere();
492 }
493 }
494
495 /**
496 * Loads value into virtual register. Contrary to {@link #load(Value)} this handles
497 * RegisterValues (i.e. values corresponding to fixed physical registers) correctly, by not
498 * creating an unnecessary move into a virtual register.
499 *
500 * This avoids generating the following code: mov x0, x19 # x19 is fixed thread register ldr x0,
501 * [x0] instead of: ldr x0, [x19].
502 */
503 protected AllocatableValue loadReg(Value val) {
504 if (!(val instanceof Variable || val instanceof RegisterValue)) {
505 return emitMove(val);
506 }
507 return (AllocatableValue) val;
508 }
509
510 @Override
511 public void emitPause() {
512 append(new AArch64PauseOp());
513 }
514
515 public abstract void emitCCall(long address, CallingConvention nativeCallingConvention, Value[] args);
516 }