1 /*
2 * Copyright (c) 2013, 2018, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23
24
25 package org.graalvm.compiler.core.aarch64;
26
27 import static org.graalvm.compiler.lir.LIRValueUtil.asJavaConstant;
28 import static org.graalvm.compiler.lir.LIRValueUtil.isJavaConstant;
29
30 import java.util.function.Function;
31
32 import org.graalvm.compiler.asm.aarch64.AArch64Address.AddressingMode;
33 import org.graalvm.compiler.asm.aarch64.AArch64Assembler.ConditionFlag;
34 import org.graalvm.compiler.asm.aarch64.AArch64MacroAssembler;
35 import org.graalvm.compiler.core.common.LIRKind;
36 import org.graalvm.compiler.core.common.calc.Condition;
37 import org.graalvm.compiler.core.common.spi.LIRKindTool;
38 import org.graalvm.compiler.debug.GraalError;
39 import org.graalvm.compiler.lir.LIRFrameState;
40 import org.graalvm.compiler.lir.LIRValueUtil;
41 import org.graalvm.compiler.lir.LabelRef;
42 import org.graalvm.compiler.lir.StandardOp;
43 import org.graalvm.compiler.lir.SwitchStrategy;
44 import org.graalvm.compiler.lir.Variable;
45 import org.graalvm.compiler.lir.aarch64.AArch64AddressValue;
46 import org.graalvm.compiler.lir.aarch64.AArch64ArithmeticOp;
47 import org.graalvm.compiler.lir.aarch64.AArch64ArrayCompareToOp;
48 import org.graalvm.compiler.lir.aarch64.AArch64ArrayEqualsOp;
49 import org.graalvm.compiler.lir.aarch64.AArch64ByteSwapOp;
50 import org.graalvm.compiler.lir.aarch64.AArch64Compare;
51 import org.graalvm.compiler.lir.aarch64.AArch64ControlFlow;
52 import org.graalvm.compiler.lir.aarch64.AArch64ControlFlow.BranchOp;
53 import org.graalvm.compiler.lir.aarch64.AArch64ControlFlow.CondMoveOp;
54 import org.graalvm.compiler.lir.aarch64.AArch64ControlFlow.StrategySwitchOp;
55 import org.graalvm.compiler.lir.aarch64.AArch64ControlFlow.TableSwitchOp;
56 import org.graalvm.compiler.lir.aarch64.AArch64LIRFlagsVersioned;
57 import org.graalvm.compiler.lir.aarch64.AArch64Move;
58 import org.graalvm.compiler.lir.aarch64.AArch64AtomicMove.AtomicReadAndAddOp;
59 import org.graalvm.compiler.lir.aarch64.AArch64AtomicMove.AtomicReadAndAddLSEOp;
60 import org.graalvm.compiler.lir.aarch64.AArch64AtomicMove.CompareAndSwapOp;
61 import org.graalvm.compiler.lir.aarch64.AArch64AtomicMove.AtomicReadAndWriteOp;
62 import org.graalvm.compiler.lir.aarch64.AArch64Move.MembarOp;
63 import org.graalvm.compiler.lir.aarch64.AArch64PauseOp;
64 import org.graalvm.compiler.lir.aarch64.AArch64SpeculativeBarrier;
65 import org.graalvm.compiler.lir.gen.LIRGenerationResult;
66 import org.graalvm.compiler.lir.gen.LIRGenerator;
67 import org.graalvm.compiler.phases.util.Providers;
68
69 import jdk.vm.ci.aarch64.AArch64;
70 import jdk.vm.ci.aarch64.AArch64Kind;
71 import jdk.vm.ci.code.CallingConvention;
72 import jdk.vm.ci.code.RegisterValue;
73 import jdk.vm.ci.meta.AllocatableValue;
74 import jdk.vm.ci.meta.JavaConstant;
75 import jdk.vm.ci.meta.JavaKind;
76 import jdk.vm.ci.meta.PlatformKind;
77 import jdk.vm.ci.meta.PrimitiveConstant;
78 import jdk.vm.ci.meta.Value;
79 import jdk.vm.ci.meta.ValueKind;
80
81 public abstract class AArch64LIRGenerator extends LIRGenerator {
82
83 public AArch64LIRGenerator(LIRKindTool lirKindTool, AArch64ArithmeticLIRGenerator arithmeticLIRGen, MoveFactory moveFactory, Providers providers, LIRGenerationResult lirGenRes) {
84 super(lirKindTool, arithmeticLIRGen, moveFactory, providers, lirGenRes);
85 }
86
87 /**
88 * Checks whether the supplied constant can be used without loading it into a register for store
89 * operations, i.e., on the right hand side of a memory access.
90 *
91 * @param c The constant to check.
92 * @return True if the constant can be used directly, false if the constant needs to be in a
93 * register.
94 */
95 protected static final boolean canStoreConstant(JavaConstant c) {
96 // Our own code never calls this since we can't make a definite statement about whether or
97 // not we can inline a constant without knowing what kind of operation we execute. Let's be
98 // optimistic here and fix up mistakes later.
99 return true;
100 }
101
102 /**
103 * AArch64 cannot use anything smaller than a word in any instruction other than load and store.
104 */
105 @Override
106 public <K extends ValueKind<K>> K toRegisterKind(K kind) {
107 switch ((AArch64Kind) kind.getPlatformKind()) {
108 case BYTE:
109 case WORD:
110 return kind.changeType(AArch64Kind.DWORD);
111 default:
112 return kind;
113 }
114 }
115
116 @Override
117 public void emitNullCheck(Value address, LIRFrameState state) {
118 append(new AArch64Move.NullCheckOp(asAddressValue(address), state));
119 }
120
121 @Override
122 public Variable emitAddress(AllocatableValue stackslot) {
123 Variable result = newVariable(LIRKind.value(target().arch.getWordKind()));
124 append(new AArch64Move.StackLoadAddressOp(result, stackslot));
125 return result;
126 }
127
128 public AArch64AddressValue asAddressValue(Value address) {
129 if (address instanceof AArch64AddressValue) {
130 return (AArch64AddressValue) address;
131 } else {
132 return new AArch64AddressValue(address.getValueKind(), asAllocatable(address), Value.ILLEGAL, 0, 1, AddressingMode.BASE_REGISTER_ONLY);
133 }
134 }
135
136 @Override
137 public Variable emitLogicCompareAndSwap(LIRKind accessKind, Value address, Value expectedValue, Value newValue, Value trueValue, Value falseValue) {
138 Variable prevValue = newVariable(expectedValue.getValueKind());
139 Variable scratch = newVariable(LIRKind.value(AArch64Kind.DWORD));
140 append(new CompareAndSwapOp(prevValue, loadReg(expectedValue), loadReg(newValue), asAllocatable(address), scratch));
141 assert trueValue.getValueKind().equals(falseValue.getValueKind());
142 Variable result = newVariable(trueValue.getValueKind());
143 append(new CondMoveOp(result, ConditionFlag.EQ, asAllocatable(trueValue), asAllocatable(falseValue)));
144 return result;
145 }
146
147 @Override
148 public Variable emitValueCompareAndSwap(LIRKind accessKind, Value address, Value expectedValue, Value newValue) {
149 Variable result = newVariable(newValue.getValueKind());
150 Variable scratch = newVariable(LIRKind.value(AArch64Kind.WORD));
151 append(new CompareAndSwapOp(result, loadNonCompareConst(expectedValue), loadReg(newValue), asAllocatable(address), scratch));
152 return result;
153 }
154
155 @Override
156 public Value emitAtomicReadAndWrite(Value address, ValueKind<?> kind, Value newValue) {
157 Variable result = newVariable(kind);
158 Variable scratch = newVariable(kind);
159 append(new AtomicReadAndWriteOp((AArch64Kind) kind.getPlatformKind(), asAllocatable(result), asAllocatable(address), asAllocatable(newValue), asAllocatable(scratch)));
160 return result;
161 }
162
163 @Override
164 public Value emitAtomicReadAndAdd(Value address, ValueKind<?> kind, Value delta) {
165 Variable result = newVariable(kind);
166 if (AArch64LIRFlagsVersioned.useLSE(target().arch)) {
167 append(new AtomicReadAndAddLSEOp((AArch64Kind) kind.getPlatformKind(), asAllocatable(result), asAllocatable(address), asAllocatable(delta)));
168 } else {
169 append(new AtomicReadAndAddOp((AArch64Kind) kind.getPlatformKind(), asAllocatable(result), asAllocatable(address), delta));
170 }
171 return result;
172 }
173
174 @Override
175 public void emitMembar(int barriers) {
176 int necessaryBarriers = target().arch.requiredBarriers(barriers);
177 if (target().isMP && necessaryBarriers != 0) {
178 append(new MembarOp(necessaryBarriers));
179 }
180 }
181
182 @Override
183 public void emitJump(LabelRef label) {
184 assert label != null;
185 append(new StandardOp.JumpOp(label));
186 }
187
188 @Override
189 public void emitOverflowCheckBranch(LabelRef overflow, LabelRef noOverflow, LIRKind cmpKind, double overflowProbability) {
190 append(new AArch64ControlFlow.BranchOp(ConditionFlag.VS, overflow, noOverflow, overflowProbability));
191 }
192
193 /**
194 * Branches to label if (left & right) == 0. If negated is true branchse on non-zero instead.
195 *
196 * @param left Integer kind. Non null.
197 * @param right Integer kind. Non null.
198 * @param trueDestination destination if left & right == 0. Non null.
199 * @param falseDestination destination if left & right != 0. Non null
200 * @param trueSuccessorProbability hoistoric probability that comparison is true
201 */
202 @Override
203 public void emitIntegerTestBranch(Value left, Value right, LabelRef trueDestination, LabelRef falseDestination, double trueSuccessorProbability) {
204 assert ((AArch64Kind) left.getPlatformKind()).isInteger() && left.getPlatformKind() == right.getPlatformKind();
205 ((AArch64ArithmeticLIRGenerator) getArithmetic()).emitBinary(LIRKind.combine(left, right), AArch64ArithmeticOp.ANDS, true, left, right);
206 append(new AArch64ControlFlow.BranchOp(ConditionFlag.EQ, trueDestination, falseDestination, trueSuccessorProbability));
207 }
208
209 /**
210 * Conditionally move trueValue into new variable if cond + unorderedIsTrue is true, else
211 * falseValue.
212 *
213 * @param left Arbitrary value. Has to have same type as right. Non null.
214 * @param right Arbitrary value. Has to have same type as left. Non null.
215 * @param cond condition that decides whether to move trueValue or falseValue into result. Non
216 * null.
217 * @param unorderedIsTrue defines whether floating-point comparisons consider unordered true or
218 * not. Ignored for integer comparisons.
219 * @param trueValue arbitrary value same type as falseValue. Non null.
220 * @param falseValue arbitrary value same type as trueValue. Non null.
221 * @return value containing trueValue if cond + unorderedIsTrue is true, else falseValue. Non
222 * null.
223 */
224 @Override
225 public Variable emitConditionalMove(PlatformKind cmpKind, Value left, Value right, Condition cond, boolean unorderedIsTrue, Value trueValue, Value falseValue) {
226 boolean mirrored = emitCompare(cmpKind, left, right, cond, unorderedIsTrue);
227 Condition finalCondition = mirrored ? cond.mirror() : cond;
228 boolean finalUnorderedIsTrue = mirrored ? !unorderedIsTrue : unorderedIsTrue;
229 ConditionFlag cmpCondition = toConditionFlag(((AArch64Kind) cmpKind).isInteger(), finalCondition, finalUnorderedIsTrue);
230 Variable result = newVariable(trueValue.getValueKind());
231 append(new CondMoveOp(result, cmpCondition, loadReg(trueValue), loadReg(falseValue)));
232 return result;
233 }
234
235 @Override
236 public void emitCompareBranch(PlatformKind cmpKind, Value left, Value right, Condition cond, boolean unorderedIsTrue, LabelRef trueDestination, LabelRef falseDestination,
237 double trueDestinationProbability) {
238 boolean mirrored = emitCompare(cmpKind, left, right, cond, unorderedIsTrue);
239 Condition finalCondition = mirrored ? cond.mirror() : cond;
240 boolean finalUnorderedIsTrue = mirrored ? !unorderedIsTrue : unorderedIsTrue;
241 ConditionFlag cmpCondition = toConditionFlag(((AArch64Kind) cmpKind).isInteger(), finalCondition, finalUnorderedIsTrue);
242 append(new BranchOp(cmpCondition, trueDestination, falseDestination, trueDestinationProbability));
243 }
244
245 private static ConditionFlag toConditionFlag(boolean isInt, Condition cond, boolean unorderedIsTrue) {
246 return isInt ? toIntConditionFlag(cond) : toFloatConditionFlag(cond, unorderedIsTrue);
247 }
248
249 /**
250 * Takes a Condition and unorderedIsTrue flag and returns the correct Aarch64 specific
251 * ConditionFlag. Note: This is only correct if the emitCompare code for floats has correctly
252 * handled the case of 'EQ && unorderedIsTrue', respectively 'NE && !unorderedIsTrue'!
253 */
254 private static ConditionFlag toFloatConditionFlag(Condition cond, boolean unorderedIsTrue) {
255 switch (cond) {
256 case LT:
257 return unorderedIsTrue ? ConditionFlag.LT : ConditionFlag.LO;
258 case LE:
259 return unorderedIsTrue ? ConditionFlag.LE : ConditionFlag.LS;
260 case GE:
261 return unorderedIsTrue ? ConditionFlag.PL : ConditionFlag.GE;
262 case GT:
263 return unorderedIsTrue ? ConditionFlag.HI : ConditionFlag.GT;
264 case EQ:
265 return ConditionFlag.EQ;
266 case NE:
267 return ConditionFlag.NE;
268 default:
269 throw GraalError.shouldNotReachHere();
270 }
271 }
272
273 /**
274 * Takes a Condition and returns the correct Aarch64 specific ConditionFlag.
275 */
276 private static ConditionFlag toIntConditionFlag(Condition cond) {
277 switch (cond) {
278 case EQ:
279 return ConditionFlag.EQ;
280 case NE:
281 return ConditionFlag.NE;
282 case LT:
283 return ConditionFlag.LT;
284 case LE:
285 return ConditionFlag.LE;
286 case GT:
287 return ConditionFlag.GT;
288 case GE:
289 return ConditionFlag.GE;
290 case AE:
291 return ConditionFlag.HS;
292 case BE:
293 return ConditionFlag.LS;
294 case AT:
295 return ConditionFlag.HI;
296 case BT:
297 return ConditionFlag.LO;
298 default:
299 throw GraalError.shouldNotReachHere();
300 }
301 }
302
303 /**
304 * This method emits the compare instruction, and may reorder the operands. It returns true if
305 * it did so.
306 *
307 * @param a the left operand of the comparison. Has to have same type as b. Non null.
308 * @param b the right operand of the comparison. Has to have same type as a. Non null.
309 * @return true if mirrored (i.e. "b cmp a" instead of "a cmp b" was done).
310 */
311 protected boolean emitCompare(PlatformKind cmpKind, Value a, Value b, Condition condition, boolean unorderedIsTrue) {
312 Value left;
313 Value right;
314 boolean mirrored;
315 AArch64Kind kind = (AArch64Kind) cmpKind;
316 if (kind.isInteger()) {
317 Value aExt = a;
318 Value bExt = b;
319
320 int compareBytes = cmpKind.getSizeInBytes();
321 // AArch64 compares 32 or 64 bits: sign extend a and b as required.
322 if (compareBytes < a.getPlatformKind().getSizeInBytes()) {
323 aExt = arithmeticLIRGen.emitSignExtend(a, compareBytes * 8, 64);
324 }
325 if (compareBytes < b.getPlatformKind().getSizeInBytes()) {
326 bExt = arithmeticLIRGen.emitSignExtend(b, compareBytes * 8, 64);
327 }
328
329 if (LIRValueUtil.isVariable(bExt)) {
330 left = load(bExt);
331 right = loadNonConst(aExt);
332 mirrored = true;
333 } else {
334 left = load(aExt);
335 right = loadNonConst(bExt);
336 mirrored = false;
337 }
338 append(new AArch64Compare.CompareOp(left, loadNonCompareConst(right)));
339 } else if (kind.isSIMD()) {
340 if (AArch64Compare.FloatCompareOp.isFloatCmpConstant(a, condition, unorderedIsTrue)) {
341 left = load(b);
342 right = a;
343 mirrored = true;
344 } else if (AArch64Compare.FloatCompareOp.isFloatCmpConstant(b, condition, unorderedIsTrue)) {
345 left = load(a);
346 right = b;
347 mirrored = false;
348 } else {
349 left = load(a);
350 right = loadReg(b);
351 mirrored = false;
352 }
353 append(new AArch64Compare.FloatCompareOp(left, asAllocatable(right), condition, unorderedIsTrue));
354 } else {
355 throw GraalError.shouldNotReachHere();
356 }
357 return mirrored;
358 }
359
360 /**
361 * If value is a constant that cannot be used directly with a gpCompare instruction load it into
362 * a register and return the register, otherwise return constant value unchanged.
363 */
364 protected Value loadNonCompareConst(Value value) {
365 if (!isCompareConstant(value)) {
366 return loadReg(value);
367 }
368 return value;
369 }
370
371 /**
372 * Checks whether value can be used directly with a gpCompare instruction. This is <b>not</b>
373 * the same as {@link AArch64ArithmeticLIRGenerator#isArithmeticConstant(JavaConstant)}, because
374 * 0.0 is a valid compare constant for floats, while there are no arithmetic constants for
375 * floats.
376 *
377 * @param value any type. Non null.
378 * @return true if value can be used directly in comparison instruction, false otherwise.
379 */
380 public boolean isCompareConstant(Value value) {
381 if (isJavaConstant(value)) {
382 JavaConstant constant = asJavaConstant(value);
383 if (constant instanceof PrimitiveConstant) {
384 final long longValue = constant.asLong();
385 long maskedValue;
386 switch (constant.getJavaKind()) {
387 case Boolean:
388 case Byte:
389 maskedValue = longValue & 0xFF;
390 break;
391 case Char:
392 case Short:
393 maskedValue = longValue & 0xFFFF;
394 break;
395 case Int:
396 maskedValue = longValue & 0xFFFF_FFFF;
397 break;
398 case Long:
399 maskedValue = longValue;
400 break;
401 default:
402 throw GraalError.shouldNotReachHere();
403 }
404 return AArch64MacroAssembler.isArithmeticImmediate(maskedValue);
405 } else {
406 return constant.isDefaultForKind();
407 }
408 }
409 return false;
410 }
411
412 /**
413 * Moves trueValue into result if (left & right) == 0, else falseValue.
414 *
415 * @param left Integer kind. Non null.
416 * @param right Integer kind. Non null.
417 * @param trueValue Integer kind. Non null.
418 * @param falseValue Integer kind. Non null.
419 * @return virtual register containing trueValue if (left & right) == 0, else falseValue.
420 */
421 @Override
422 public Variable emitIntegerTestMove(Value left, Value right, Value trueValue, Value falseValue) {
423 assert ((AArch64Kind) left.getPlatformKind()).isInteger() && ((AArch64Kind) right.getPlatformKind()).isInteger();
424 assert ((AArch64Kind) trueValue.getPlatformKind()).isInteger() && ((AArch64Kind) falseValue.getPlatformKind()).isInteger();
425 ((AArch64ArithmeticLIRGenerator) getArithmetic()).emitBinary(left.getValueKind(), AArch64ArithmeticOp.ANDS, true, left, right);
426 Variable result = newVariable(trueValue.getValueKind());
427 append(new CondMoveOp(result, ConditionFlag.EQ, load(trueValue), load(falseValue)));
428 return result;
429 }
430
431 @Override
432 public void emitStrategySwitch(SwitchStrategy strategy, Variable key, LabelRef[] keyTargets, LabelRef defaultTarget) {
433 append(createStrategySwitchOp(strategy, keyTargets, defaultTarget, key, newVariable(key.getValueKind()), AArch64LIRGenerator::toIntConditionFlag));
434 }
435
436 protected StrategySwitchOp createStrategySwitchOp(SwitchStrategy strategy, LabelRef[] keyTargets, LabelRef defaultTarget, Variable key, AllocatableValue scratchValue,
437 Function<Condition, ConditionFlag> converter) {
438 return new StrategySwitchOp(strategy, keyTargets, defaultTarget, key, scratchValue, converter);
439 }
440
441 @Override
442 protected void emitTableSwitch(int lowKey, LabelRef defaultTarget, LabelRef[] targets, Value key) {
443 append(new TableSwitchOp(lowKey, defaultTarget, targets, key, newVariable(LIRKind.value(target().arch.getWordKind())), newVariable(key.getValueKind())));
444 }
445
446 @Override
447 public Variable emitByteSwap(Value input) {
448 Variable result = newVariable(LIRKind.combine(input));
449 append(new AArch64ByteSwapOp(result, input));
450 return result;
451 }
452
453 @Override
454 public Variable emitArrayCompareTo(JavaKind kind1, JavaKind kind2, Value array1, Value array2, Value length1, Value length2) {
455 LIRKind resultKind = LIRKind.value(AArch64Kind.DWORD);
456 // DMS TODO: check calling conversion and registers used
457 RegisterValue res = AArch64.r0.asValue(resultKind);
458 RegisterValue cnt1 = AArch64.r1.asValue(length1.getValueKind());
459 RegisterValue cnt2 = AArch64.r2.asValue(length2.getValueKind());
460 emitMove(cnt1, length1);
461 emitMove(cnt2, length2);
462 append(new AArch64ArrayCompareToOp(this, kind1, kind2, res, array1, array2, cnt1, cnt2));
463 Variable result = newVariable(resultKind);
464 emitMove(result, res);
465 return result;
466 }
467
468 @Override
469 public Variable emitArrayEquals(JavaKind kind, Value array1, Value array2, Value length, int constantLength, boolean directPointers) {
470 Variable result = newVariable(LIRKind.value(AArch64Kind.DWORD));
471 append(new AArch64ArrayEqualsOp(this, kind, result, array1, array2, asAllocatable(length), directPointers));
472 return result;
473 }
474
475 @Override
476 protected JavaConstant zapValueForKind(PlatformKind kind) {
477 long dead = 0xDEADDEADDEADDEADL;
478 switch ((AArch64Kind) kind) {
479 case BYTE:
480 return JavaConstant.forByte((byte) dead);
481 case WORD:
482 return JavaConstant.forShort((short) dead);
483 case DWORD:
484 return JavaConstant.forInt((int) dead);
485 case QWORD:
486 return JavaConstant.forLong(dead);
487 case SINGLE:
488 return JavaConstant.forFloat(Float.intBitsToFloat((int) dead));
489 case DOUBLE:
490 return JavaConstant.forDouble(Double.longBitsToDouble(dead));
491 default:
492 throw GraalError.shouldNotReachHere();
493 }
494 }
495
496 /**
497 * Loads value into virtual register. Contrary to {@link #load(Value)} this handles
498 * RegisterValues (i.e. values corresponding to fixed physical registers) correctly, by not
499 * creating an unnecessary move into a virtual register.
500 *
501 * This avoids generating the following code: mov x0, x19 # x19 is fixed thread register ldr x0,
502 * [x0] instead of: ldr x0, [x19].
503 */
504 protected AllocatableValue loadReg(Value val) {
505 if (!(val instanceof Variable || val instanceof RegisterValue)) {
506 return emitMove(val);
507 }
508 return (AllocatableValue) val;
509 }
510
511 @Override
512 public void emitPause() {
513 append(new AArch64PauseOp());
514 }
515
516 public abstract void emitCCall(long address, CallingConvention nativeCallingConvention, Value[] args);
517
518 @Override
519 public void emitSpeculationFence() {
520 append(new AArch64SpeculativeBarrier());
521 }
522 }