1 /* 2 * Copyright (c) 2013, 2018, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 */ 23 24 25 package org.graalvm.compiler.core.aarch64; 26 27 import static org.graalvm.compiler.lir.LIRValueUtil.asJavaConstant; 28 import static org.graalvm.compiler.lir.LIRValueUtil.isJavaConstant; 29 30 import java.util.function.Function; 31 32 import org.graalvm.compiler.asm.aarch64.AArch64Address.AddressingMode; 33 import org.graalvm.compiler.asm.aarch64.AArch64Assembler.ConditionFlag; 34 import org.graalvm.compiler.asm.aarch64.AArch64MacroAssembler; 35 import org.graalvm.compiler.core.common.LIRKind; 36 import org.graalvm.compiler.core.common.calc.Condition; 37 import org.graalvm.compiler.core.common.spi.LIRKindTool; 38 import org.graalvm.compiler.debug.GraalError; 39 import org.graalvm.compiler.lir.LIRFrameState; 40 import org.graalvm.compiler.lir.LIRValueUtil; 41 import org.graalvm.compiler.lir.LabelRef; 42 import org.graalvm.compiler.lir.StandardOp; 43 import org.graalvm.compiler.lir.SwitchStrategy; 44 import org.graalvm.compiler.lir.Variable; 45 import org.graalvm.compiler.lir.aarch64.AArch64AddressValue; 46 import org.graalvm.compiler.lir.aarch64.AArch64ArithmeticOp; 47 import org.graalvm.compiler.lir.aarch64.AArch64ArrayCompareToOp; 48 import org.graalvm.compiler.lir.aarch64.AArch64ArrayEqualsOp; 49 import org.graalvm.compiler.lir.aarch64.AArch64ByteSwapOp; 50 import org.graalvm.compiler.lir.aarch64.AArch64Compare; 51 import org.graalvm.compiler.lir.aarch64.AArch64ControlFlow; 52 import org.graalvm.compiler.lir.aarch64.AArch64ControlFlow.BranchOp; 53 import org.graalvm.compiler.lir.aarch64.AArch64ControlFlow.CondMoveOp; 54 import org.graalvm.compiler.lir.aarch64.AArch64ControlFlow.StrategySwitchOp; 55 import org.graalvm.compiler.lir.aarch64.AArch64ControlFlow.TableSwitchOp; 56 import org.graalvm.compiler.lir.aarch64.AArch64LIRFlagsVersioned; 57 import org.graalvm.compiler.lir.aarch64.AArch64Move; 58 import org.graalvm.compiler.lir.aarch64.AArch64AtomicMove.AtomicReadAndAddOp; 59 import org.graalvm.compiler.lir.aarch64.AArch64AtomicMove.AtomicReadAndAddLSEOp; 60 import org.graalvm.compiler.lir.aarch64.AArch64AtomicMove.CompareAndSwapOp; 61 import org.graalvm.compiler.lir.aarch64.AArch64AtomicMove.AtomicReadAndWriteOp; 62 import org.graalvm.compiler.lir.aarch64.AArch64Move.MembarOp; 63 import org.graalvm.compiler.lir.aarch64.AArch64PauseOp; 64 import org.graalvm.compiler.lir.gen.LIRGenerationResult; 65 import org.graalvm.compiler.lir.gen.LIRGenerator; 66 import org.graalvm.compiler.phases.util.Providers; 67 68 import jdk.vm.ci.aarch64.AArch64; 69 import jdk.vm.ci.aarch64.AArch64Kind; 70 import jdk.vm.ci.code.CallingConvention; 71 import jdk.vm.ci.code.RegisterValue; 72 import jdk.vm.ci.meta.AllocatableValue; 73 import jdk.vm.ci.meta.JavaConstant; 74 import jdk.vm.ci.meta.JavaKind; 75 import jdk.vm.ci.meta.PlatformKind; 76 import jdk.vm.ci.meta.PrimitiveConstant; 77 import jdk.vm.ci.meta.Value; 78 import jdk.vm.ci.meta.ValueKind; 79 80 public abstract class AArch64LIRGenerator extends LIRGenerator { 81 82 public AArch64LIRGenerator(LIRKindTool lirKindTool, AArch64ArithmeticLIRGenerator arithmeticLIRGen, MoveFactory moveFactory, Providers providers, LIRGenerationResult lirGenRes) { 83 super(lirKindTool, arithmeticLIRGen, moveFactory, providers, lirGenRes); 84 } 85 86 /** 87 * Checks whether the supplied constant can be used without loading it into a register for store 88 * operations, i.e., on the right hand side of a memory access. 89 * 90 * @param c The constant to check. 91 * @return True if the constant can be used directly, false if the constant needs to be in a 92 * register. 93 */ 94 protected static final boolean canStoreConstant(JavaConstant c) { 95 // Our own code never calls this since we can't make a definite statement about whether or 96 // not we can inline a constant without knowing what kind of operation we execute. Let's be 97 // optimistic here and fix up mistakes later. 98 return true; 99 } 100 101 /** 102 * AArch64 cannot use anything smaller than a word in any instruction other than load and store. 103 */ 104 @Override 105 public <K extends ValueKind<K>> K toRegisterKind(K kind) { 106 switch ((AArch64Kind) kind.getPlatformKind()) { 107 case BYTE: 108 case WORD: 109 return kind.changeType(AArch64Kind.DWORD); 110 default: 111 return kind; 112 } 113 } 114 115 @Override 116 public void emitNullCheck(Value address, LIRFrameState state) { 117 append(new AArch64Move.NullCheckOp(asAddressValue(address), state)); 118 } 119 120 @Override 121 public Variable emitAddress(AllocatableValue stackslot) { 122 Variable result = newVariable(LIRKind.value(target().arch.getWordKind())); 123 append(new AArch64Move.StackLoadAddressOp(result, stackslot)); 124 return result; 125 } 126 127 public AArch64AddressValue asAddressValue(Value address) { 128 if (address instanceof AArch64AddressValue) { 129 return (AArch64AddressValue) address; 130 } else { 131 return new AArch64AddressValue(address.getValueKind(), asAllocatable(address), Value.ILLEGAL, 0, 1, AddressingMode.BASE_REGISTER_ONLY); 132 } 133 } 134 135 @Override 136 public Variable emitLogicCompareAndSwap(LIRKind accessKind, Value address, Value expectedValue, Value newValue, Value trueValue, Value falseValue) { 137 Variable prevValue = newVariable(expectedValue.getValueKind()); 138 Variable scratch = newVariable(LIRKind.value(AArch64Kind.DWORD)); 139 append(new CompareAndSwapOp(prevValue, loadReg(expectedValue), loadReg(newValue), asAllocatable(address), scratch)); 140 assert trueValue.getValueKind().equals(falseValue.getValueKind()); 141 Variable result = newVariable(trueValue.getValueKind()); 142 append(new CondMoveOp(result, ConditionFlag.EQ, asAllocatable(trueValue), asAllocatable(falseValue))); 143 return result; 144 } 145 146 @Override 147 public Variable emitValueCompareAndSwap(LIRKind accessKind, Value address, Value expectedValue, Value newValue) { 148 Variable result = newVariable(newValue.getValueKind()); 149 Variable scratch = newVariable(LIRKind.value(AArch64Kind.WORD)); 150 append(new CompareAndSwapOp(result, loadNonCompareConst(expectedValue), loadReg(newValue), asAllocatable(address), scratch)); 151 return result; 152 } 153 154 @Override 155 public Value emitAtomicReadAndWrite(Value address, ValueKind<?> kind, Value newValue) { 156 Variable result = newVariable(kind); 157 Variable scratch = newVariable(kind); 158 append(new AtomicReadAndWriteOp((AArch64Kind) kind.getPlatformKind(), asAllocatable(result), asAllocatable(address), asAllocatable(newValue), asAllocatable(scratch))); 159 return result; 160 } 161 162 @Override 163 public Value emitAtomicReadAndAdd(Value address, ValueKind<?> kind, Value delta) { 164 Variable result = newVariable(kind); 165 if (AArch64LIRFlagsVersioned.useLSE(target().arch)) { 166 append(new AtomicReadAndAddLSEOp((AArch64Kind) kind.getPlatformKind(), asAllocatable(result), asAllocatable(address), asAllocatable(delta))); 167 } else { 168 append(new AtomicReadAndAddOp((AArch64Kind) kind.getPlatformKind(), asAllocatable(result), asAllocatable(address), delta)); 169 } 170 return result; 171 } 172 173 @Override 174 public void emitMembar(int barriers) { 175 int necessaryBarriers = target().arch.requiredBarriers(barriers); 176 if (target().isMP && necessaryBarriers != 0) { 177 append(new MembarOp(necessaryBarriers)); 178 } 179 } 180 181 @Override 182 public void emitJump(LabelRef label) { 183 assert label != null; 184 append(new StandardOp.JumpOp(label)); 185 } 186 187 @Override 188 public void emitOverflowCheckBranch(LabelRef overflow, LabelRef noOverflow, LIRKind cmpKind, double overflowProbability) { 189 append(new AArch64ControlFlow.BranchOp(ConditionFlag.VS, overflow, noOverflow, overflowProbability)); 190 } 191 192 /** 193 * Branches to label if (left & right) == 0. If negated is true branchse on non-zero instead. 194 * 195 * @param left Integer kind. Non null. 196 * @param right Integer kind. Non null. 197 * @param trueDestination destination if left & right == 0. Non null. 198 * @param falseDestination destination if left & right != 0. Non null 199 * @param trueSuccessorProbability hoistoric probability that comparison is true 200 */ 201 @Override 202 public void emitIntegerTestBranch(Value left, Value right, LabelRef trueDestination, LabelRef falseDestination, double trueSuccessorProbability) { 203 assert ((AArch64Kind) left.getPlatformKind()).isInteger() && left.getPlatformKind() == right.getPlatformKind(); 204 ((AArch64ArithmeticLIRGenerator) getArithmetic()).emitBinary(LIRKind.combine(left, right), AArch64ArithmeticOp.ANDS, true, left, right); 205 append(new AArch64ControlFlow.BranchOp(ConditionFlag.EQ, trueDestination, falseDestination, trueSuccessorProbability)); 206 } 207 208 /** 209 * Conditionally move trueValue into new variable if cond + unorderedIsTrue is true, else 210 * falseValue. 211 * 212 * @param left Arbitrary value. Has to have same type as right. Non null. 213 * @param right Arbitrary value. Has to have same type as left. Non null. 214 * @param cond condition that decides whether to move trueValue or falseValue into result. Non 215 * null. 216 * @param unorderedIsTrue defines whether floating-point comparisons consider unordered true or 217 * not. Ignored for integer comparisons. 218 * @param trueValue arbitrary value same type as falseValue. Non null. 219 * @param falseValue arbitrary value same type as trueValue. Non null. 220 * @return value containing trueValue if cond + unorderedIsTrue is true, else falseValue. Non 221 * null. 222 */ 223 @Override 224 public Variable emitConditionalMove(PlatformKind cmpKind, Value left, Value right, Condition cond, boolean unorderedIsTrue, Value trueValue, Value falseValue) { 225 boolean mirrored = emitCompare(cmpKind, left, right, cond, unorderedIsTrue); 226 Condition finalCondition = mirrored ? cond.mirror() : cond; 227 boolean finalUnorderedIsTrue = mirrored ? !unorderedIsTrue : unorderedIsTrue; 228 ConditionFlag cmpCondition = toConditionFlag(((AArch64Kind) cmpKind).isInteger(), finalCondition, finalUnorderedIsTrue); 229 Variable result = newVariable(trueValue.getValueKind()); 230 append(new CondMoveOp(result, cmpCondition, loadReg(trueValue), loadReg(falseValue))); 231 return result; 232 } 233 234 @Override 235 public void emitCompareBranch(PlatformKind cmpKind, Value left, Value right, Condition cond, boolean unorderedIsTrue, LabelRef trueDestination, LabelRef falseDestination, 236 double trueDestinationProbability) { 237 boolean mirrored = emitCompare(cmpKind, left, right, cond, unorderedIsTrue); 238 Condition finalCondition = mirrored ? cond.mirror() : cond; 239 boolean finalUnorderedIsTrue = mirrored ? !unorderedIsTrue : unorderedIsTrue; 240 ConditionFlag cmpCondition = toConditionFlag(((AArch64Kind) cmpKind).isInteger(), finalCondition, finalUnorderedIsTrue); 241 append(new BranchOp(cmpCondition, trueDestination, falseDestination, trueDestinationProbability)); 242 } 243 244 private static ConditionFlag toConditionFlag(boolean isInt, Condition cond, boolean unorderedIsTrue) { 245 return isInt ? toIntConditionFlag(cond) : toFloatConditionFlag(cond, unorderedIsTrue); 246 } 247 248 /** 249 * Takes a Condition and unorderedIsTrue flag and returns the correct Aarch64 specific 250 * ConditionFlag. Note: This is only correct if the emitCompare code for floats has correctly 251 * handled the case of 'EQ && unorderedIsTrue', respectively 'NE && !unorderedIsTrue'! 252 */ 253 private static ConditionFlag toFloatConditionFlag(Condition cond, boolean unorderedIsTrue) { 254 switch (cond) { 255 case LT: 256 return unorderedIsTrue ? ConditionFlag.LT : ConditionFlag.LO; 257 case LE: 258 return unorderedIsTrue ? ConditionFlag.LE : ConditionFlag.LS; 259 case GE: 260 return unorderedIsTrue ? ConditionFlag.PL : ConditionFlag.GE; 261 case GT: 262 return unorderedIsTrue ? ConditionFlag.HI : ConditionFlag.GT; 263 case EQ: 264 return ConditionFlag.EQ; 265 case NE: 266 return ConditionFlag.NE; 267 default: 268 throw GraalError.shouldNotReachHere(); 269 } 270 } 271 272 /** 273 * Takes a Condition and returns the correct Aarch64 specific ConditionFlag. 274 */ 275 private static ConditionFlag toIntConditionFlag(Condition cond) { 276 switch (cond) { 277 case EQ: 278 return ConditionFlag.EQ; 279 case NE: 280 return ConditionFlag.NE; 281 case LT: 282 return ConditionFlag.LT; 283 case LE: 284 return ConditionFlag.LE; 285 case GT: 286 return ConditionFlag.GT; 287 case GE: 288 return ConditionFlag.GE; 289 case AE: 290 return ConditionFlag.HS; 291 case BE: 292 return ConditionFlag.LS; 293 case AT: 294 return ConditionFlag.HI; 295 case BT: 296 return ConditionFlag.LO; 297 default: 298 throw GraalError.shouldNotReachHere(); 299 } 300 } 301 302 /** 303 * This method emits the compare instruction, and may reorder the operands. It returns true if 304 * it did so. 305 * 306 * @param a the left operand of the comparison. Has to have same type as b. Non null. 307 * @param b the right operand of the comparison. Has to have same type as a. Non null. 308 * @return true if mirrored (i.e. "b cmp a" instead of "a cmp b" was done). 309 */ 310 protected boolean emitCompare(PlatformKind cmpKind, Value a, Value b, Condition condition, boolean unorderedIsTrue) { 311 Value left; 312 Value right; 313 boolean mirrored; 314 AArch64Kind kind = (AArch64Kind) cmpKind; 315 if (kind.isInteger()) { 316 Value aExt = a; 317 Value bExt = b; 318 319 int compareBytes = cmpKind.getSizeInBytes(); 320 // AArch64 compares 32 or 64 bits: sign extend a and b as required. 321 if (compareBytes < a.getPlatformKind().getSizeInBytes()) { 322 aExt = arithmeticLIRGen.emitSignExtend(a, compareBytes * 8, 64); 323 } 324 if (compareBytes < b.getPlatformKind().getSizeInBytes()) { 325 bExt = arithmeticLIRGen.emitSignExtend(b, compareBytes * 8, 64); 326 } 327 328 if (LIRValueUtil.isVariable(bExt)) { 329 left = load(bExt); 330 right = loadNonConst(aExt); 331 mirrored = true; 332 } else { 333 left = load(aExt); 334 right = loadNonConst(bExt); 335 mirrored = false; 336 } 337 append(new AArch64Compare.CompareOp(left, loadNonCompareConst(right))); 338 } else if (kind.isSIMD()) { 339 if (AArch64Compare.FloatCompareOp.isFloatCmpConstant(a, condition, unorderedIsTrue)) { 340 left = load(b); 341 right = a; 342 mirrored = true; 343 } else if (AArch64Compare.FloatCompareOp.isFloatCmpConstant(b, condition, unorderedIsTrue)) { 344 left = load(a); 345 right = b; 346 mirrored = false; 347 } else { 348 left = load(a); 349 right = loadReg(b); 350 mirrored = false; 351 } 352 append(new AArch64Compare.FloatCompareOp(left, asAllocatable(right), condition, unorderedIsTrue)); 353 } else { 354 throw GraalError.shouldNotReachHere(); 355 } 356 return mirrored; 357 } 358 359 /** 360 * If value is a constant that cannot be used directly with a gpCompare instruction load it into 361 * a register and return the register, otherwise return constant value unchanged. 362 */ 363 protected Value loadNonCompareConst(Value value) { 364 if (!isCompareConstant(value)) { 365 return loadReg(value); 366 } 367 return value; 368 } 369 370 /** 371 * Checks whether value can be used directly with a gpCompare instruction. This is <b>not</b> 372 * the same as {@link AArch64ArithmeticLIRGenerator#isArithmeticConstant(JavaConstant)}, because 373 * 0.0 is a valid compare constant for floats, while there are no arithmetic constants for 374 * floats. 375 * 376 * @param value any type. Non null. 377 * @return true if value can be used directly in comparison instruction, false otherwise. 378 */ 379 public boolean isCompareConstant(Value value) { 380 if (isJavaConstant(value)) { 381 JavaConstant constant = asJavaConstant(value); 382 if (constant instanceof PrimitiveConstant) { 383 final long longValue = constant.asLong(); 384 long maskedValue; 385 switch (constant.getJavaKind()) { 386 case Boolean: 387 case Byte: 388 maskedValue = longValue & 0xFF; 389 break; 390 case Char: 391 case Short: 392 maskedValue = longValue & 0xFFFF; 393 break; 394 case Int: 395 maskedValue = longValue & 0xFFFF_FFFF; 396 break; 397 case Long: 398 maskedValue = longValue; 399 break; 400 default: 401 throw GraalError.shouldNotReachHere(); 402 } 403 return AArch64MacroAssembler.isArithmeticImmediate(maskedValue); 404 } else { 405 return constant.isDefaultForKind(); 406 } 407 } 408 return false; 409 } 410 411 /** 412 * Moves trueValue into result if (left & right) == 0, else falseValue. 413 * 414 * @param left Integer kind. Non null. 415 * @param right Integer kind. Non null. 416 * @param trueValue Integer kind. Non null. 417 * @param falseValue Integer kind. Non null. 418 * @return virtual register containing trueValue if (left & right) == 0, else falseValue. 419 */ 420 @Override 421 public Variable emitIntegerTestMove(Value left, Value right, Value trueValue, Value falseValue) { 422 assert ((AArch64Kind) left.getPlatformKind()).isInteger() && ((AArch64Kind) right.getPlatformKind()).isInteger(); 423 assert ((AArch64Kind) trueValue.getPlatformKind()).isInteger() && ((AArch64Kind) falseValue.getPlatformKind()).isInteger(); 424 ((AArch64ArithmeticLIRGenerator) getArithmetic()).emitBinary(left.getValueKind(), AArch64ArithmeticOp.ANDS, true, left, right); 425 Variable result = newVariable(trueValue.getValueKind()); 426 append(new CondMoveOp(result, ConditionFlag.EQ, load(trueValue), load(falseValue))); 427 return result; 428 } 429 430 @Override 431 public void emitStrategySwitch(SwitchStrategy strategy, Variable key, LabelRef[] keyTargets, LabelRef defaultTarget) { 432 append(createStrategySwitchOp(strategy, keyTargets, defaultTarget, key, newVariable(key.getValueKind()), AArch64LIRGenerator::toIntConditionFlag)); 433 } 434 435 protected StrategySwitchOp createStrategySwitchOp(SwitchStrategy strategy, LabelRef[] keyTargets, LabelRef defaultTarget, Variable key, AllocatableValue scratchValue, 436 Function<Condition, ConditionFlag> converter) { 437 return new StrategySwitchOp(strategy, keyTargets, defaultTarget, key, scratchValue, converter); 438 } 439 440 @Override 441 protected void emitTableSwitch(int lowKey, LabelRef defaultTarget, LabelRef[] targets, Value key) { 442 append(new TableSwitchOp(lowKey, defaultTarget, targets, key, newVariable(LIRKind.value(target().arch.getWordKind())), newVariable(key.getValueKind()))); 443 } 444 445 @Override 446 public Variable emitByteSwap(Value input) { 447 Variable result = newVariable(LIRKind.combine(input)); 448 append(new AArch64ByteSwapOp(result, input)); 449 return result; 450 } 451 452 @Override 453 public Variable emitArrayCompareTo(JavaKind kind1, JavaKind kind2, Value array1, Value array2, Value length1, Value length2) { 454 LIRKind resultKind = LIRKind.value(AArch64Kind.DWORD); 455 // DMS TODO: check calling conversion and registers used 456 RegisterValue res = AArch64.r0.asValue(resultKind); 457 RegisterValue cnt1 = AArch64.r1.asValue(length1.getValueKind()); 458 RegisterValue cnt2 = AArch64.r2.asValue(length2.getValueKind()); 459 emitMove(cnt1, length1); 460 emitMove(cnt2, length2); 461 append(new AArch64ArrayCompareToOp(this, kind1, kind2, res, array1, array2, cnt1, cnt2)); 462 Variable result = newVariable(resultKind); 463 emitMove(result, res); 464 return result; 465 } 466 467 @Override 468 public Variable emitArrayEquals(JavaKind kind, Value array1, Value array2, Value length) { 469 Variable result = newVariable(LIRKind.value(AArch64Kind.DWORD)); 470 append(new AArch64ArrayEqualsOp(this, kind, result, array1, array2, asAllocatable(length))); 471 return result; 472 } 473 474 @Override 475 protected JavaConstant zapValueForKind(PlatformKind kind) { 476 long dead = 0xDEADDEADDEADDEADL; 477 switch ((AArch64Kind) kind) { 478 case BYTE: 479 return JavaConstant.forByte((byte) dead); 480 case WORD: 481 return JavaConstant.forShort((short) dead); 482 case DWORD: 483 return JavaConstant.forInt((int) dead); 484 case QWORD: 485 return JavaConstant.forLong(dead); 486 case SINGLE: 487 return JavaConstant.forFloat(Float.intBitsToFloat((int) dead)); 488 case DOUBLE: 489 return JavaConstant.forDouble(Double.longBitsToDouble(dead)); 490 default: 491 throw GraalError.shouldNotReachHere(); 492 } 493 } 494 495 /** 496 * Loads value into virtual register. Contrary to {@link #load(Value)} this handles 497 * RegisterValues (i.e. values corresponding to fixed physical registers) correctly, by not 498 * creating an unnecessary move into a virtual register. 499 * 500 * This avoids generating the following code: mov x0, x19 # x19 is fixed thread register ldr x0, 501 * [x0] instead of: ldr x0, [x19]. 502 */ 503 protected AllocatableValue loadReg(Value val) { 504 if (!(val instanceof Variable || val instanceof RegisterValue)) { 505 return emitMove(val); 506 } 507 return (AllocatableValue) val; 508 } 509 510 @Override 511 public void emitPause() { 512 append(new AArch64PauseOp()); 513 } 514 515 public abstract void emitCCall(long address, CallingConvention nativeCallingConvention, Value[] args); 516 }