1 /* 2 * Copyright (c) 2013, 2018, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 */ 23 24 25 package org.graalvm.compiler.lir.aarch64; 26 27 import static org.graalvm.compiler.lir.LIRInstruction.OperandFlag.COMPOSITE; 28 import static org.graalvm.compiler.lir.LIRInstruction.OperandFlag.HINT; 29 import static org.graalvm.compiler.lir.LIRInstruction.OperandFlag.REG; 30 import static org.graalvm.compiler.lir.LIRInstruction.OperandFlag.STACK; 31 import static org.graalvm.compiler.lir.LIRInstruction.OperandFlag.UNINITIALIZED; 32 import static org.graalvm.compiler.lir.LIRValueUtil.asJavaConstant; 33 import static org.graalvm.compiler.lir.LIRValueUtil.isJavaConstant; 34 import static jdk.vm.ci.aarch64.AArch64.sp; 35 import static jdk.vm.ci.aarch64.AArch64.zr; 36 import static jdk.vm.ci.code.ValueUtil.asAllocatableValue; 37 import static jdk.vm.ci.code.ValueUtil.asRegister; 38 import static jdk.vm.ci.code.ValueUtil.asStackSlot; 39 import static jdk.vm.ci.code.ValueUtil.isRegister; 40 import static jdk.vm.ci.code.ValueUtil.isStackSlot; 41 42 import org.graalvm.compiler.asm.aarch64.AArch64Address; 43 import org.graalvm.compiler.asm.aarch64.AArch64MacroAssembler; 44 import org.graalvm.compiler.asm.aarch64.AArch64MacroAssembler.ScratchRegister; 45 import org.graalvm.compiler.core.common.LIRKind; 46 import org.graalvm.compiler.core.common.type.DataPointerConstant; 47 import org.graalvm.compiler.debug.GraalError; 48 import org.graalvm.compiler.lir.LIRFrameState; 49 import org.graalvm.compiler.lir.LIRInstructionClass; 50 import org.graalvm.compiler.lir.Opcode; 51 import org.graalvm.compiler.lir.StandardOp; 52 import org.graalvm.compiler.lir.StandardOp.LoadConstantOp; 53 import org.graalvm.compiler.lir.StandardOp.NullCheck; 54 import org.graalvm.compiler.lir.StandardOp.ValueMoveOp; 55 import org.graalvm.compiler.lir.VirtualStackSlot; 56 import org.graalvm.compiler.lir.asm.CompilationResultBuilder; 57 58 import jdk.vm.ci.aarch64.AArch64Kind; 59 import jdk.vm.ci.code.Register; 60 import jdk.vm.ci.code.StackSlot; 61 import jdk.vm.ci.meta.AllocatableValue; 62 import jdk.vm.ci.meta.Constant; 63 import jdk.vm.ci.meta.JavaConstant; 64 import jdk.vm.ci.meta.PlatformKind; 65 import jdk.vm.ci.meta.Value; 66 67 public class AArch64Move { 68 69 public static class LoadInlineConstant extends AArch64LIRInstruction implements LoadConstantOp { 70 public static final LIRInstructionClass<LoadInlineConstant> TYPE = LIRInstructionClass.create(LoadInlineConstant.class); 71 72 private JavaConstant constant; 73 @Def({REG, STACK}) AllocatableValue result; 74 75 public LoadInlineConstant(JavaConstant constant, AllocatableValue result) { 76 super(TYPE); 77 this.constant = constant; 78 this.result = result; 79 } 80 81 @Override 82 public void emitCode(CompilationResultBuilder crb, AArch64MacroAssembler masm) { 83 if (isRegister(result)) { 84 const2reg(crb, masm, result, constant); 85 } else if (isStackSlot(result)) { 86 StackSlot slot = asStackSlot(result); 87 const2stack(crb, masm, slot, constant); 88 } 89 } 90 91 @Override 92 public Constant getConstant() { 93 return constant; 94 } 95 96 @Override 97 public AllocatableValue getResult() { 98 return result; 99 } 100 } 101 102 @Opcode("MOVE") 103 public static class Move extends AArch64LIRInstruction implements ValueMoveOp { 104 public static final LIRInstructionClass<Move> TYPE = LIRInstructionClass.create(Move.class); 105 106 @Def({REG, STACK, HINT}) protected AllocatableValue result; 107 @Use({REG, STACK}) protected AllocatableValue input; 108 109 public Move(AllocatableValue result, AllocatableValue input) { 110 super(TYPE); 111 this.result = result; 112 this.input = input; 113 assert !(isStackSlot(result) && isStackSlot(input)); 114 } 115 116 @Override 117 public void emitCode(CompilationResultBuilder crb, AArch64MacroAssembler masm) { 118 move(crb, masm, getResult(), getInput()); 119 } 120 121 @Override 122 public AllocatableValue getInput() { 123 return input; 124 } 125 126 @Override 127 public AllocatableValue getResult() { 128 return result; 129 } 130 } 131 132 public static class LoadAddressOp extends AArch64LIRInstruction { 133 public static final LIRInstructionClass<LoadAddressOp> TYPE = LIRInstructionClass.create(LoadAddressOp.class); 134 135 @Def protected AllocatableValue result; 136 @Use(COMPOSITE) protected AArch64AddressValue address; 137 138 public LoadAddressOp(AllocatableValue result, AArch64AddressValue address) { 139 super(TYPE); 140 this.result = result; 141 this.address = address; 142 } 143 144 @Override 145 public void emitCode(CompilationResultBuilder crb, AArch64MacroAssembler masm) { 146 Register dst = asRegister(result); 147 AArch64Address adr = address.toAddress(); 148 masm.loadAddress(dst, adr, address.getScaleFactor()); 149 } 150 } 151 152 public static class LoadDataOp extends AArch64LIRInstruction { 153 public static final LIRInstructionClass<LoadDataOp> TYPE = LIRInstructionClass.create(LoadDataOp.class); 154 155 @Def protected AllocatableValue result; 156 private final DataPointerConstant data; 157 158 public LoadDataOp(AllocatableValue result, DataPointerConstant data) { 159 super(TYPE); 160 this.result = result; 161 this.data = data; 162 } 163 164 @Override 165 public void emitCode(CompilationResultBuilder crb, AArch64MacroAssembler masm) { 166 Register dst = asRegister(result); 167 if (crb.compilationResult.isImmutablePIC()) { 168 crb.recordDataReferenceInCode(data); 169 masm.addressOf(dst); 170 } else { 171 masm.loadAddress(dst, (AArch64Address) crb.recordDataReferenceInCode(data), data.getAlignment()); 172 } 173 } 174 } 175 176 public static class StackLoadAddressOp extends AArch64LIRInstruction { 177 public static final LIRInstructionClass<StackLoadAddressOp> TYPE = LIRInstructionClass.create(StackLoadAddressOp.class); 178 179 @Def protected AllocatableValue result; 180 @Use({STACK, UNINITIALIZED}) protected AllocatableValue slot; 181 182 public StackLoadAddressOp(AllocatableValue result, AllocatableValue slot) { 183 super(TYPE); 184 assert slot instanceof VirtualStackSlot || slot instanceof StackSlot; 185 this.result = result; 186 this.slot = slot; 187 } 188 189 @Override 190 public void emitCode(CompilationResultBuilder crb, AArch64MacroAssembler masm) { 191 try (ScratchRegister addrReg = masm.getScratchRegister()) { 192 AArch64Address address = loadStackSlotAddress(crb, masm, (StackSlot) slot, addrReg.getRegister()); 193 PlatformKind kind = AArch64Kind.QWORD; 194 masm.loadAddress(asRegister(result, kind), address, kind.getSizeInBytes()); 195 } 196 } 197 } 198 199 public static class MembarOp extends AArch64LIRInstruction { 200 public static final LIRInstructionClass<MembarOp> TYPE = LIRInstructionClass.create(MembarOp.class); 201 202 // For future use. 203 @SuppressWarnings("unused") private final int barriers; 204 205 public MembarOp(int barriers) { 206 super(TYPE); 207 this.barriers = barriers; 208 } 209 210 @Override 211 // The odd-looking @SuppressWarnings("all") is here because of 212 // a compiler bug which warns that crb is unused, and also 213 // warns that @SuppressWarnings("unused") is unnecessary. 214 public void emitCode(@SuppressWarnings("all") CompilationResultBuilder crb, AArch64MacroAssembler masm) { 215 // As I understand it load acquire/store release have the same semantics as on IA64 216 // and allow us to handle LoadStore, LoadLoad and StoreStore without an explicit 217 // barrier. 218 // But Graal support to figure out if a load/store is volatile is non-existant so for 219 // now just use memory barriers everywhere. 220 // if ((barrier & MemoryBarriers.STORE_LOAD) != 0) { 221 masm.dmb(AArch64MacroAssembler.BarrierKind.ANY_ANY); 222 // } 223 } 224 } 225 226 abstract static class MemOp extends AArch64LIRInstruction implements StandardOp.ImplicitNullCheck { 227 228 protected final AArch64Kind kind; 229 @Use({COMPOSITE}) protected AArch64AddressValue addressValue; 230 @State protected LIRFrameState state; 231 232 MemOp(LIRInstructionClass<? extends MemOp> c, AArch64Kind kind, AArch64AddressValue address, LIRFrameState state) { 233 super(c); 234 this.kind = kind; 235 this.addressValue = address; 236 this.state = state; 237 } 238 239 protected abstract void emitMemAccess(CompilationResultBuilder crb, AArch64MacroAssembler masm); 240 241 @Override 242 public void emitCode(CompilationResultBuilder crb, AArch64MacroAssembler masm) { 243 if (state != null) { 244 crb.recordImplicitException(masm.position(), state); 245 } 246 emitMemAccess(crb, masm); 247 } 248 249 @Override 250 public boolean makeNullCheckFor(Value value, LIRFrameState nullCheckState, int implicitNullCheckLimit) { 251 int displacement = addressValue.getDisplacement(); 252 if (state == null && value.equals(addressValue.getBase()) && addressValue.getOffset().equals(Value.ILLEGAL) && displacement >= 0 && displacement < implicitNullCheckLimit) { 253 state = nullCheckState; 254 return true; 255 } 256 return false; 257 } 258 } 259 260 public static final class LoadOp extends MemOp { 261 public static final LIRInstructionClass<LoadOp> TYPE = LIRInstructionClass.create(LoadOp.class); 262 263 @Def protected AllocatableValue result; 264 265 public LoadOp(AArch64Kind kind, AllocatableValue result, AArch64AddressValue address, LIRFrameState state) { 266 super(TYPE, kind, address, state); 267 this.result = result; 268 } 269 270 @Override 271 protected void emitMemAccess(CompilationResultBuilder crb, AArch64MacroAssembler masm) { 272 AArch64Address address = addressValue.toAddress(); 273 Register dst = asRegister(result); 274 275 int destSize = result.getPlatformKind().getSizeInBytes() * Byte.SIZE; 276 int srcSize = kind.getSizeInBytes() * Byte.SIZE; 277 if (kind.isInteger()) { 278 masm.ldr(srcSize, dst, address); 279 } else { 280 assert srcSize == destSize; 281 masm.fldr(srcSize, dst, address); 282 } 283 } 284 } 285 286 public static class StoreOp extends MemOp { 287 public static final LIRInstructionClass<StoreOp> TYPE = LIRInstructionClass.create(StoreOp.class); 288 @Use protected AllocatableValue input; 289 290 public StoreOp(AArch64Kind kind, AArch64AddressValue address, AllocatableValue input, LIRFrameState state) { 291 super(TYPE, kind, address, state); 292 this.input = input; 293 } 294 295 @Override 296 protected void emitMemAccess(CompilationResultBuilder crb, AArch64MacroAssembler masm) { 297 emitStore(crb, masm, kind, addressValue.toAddress(), input); 298 } 299 } 300 301 public static final class StoreConstantOp extends MemOp { 302 public static final LIRInstructionClass<StoreConstantOp> TYPE = LIRInstructionClass.create(StoreConstantOp.class); 303 304 protected final JavaConstant input; 305 306 public StoreConstantOp(AArch64Kind kind, AArch64AddressValue address, JavaConstant input, LIRFrameState state) { 307 super(TYPE, kind, address, state); 308 this.input = input; 309 if (!input.isDefaultForKind()) { 310 throw GraalError.shouldNotReachHere("Can only store null constants to memory"); 311 } 312 } 313 314 @Override 315 public void emitMemAccess(CompilationResultBuilder crb, AArch64MacroAssembler masm) { 316 emitStore(crb, masm, kind, addressValue.toAddress(), zr.asValue(LIRKind.combine(addressValue))); 317 } 318 } 319 320 public static final class NullCheckOp extends AArch64LIRInstruction implements NullCheck { 321 public static final LIRInstructionClass<NullCheckOp> TYPE = LIRInstructionClass.create(NullCheckOp.class); 322 323 @Use(COMPOSITE) protected AArch64AddressValue address; 324 @State protected LIRFrameState state; 325 326 public NullCheckOp(AArch64AddressValue address, LIRFrameState state) { 327 super(TYPE); 328 this.address = address; 329 this.state = state; 330 } 331 332 @Override 333 public void emitCode(CompilationResultBuilder crb, AArch64MacroAssembler masm) { 334 crb.recordImplicitException(masm.position(), state); 335 masm.ldr(64, zr, address.toAddress()); 336 } 337 338 @Override 339 public Value getCheckedValue() { 340 return address.base; 341 } 342 343 @Override 344 public LIRFrameState getState() { 345 return state; 346 } 347 } 348 349 private static void emitStore(@SuppressWarnings("unused") CompilationResultBuilder crb, AArch64MacroAssembler masm, AArch64Kind kind, AArch64Address dst, Value src) { 350 int destSize = kind.getSizeInBytes() * Byte.SIZE; 351 if (kind.isInteger()) { 352 masm.str(destSize, asRegister(src), dst); 353 } else { 354 masm.fstr(destSize, asRegister(src), dst); 355 } 356 } 357 358 public static void move(CompilationResultBuilder crb, AArch64MacroAssembler masm, AllocatableValue result, Value input) { 359 if (isRegister(input)) { 360 if (isRegister(result)) { 361 reg2reg(crb, masm, result, asAllocatableValue(input)); 362 } else if (isStackSlot(result)) { 363 reg2stack(crb, masm, result, asAllocatableValue(input)); 364 } else { 365 throw GraalError.shouldNotReachHere(); 366 } 367 } else if (isStackSlot(input)) { 368 if (isRegister(result)) { 369 stack2reg(crb, masm, result, asAllocatableValue(input)); 370 } else if (isStackSlot(result)) { 371 emitStackMove(crb, masm, result, input); 372 } else { 373 throw GraalError.shouldNotReachHere(); 374 } 375 } else if (isJavaConstant(input)) { 376 if (isRegister(result)) { 377 const2reg(crb, masm, result, asJavaConstant(input)); 378 } else { 379 throw GraalError.shouldNotReachHere(); 380 } 381 } else { 382 throw GraalError.shouldNotReachHere(); 383 } 384 } 385 386 private static void emitStackMove(CompilationResultBuilder crb, AArch64MacroAssembler masm, AllocatableValue result, Value input) { 387 try (ScratchRegister r1 = masm.getScratchRegister()) { 388 try (ScratchRegister r2 = masm.getScratchRegister()) { 389 Register rscratch1 = r1.getRegister(); 390 Register rscratch2 = r2.getRegister(); 391 // use the slot kind to define the operand size 392 PlatformKind kind = input.getPlatformKind(); 393 final int size = kind.getSizeInBytes() * Byte.SIZE; 394 395 // Always perform stack -> stack copies through integer registers 396 crb.blockComment("[stack -> stack copy]"); 397 AArch64Address src = loadStackSlotAddress(crb, masm, asStackSlot(input), rscratch2); 398 masm.ldr(size, rscratch1, src); 399 AArch64Address dst = loadStackSlotAddress(crb, masm, asStackSlot(result), rscratch2); 400 masm.str(size, rscratch1, dst); 401 } 402 } 403 } 404 405 private static void reg2reg(@SuppressWarnings("unused") CompilationResultBuilder crb, AArch64MacroAssembler masm, AllocatableValue result, AllocatableValue input) { 406 Register dst = asRegister(result); 407 Register src = asRegister(input); 408 if (src.equals(dst)) { 409 return; 410 } 411 AArch64Kind kind = (AArch64Kind) input.getPlatformKind(); 412 int size = kind.getSizeInBytes() * Byte.SIZE; 413 if (kind.isInteger()) { 414 masm.mov(size, dst, src); 415 } else { 416 masm.fmov(size, dst, src); 417 } 418 } 419 420 static void reg2stack(CompilationResultBuilder crb, AArch64MacroAssembler masm, AllocatableValue result, AllocatableValue input) { 421 AArch64Address dest = loadStackSlotAddress(crb, masm, asStackSlot(result), Value.ILLEGAL); 422 Register src = asRegister(input); 423 // use the slot kind to define the operand size 424 AArch64Kind kind = (AArch64Kind) result.getPlatformKind(); 425 final int size = kind.getSizeInBytes() * Byte.SIZE; 426 if (kind.isInteger()) { 427 masm.str(size, src, dest); 428 } else { 429 masm.fstr(size, src, dest); 430 } 431 } 432 433 static void stack2reg(CompilationResultBuilder crb, AArch64MacroAssembler masm, AllocatableValue result, AllocatableValue input) { 434 AArch64Kind kind = (AArch64Kind) input.getPlatformKind(); 435 // use the slot kind to define the operand size 436 final int size = kind.getSizeInBytes() * Byte.SIZE; 437 if (kind.isInteger()) { 438 AArch64Address src = loadStackSlotAddress(crb, masm, asStackSlot(input), result); 439 masm.ldr(size, asRegister(result), src); 440 } else { 441 try (ScratchRegister sc = masm.getScratchRegister()) { 442 AllocatableValue scratchRegisterValue = sc.getRegister().asValue(LIRKind.combine(input)); 443 AArch64Address src = loadStackSlotAddress(crb, masm, asStackSlot(input), scratchRegisterValue); 444 masm.fldr(size, asRegister(result), src); 445 } 446 } 447 } 448 449 private static void const2reg(CompilationResultBuilder crb, AArch64MacroAssembler masm, Value result, JavaConstant input) { 450 Register dst = asRegister(result); 451 switch (input.getJavaKind().getStackKind()) { 452 case Int: 453 final int value = input.asInt(); 454 int maskedValue; 455 switch (input.getJavaKind()) { 456 case Boolean: 457 case Byte: 458 maskedValue = value & 0xFF; 459 break; 460 case Char: 461 case Short: 462 maskedValue = value & 0xFFFF; 463 break; 464 case Int: 465 maskedValue = value; 466 break; 467 default: 468 throw GraalError.shouldNotReachHere(); 469 } 470 masm.mov(dst, maskedValue); 471 break; 472 case Long: 473 masm.mov(dst, input.asLong()); 474 break; 475 case Float: 476 if (AArch64MacroAssembler.isFloatImmediate(input.asFloat())) { 477 masm.fmov(32, dst, input.asFloat()); 478 } else if (crb.compilationResult.isImmutablePIC()) { 479 try (ScratchRegister scr = masm.getScratchRegister()) { 480 Register scratch = scr.getRegister(); 481 masm.mov(scratch, Float.floatToRawIntBits(input.asFloat())); 482 masm.fmov(32, dst, scratch); 483 } 484 } else { 485 masm.fldr(32, dst, (AArch64Address) crb.asFloatConstRef(input)); 486 } 487 break; 488 case Double: 489 if (AArch64MacroAssembler.isDoubleImmediate(input.asDouble())) { 490 masm.fmov(64, dst, input.asDouble()); 491 } else if (crb.compilationResult.isImmutablePIC()) { 492 try (ScratchRegister scr = masm.getScratchRegister()) { 493 Register scratch = scr.getRegister(); 494 masm.mov(scratch, Double.doubleToRawLongBits(input.asDouble())); 495 masm.fmov(64, dst, scratch); 496 } 497 } else { 498 masm.fldr(64, dst, (AArch64Address) crb.asDoubleConstRef(input)); 499 } 500 break; 501 case Object: 502 if (input.isNull()) { 503 masm.mov(dst, 0); 504 } else if (crb.target.inlineObjects) { 505 crb.recordInlineDataInCode(input); 506 masm.movNativeAddress(dst, 0xDEADDEADDEADDEADL); 507 } else { 508 masm.ldr(64, dst, (AArch64Address) crb.recordDataReferenceInCode(input, 8)); 509 } 510 break; 511 default: 512 throw GraalError.shouldNotReachHere("kind=" + input.getJavaKind().getStackKind()); 513 } 514 } 515 516 private static void const2stack(CompilationResultBuilder crb, AArch64MacroAssembler masm, Value result, JavaConstant constant) { 517 try (ScratchRegister addrReg = masm.getScratchRegister()) { 518 StackSlot slot = (StackSlot) result; 519 AArch64Address resultAddress = loadStackSlotAddress(crb, masm, slot, addrReg.getRegister()); 520 if (constant.isDefaultForKind() || constant.isNull()) { 521 emitStore(crb, masm, (AArch64Kind) result.getPlatformKind(), resultAddress, zr.asValue(LIRKind.combine(result))); 522 } else { 523 try (ScratchRegister sc = masm.getScratchRegister()) { 524 Value scratchRegisterValue = sc.getRegister().asValue(LIRKind.combine(result)); 525 const2reg(crb, masm, scratchRegisterValue, constant); 526 emitStore(crb, masm, (AArch64Kind) result.getPlatformKind(), resultAddress, scratchRegisterValue); 527 } 528 } 529 } 530 } 531 532 /** 533 * Returns AArch64Address of given StackSlot. We cannot use CompilationResultBuilder.asAddress 534 * since this calls AArch64MacroAssembler.makeAddress with displacements that may be larger than 535 * 9-bit signed, which cannot be handled by that method. 536 * 537 * Instead we create an address ourselves. We use scaled unsigned addressing since we know the 538 * transfersize, which gives us a 15-bit address range (for longs/doubles) respectively a 14-bit 539 * range (for everything else). 540 * 541 * @param scratch Scratch register that can be used to load address. If Value.ILLEGAL this 542 * instruction fails if we try to access a StackSlot that is too large to be loaded 543 * directly. 544 * @return AArch64Address of given StackSlot. Uses scratch register if necessary to do so. 545 */ 546 private static AArch64Address loadStackSlotAddress(CompilationResultBuilder crb, AArch64MacroAssembler masm, StackSlot slot, AllocatableValue scratch) { 547 Register scratchReg = Value.ILLEGAL.equals(scratch) ? zr : asRegister(scratch); 548 return loadStackSlotAddress(crb, masm, slot, scratchReg); 549 } 550 551 private static AArch64Address loadStackSlotAddress(CompilationResultBuilder crb, AArch64MacroAssembler masm, StackSlot slot, Register scratchReg) { 552 int displacement = crb.frameMap.offsetForStackSlot(slot); 553 int transferSize = slot.getPlatformKind().getSizeInBytes(); 554 return masm.makeAddress(sp, displacement, scratchReg, transferSize, /* allowOverwrite */false); 555 } 556 557 }