--- /dev/null 2017-01-22 10:16:57.869617664 -0800 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64Move.java 2017-02-15 17:05:00.242385666 -0800 @@ -0,0 +1,721 @@ +/* + * Copyright (c) 2011, 2016, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package org.graalvm.compiler.lir.amd64; + +import static org.graalvm.compiler.lir.LIRInstruction.OperandFlag.COMPOSITE; +import static org.graalvm.compiler.lir.LIRInstruction.OperandFlag.HINT; +import static org.graalvm.compiler.lir.LIRInstruction.OperandFlag.REG; +import static org.graalvm.compiler.lir.LIRInstruction.OperandFlag.STACK; +import static org.graalvm.compiler.lir.LIRInstruction.OperandFlag.UNINITIALIZED; +import static org.graalvm.compiler.lir.LIRValueUtil.asJavaConstant; +import static org.graalvm.compiler.lir.LIRValueUtil.isJavaConstant; +import static java.lang.Double.doubleToRawLongBits; +import static java.lang.Float.floatToRawIntBits; +import static jdk.vm.ci.code.ValueUtil.asRegister; +import static jdk.vm.ci.code.ValueUtil.isRegister; +import static jdk.vm.ci.code.ValueUtil.isStackSlot; + +import org.graalvm.compiler.asm.NumUtil; +import org.graalvm.compiler.asm.amd64.AMD64Address; +import org.graalvm.compiler.asm.amd64.AMD64Assembler.AMD64MIOp; +import org.graalvm.compiler.asm.amd64.AMD64Assembler.AMD64MOp; +import org.graalvm.compiler.asm.amd64.AMD64Assembler.OperandSize; +import org.graalvm.compiler.asm.amd64.AMD64MacroAssembler; +import org.graalvm.compiler.core.common.type.DataPointerConstant; +import org.graalvm.compiler.debug.GraalError; +import org.graalvm.compiler.lir.LIRFrameState; +import org.graalvm.compiler.lir.LIRInstructionClass; +import org.graalvm.compiler.lir.Opcode; +import org.graalvm.compiler.lir.StandardOp.LoadConstantOp; +import org.graalvm.compiler.lir.StandardOp.NullCheck; +import org.graalvm.compiler.lir.StandardOp.ValueMoveOp; +import org.graalvm.compiler.lir.VirtualStackSlot; +import org.graalvm.compiler.lir.asm.CompilationResultBuilder; + +import jdk.vm.ci.amd64.AMD64; +import jdk.vm.ci.amd64.AMD64Kind; +import jdk.vm.ci.code.Register; +import jdk.vm.ci.code.StackSlot; +import jdk.vm.ci.meta.AllocatableValue; +import jdk.vm.ci.meta.Constant; +import jdk.vm.ci.meta.JavaConstant; +import jdk.vm.ci.meta.Value; + +public class AMD64Move { + + private abstract static class AbstractMoveOp extends AMD64LIRInstruction implements ValueMoveOp { + public static final LIRInstructionClass TYPE = LIRInstructionClass.create(AbstractMoveOp.class); + + private AMD64Kind moveKind; + + protected AbstractMoveOp(LIRInstructionClass c, AMD64Kind moveKind) { + super(c); + this.moveKind = moveKind; + } + + @Override + public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) { + move(moveKind, crb, masm, getResult(), getInput()); + } + } + + @Opcode("MOVE") + public static final class MoveToRegOp extends AbstractMoveOp { + public static final LIRInstructionClass TYPE = LIRInstructionClass.create(MoveToRegOp.class); + + @Def({REG, HINT}) protected AllocatableValue result; + @Use({REG, STACK}) protected AllocatableValue input; + + public MoveToRegOp(AMD64Kind moveKind, AllocatableValue result, AllocatableValue input) { + super(TYPE, moveKind); + this.result = result; + this.input = input; + } + + @Override + public AllocatableValue getInput() { + return input; + } + + @Override + public AllocatableValue getResult() { + return result; + } + } + + @Opcode("MOVE") + public static final class MoveFromRegOp extends AbstractMoveOp { + public static final LIRInstructionClass TYPE = LIRInstructionClass.create(MoveFromRegOp.class); + + @Def({REG, STACK}) protected AllocatableValue result; + @Use({REG, HINT}) protected AllocatableValue input; + + public MoveFromRegOp(AMD64Kind moveKind, AllocatableValue result, AllocatableValue input) { + super(TYPE, moveKind); + this.result = result; + this.input = input; + } + + @Override + public AllocatableValue getInput() { + return input; + } + + @Override + public AllocatableValue getResult() { + return result; + } + } + + @Opcode("MOVE") + public static class MoveFromConstOp extends AMD64LIRInstruction implements LoadConstantOp { + public static final LIRInstructionClass TYPE = LIRInstructionClass.create(MoveFromConstOp.class); + + @Def({REG, STACK}) protected AllocatableValue result; + private final JavaConstant input; + + public MoveFromConstOp(AllocatableValue result, JavaConstant input) { + super(TYPE); + this.result = result; + this.input = input; + } + + @Override + public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) { + if (isRegister(result)) { + const2reg(crb, masm, asRegister(result), input); + } else { + assert isStackSlot(result); + const2stack(crb, masm, result, input); + } + } + + @Override + public Constant getConstant() { + return input; + } + + @Override + public AllocatableValue getResult() { + return result; + } + } + + @Opcode("STACKMOVE") + public static final class AMD64StackMove extends AMD64LIRInstruction implements ValueMoveOp { + public static final LIRInstructionClass TYPE = LIRInstructionClass.create(AMD64StackMove.class); + + @Def({STACK}) protected AllocatableValue result; + @Use({STACK, HINT}) protected AllocatableValue input; + @Alive({OperandFlag.STACK, OperandFlag.UNINITIALIZED}) private AllocatableValue backupSlot; + + private Register scratch; + + public AMD64StackMove(AllocatableValue result, AllocatableValue input, Register scratch, AllocatableValue backupSlot) { + super(TYPE); + this.result = result; + this.input = input; + this.backupSlot = backupSlot; + this.scratch = scratch; + } + + @Override + public AllocatableValue getInput() { + return input; + } + + @Override + public AllocatableValue getResult() { + return result; + } + + public Register getScratchRegister() { + return scratch; + } + + public AllocatableValue getBackupSlot() { + return backupSlot; + } + + @Override + public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) { + AMD64Kind backupKind = (AMD64Kind) backupSlot.getPlatformKind(); + if (backupKind.isXMM()) { + // graal doesn't use vector values, so it's safe to backup using DOUBLE + backupKind = AMD64Kind.DOUBLE; + } + + // backup scratch register + reg2stack(backupKind, crb, masm, backupSlot, scratch); + // move stack slot + stack2reg((AMD64Kind) getInput().getPlatformKind(), crb, masm, scratch, getInput()); + reg2stack((AMD64Kind) getResult().getPlatformKind(), crb, masm, getResult(), scratch); + // restore scratch register + stack2reg(backupKind, crb, masm, scratch, backupSlot); + } + } + + @Opcode("MULTISTACKMOVE") + public static final class AMD64MultiStackMove extends AMD64LIRInstruction { + public static final LIRInstructionClass TYPE = LIRInstructionClass.create(AMD64MultiStackMove.class); + + @Def({STACK}) protected AllocatableValue[] results; + @Use({STACK}) protected Value[] inputs; + @Alive({OperandFlag.STACK, OperandFlag.UNINITIALIZED}) private AllocatableValue backupSlot; + + private Register scratch; + + public AMD64MultiStackMove(AllocatableValue[] results, Value[] inputs, Register scratch, AllocatableValue backupSlot) { + super(TYPE); + this.results = results; + this.inputs = inputs; + this.backupSlot = backupSlot; + this.scratch = scratch; + } + + @Override + public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) { + AMD64Kind backupKind = (AMD64Kind) backupSlot.getPlatformKind(); + if (backupKind.isXMM()) { + // graal doesn't use vector values, so it's safe to backup using DOUBLE + backupKind = AMD64Kind.DOUBLE; + } + + // backup scratch register + move(backupKind, crb, masm, backupSlot, scratch.asValue(backupSlot.getValueKind())); + for (int i = 0; i < results.length; i++) { + Value input = inputs[i]; + AllocatableValue result = results[i]; + // move stack slot + move((AMD64Kind) input.getPlatformKind(), crb, masm, scratch.asValue(input.getValueKind()), input); + move((AMD64Kind) result.getPlatformKind(), crb, masm, result, scratch.asValue(result.getValueKind())); + } + // restore scratch register + move(backupKind, crb, masm, scratch.asValue(backupSlot.getValueKind()), backupSlot); + } + } + + @Opcode("STACKMOVE") + public static final class AMD64PushPopStackMove extends AMD64LIRInstruction implements ValueMoveOp { + public static final LIRInstructionClass TYPE = LIRInstructionClass.create(AMD64PushPopStackMove.class); + + @Def({STACK}) protected AllocatableValue result; + @Use({STACK, HINT}) protected AllocatableValue input; + private final OperandSize size; + + public AMD64PushPopStackMove(OperandSize size, AllocatableValue result, AllocatableValue input) { + super(TYPE); + this.result = result; + this.input = input; + this.size = size; + } + + @Override + public AllocatableValue getInput() { + return input; + } + + @Override + public AllocatableValue getResult() { + return result; + } + + @Override + public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) { + AMD64MOp.PUSH.emit(masm, size, (AMD64Address) crb.asAddress(input)); + AMD64MOp.POP.emit(masm, size, (AMD64Address) crb.asAddress(result)); + } + } + + public static final class LeaOp extends AMD64LIRInstruction { + public static final LIRInstructionClass TYPE = LIRInstructionClass.create(LeaOp.class); + + @Def({REG}) protected AllocatableValue result; + @Use({COMPOSITE, UNINITIALIZED}) protected AMD64AddressValue address; + + public LeaOp(AllocatableValue result, AMD64AddressValue address) { + super(TYPE); + this.result = result; + this.address = address; + } + + @Override + public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) { + masm.leaq(asRegister(result, AMD64Kind.QWORD), address.toAddress()); + } + } + + public static final class LeaDataOp extends AMD64LIRInstruction { + public static final LIRInstructionClass TYPE = LIRInstructionClass.create(LeaDataOp.class); + + @Def({REG}) protected AllocatableValue result; + private final DataPointerConstant data; + + public LeaDataOp(AllocatableValue result, DataPointerConstant data) { + super(TYPE); + this.result = result; + this.data = data; + } + + @Override + public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) { + masm.leaq(asRegister(result), (AMD64Address) crb.recordDataReferenceInCode(data)); + } + } + + public static final class StackLeaOp extends AMD64LIRInstruction { + public static final LIRInstructionClass TYPE = LIRInstructionClass.create(StackLeaOp.class); + + @Def({REG}) protected AllocatableValue result; + @Use({STACK, UNINITIALIZED}) protected AllocatableValue slot; + + public StackLeaOp(AllocatableValue result, AllocatableValue slot) { + super(TYPE); + this.result = result; + this.slot = slot; + assert slot instanceof VirtualStackSlot || slot instanceof StackSlot; + } + + @Override + public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) { + masm.leaq(asRegister(result, AMD64Kind.QWORD), (AMD64Address) crb.asAddress(slot)); + } + } + + public static final class MembarOp extends AMD64LIRInstruction { + public static final LIRInstructionClass TYPE = LIRInstructionClass.create(MembarOp.class); + + private final int barriers; + + public MembarOp(final int barriers) { + super(TYPE); + this.barriers = barriers; + } + + @Override + public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) { + masm.membar(barriers); + } + } + + public static final class NullCheckOp extends AMD64LIRInstruction implements NullCheck { + public static final LIRInstructionClass TYPE = LIRInstructionClass.create(NullCheckOp.class); + + @Use({COMPOSITE}) protected AMD64AddressValue address; + @State protected LIRFrameState state; + + public NullCheckOp(AMD64AddressValue address, LIRFrameState state) { + super(TYPE); + this.address = address; + this.state = state; + } + + @Override + public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) { + crb.recordImplicitException(masm.position(), state); + masm.nullCheck(address.toAddress()); + } + + @Override + public Value getCheckedValue() { + return address.base; + } + + @Override + public LIRFrameState getState() { + return state; + } + } + + @Opcode("CAS") + public static final class CompareAndSwapOp extends AMD64LIRInstruction { + public static final LIRInstructionClass TYPE = LIRInstructionClass.create(CompareAndSwapOp.class); + + private final AMD64Kind accessKind; + + @Def protected AllocatableValue result; + @Use({COMPOSITE}) protected AMD64AddressValue address; + @Use protected AllocatableValue cmpValue; + @Use protected AllocatableValue newValue; + + public CompareAndSwapOp(AMD64Kind accessKind, AllocatableValue result, AMD64AddressValue address, AllocatableValue cmpValue, AllocatableValue newValue) { + super(TYPE); + this.accessKind = accessKind; + this.result = result; + this.address = address; + this.cmpValue = cmpValue; + this.newValue = newValue; + } + + @Override + public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) { + assert asRegister(cmpValue).equals(AMD64.rax) && asRegister(result).equals(AMD64.rax); + + if (crb.target.isMP) { + masm.lock(); + } + switch (accessKind) { + case DWORD: + masm.cmpxchgl(asRegister(newValue), address.toAddress()); + break; + case QWORD: + masm.cmpxchgq(asRegister(newValue), address.toAddress()); + break; + default: + throw GraalError.shouldNotReachHere(); + } + } + } + + @Opcode("ATOMIC_READ_AND_ADD") + public static final class AtomicReadAndAddOp extends AMD64LIRInstruction { + public static final LIRInstructionClass TYPE = LIRInstructionClass.create(AtomicReadAndAddOp.class); + + private final AMD64Kind accessKind; + + @Def protected AllocatableValue result; + @Alive({COMPOSITE}) protected AMD64AddressValue address; + @Use protected AllocatableValue delta; + + public AtomicReadAndAddOp(AMD64Kind accessKind, AllocatableValue result, AMD64AddressValue address, AllocatableValue delta) { + super(TYPE); + this.accessKind = accessKind; + this.result = result; + this.address = address; + this.delta = delta; + } + + @Override + public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) { + move(accessKind, crb, masm, result, delta); + if (crb.target.isMP) { + masm.lock(); + } + switch (accessKind) { + case DWORD: + masm.xaddl(address.toAddress(), asRegister(result)); + break; + case QWORD: + masm.xaddq(address.toAddress(), asRegister(result)); + break; + default: + throw GraalError.shouldNotReachHere(); + } + } + } + + @Opcode("ATOMIC_READ_AND_WRITE") + public static final class AtomicReadAndWriteOp extends AMD64LIRInstruction { + public static final LIRInstructionClass TYPE = LIRInstructionClass.create(AtomicReadAndWriteOp.class); + + private final AMD64Kind accessKind; + + @Def protected AllocatableValue result; + @Alive({COMPOSITE}) protected AMD64AddressValue address; + @Use protected AllocatableValue newValue; + + public AtomicReadAndWriteOp(AMD64Kind accessKind, AllocatableValue result, AMD64AddressValue address, AllocatableValue newValue) { + super(TYPE); + this.accessKind = accessKind; + this.result = result; + this.address = address; + this.newValue = newValue; + } + + @Override + public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) { + move(accessKind, crb, masm, result, newValue); + switch (accessKind) { + case DWORD: + masm.xchgl(asRegister(result), address.toAddress()); + break; + case QWORD: + masm.xchgq(asRegister(result), address.toAddress()); + break; + default: + throw GraalError.shouldNotReachHere(); + } + } + } + + public static void move(CompilationResultBuilder crb, AMD64MacroAssembler masm, Value result, Value input) { + move((AMD64Kind) result.getPlatformKind(), crb, masm, result, input); + } + + public static void move(AMD64Kind moveKind, CompilationResultBuilder crb, AMD64MacroAssembler masm, Value result, Value input) { + if (isRegister(input)) { + if (isRegister(result)) { + reg2reg(moveKind, masm, result, input); + } else if (isStackSlot(result)) { + reg2stack(moveKind, crb, masm, result, asRegister(input)); + } else { + throw GraalError.shouldNotReachHere(); + } + } else if (isStackSlot(input)) { + if (isRegister(result)) { + stack2reg(moveKind, crb, masm, asRegister(result), input); + } else { + throw GraalError.shouldNotReachHere(); + } + } else if (isJavaConstant(input)) { + if (isRegister(result)) { + const2reg(crb, masm, asRegister(result), asJavaConstant(input)); + } else if (isStackSlot(result)) { + const2stack(crb, masm, result, asJavaConstant(input)); + } else { + throw GraalError.shouldNotReachHere(); + } + } else { + throw GraalError.shouldNotReachHere(); + } + } + + private static void reg2reg(AMD64Kind kind, AMD64MacroAssembler masm, Value result, Value input) { + if (asRegister(input).equals(asRegister(result))) { + return; + } + switch (kind) { + case BYTE: + case WORD: + case DWORD: + masm.movl(asRegister(result), asRegister(input)); + break; + case QWORD: + masm.movq(asRegister(result), asRegister(input)); + break; + case SINGLE: + masm.movflt(asRegister(result, AMD64Kind.SINGLE), asRegister(input, AMD64Kind.SINGLE)); + break; + case DOUBLE: + masm.movdbl(asRegister(result, AMD64Kind.DOUBLE), asRegister(input, AMD64Kind.DOUBLE)); + break; + default: + throw GraalError.shouldNotReachHere("kind=" + kind); + } + } + + public static void reg2stack(AMD64Kind kind, CompilationResultBuilder crb, AMD64MacroAssembler masm, Value result, Register input) { + AMD64Address dest = (AMD64Address) crb.asAddress(result); + switch (kind) { + case BYTE: + masm.movb(dest, input); + break; + case WORD: + masm.movw(dest, input); + break; + case DWORD: + masm.movl(dest, input); + break; + case QWORD: + masm.movq(dest, input); + break; + case SINGLE: + masm.movflt(dest, input); + break; + case DOUBLE: + masm.movsd(dest, input); + break; + default: + throw GraalError.shouldNotReachHere(); + } + } + + public static void stack2reg(AMD64Kind kind, CompilationResultBuilder crb, AMD64MacroAssembler masm, Register result, Value input) { + AMD64Address src = (AMD64Address) crb.asAddress(input); + switch (kind) { + case BYTE: + masm.movsbl(result, src); + break; + case WORD: + masm.movswl(result, src); + break; + case DWORD: + masm.movl(result, src); + break; + case QWORD: + masm.movq(result, src); + break; + case SINGLE: + masm.movflt(result, src); + break; + case DOUBLE: + masm.movdbl(result, src); + break; + default: + throw GraalError.shouldNotReachHere(); + } + } + + public static void const2reg(CompilationResultBuilder crb, AMD64MacroAssembler masm, Register result, JavaConstant input) { + /* + * Note: we use the kind of the input operand (and not the kind of the result operand) + * because they don't match in all cases. For example, an object constant can be loaded to a + * long register when unsafe casts occurred (e.g., for a write barrier where arithmetic + * operations are then performed on the pointer). + */ + switch (input.getJavaKind().getStackKind()) { + case Int: + // Do not optimize with an XOR as this instruction may be between + // a CMP and a Jcc in which case the XOR will modify the condition + // flags and interfere with the Jcc. + masm.movl(result, input.asInt()); + + break; + case Long: + // Do not optimize with an XOR as this instruction may be between + // a CMP and a Jcc in which case the XOR will modify the condition + // flags and interfere with the Jcc. + if (input.asLong() == (int) input.asLong()) { + // Sign extended to long + masm.movslq(result, (int) input.asLong()); + } else if ((input.asLong() & 0xFFFFFFFFL) == input.asLong()) { + // Zero extended to long + masm.movl(result, (int) input.asLong()); + } else { + masm.movq(result, input.asLong()); + } + break; + case Float: + // This is *not* the same as 'constant == 0.0f' in the case where constant is -0.0f + if (Float.floatToRawIntBits(input.asFloat()) == Float.floatToRawIntBits(0.0f)) { + masm.xorps(result, result); + } else { + masm.movflt(result, (AMD64Address) crb.asFloatConstRef(input)); + } + break; + case Double: + // This is *not* the same as 'constant == 0.0d' in the case where constant is -0.0d + if (Double.doubleToRawLongBits(input.asDouble()) == Double.doubleToRawLongBits(0.0d)) { + masm.xorpd(result, result); + } else { + masm.movdbl(result, (AMD64Address) crb.asDoubleConstRef(input)); + } + break; + case Object: + // Do not optimize with an XOR as this instruction may be between + // a CMP and a Jcc in which case the XOR will modify the condition + // flags and interfere with the Jcc. + if (input.isNull()) { + masm.movq(result, 0x0L); + } else if (crb.target.inlineObjects) { + crb.recordInlineDataInCode(input); + masm.movq(result, 0xDEADDEADDEADDEADL); + } else { + masm.movq(result, (AMD64Address) crb.recordDataReferenceInCode(input, 0)); + } + break; + default: + throw GraalError.shouldNotReachHere(); + } + } + + public static void const2stack(CompilationResultBuilder crb, AMD64MacroAssembler masm, Value result, JavaConstant input) { + AMD64Address dest = (AMD64Address) crb.asAddress(result); + final long imm; + switch (input.getJavaKind().getStackKind()) { + case Int: + imm = input.asInt(); + break; + case Long: + imm = input.asLong(); + break; + case Float: + imm = floatToRawIntBits(input.asFloat()); + break; + case Double: + imm = doubleToRawLongBits(input.asDouble()); + break; + case Object: + if (input.isNull()) { + imm = 0; + } else { + throw GraalError.shouldNotReachHere("Non-null object constants must be in register"); + } + break; + default: + throw GraalError.shouldNotReachHere(); + } + + switch ((AMD64Kind) result.getPlatformKind()) { + case BYTE: + assert NumUtil.isByte(imm) : "Is not in byte range: " + imm; + AMD64MIOp.MOVB.emit(masm, OperandSize.BYTE, dest, (int) imm); + break; + case WORD: + assert NumUtil.isShort(imm) : "Is not in short range: " + imm; + AMD64MIOp.MOV.emit(masm, OperandSize.WORD, dest, (int) imm); + break; + case DWORD: + case SINGLE: + assert NumUtil.isInt(imm) : "Is not in int range: " + imm; + masm.movl(dest, (int) imm); + break; + case QWORD: + case DOUBLE: + masm.movlong(dest, imm); + break; + default: + throw GraalError.shouldNotReachHere("Unknown result Kind: " + result.getPlatformKind()); + } + } +}