1 /*
   2  * Copyright (c) 2009, 2018, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  */
  23 
  24 
  25 
  26 package org.graalvm.compiler.core.amd64;
  27 
  28 import static org.graalvm.compiler.asm.amd64.AMD64Assembler.AMD64BinaryArithmetic.ADD;
  29 import static org.graalvm.compiler.asm.amd64.AMD64Assembler.AMD64BinaryArithmetic.AND;
  30 import static org.graalvm.compiler.asm.amd64.AMD64Assembler.AMD64BinaryArithmetic.OR;
  31 import static org.graalvm.compiler.asm.amd64.AMD64Assembler.AMD64BinaryArithmetic.SUB;
  32 import static org.graalvm.compiler.asm.amd64.AMD64Assembler.AMD64BinaryArithmetic.XOR;
  33 import static org.graalvm.compiler.asm.amd64.AMD64Assembler.AMD64RMOp.MOVSX;
  34 import static org.graalvm.compiler.asm.amd64.AMD64Assembler.AMD64RMOp.MOVSXB;
  35 import static org.graalvm.compiler.asm.amd64.AMD64Assembler.AMD64RMOp.MOVSXD;
  36 import static org.graalvm.compiler.asm.amd64.AMD64BaseAssembler.OperandSize.DWORD;
  37 import static org.graalvm.compiler.asm.amd64.AMD64BaseAssembler.OperandSize.QWORD;
  38 import static org.graalvm.compiler.asm.amd64.AMD64BaseAssembler.OperandSize.SD;
  39 import static org.graalvm.compiler.asm.amd64.AMD64BaseAssembler.OperandSize.SS;
  40 
  41 import org.graalvm.compiler.asm.amd64.AMD64Assembler.AMD64MIOp;
  42 import org.graalvm.compiler.asm.amd64.AMD64Assembler.AMD64RMOp;
  43 import org.graalvm.compiler.asm.amd64.AMD64Assembler.SSEOp;
  44 import org.graalvm.compiler.asm.amd64.AMD64BaseAssembler.OperandSize;
  45 import org.graalvm.compiler.core.common.LIRKind;
  46 import org.graalvm.compiler.core.common.NumUtil;
  47 import org.graalvm.compiler.core.common.calc.CanonicalCondition;
  48 import org.graalvm.compiler.core.common.calc.Condition;
  49 import org.graalvm.compiler.core.gen.NodeLIRBuilder;
  50 import org.graalvm.compiler.core.gen.NodeMatchRules;
  51 import org.graalvm.compiler.core.match.ComplexMatchResult;
  52 import org.graalvm.compiler.core.match.MatchRule;
  53 import org.graalvm.compiler.debug.GraalError;
  54 import org.graalvm.compiler.lir.LIRFrameState;
  55 import org.graalvm.compiler.lir.LIRValueUtil;
  56 import org.graalvm.compiler.lir.LabelRef;
  57 import org.graalvm.compiler.lir.amd64.AMD64AddressValue;
  58 import org.graalvm.compiler.lir.amd64.AMD64BinaryConsumer;
  59 import org.graalvm.compiler.lir.amd64.AMD64ControlFlow.BranchOp;
  60 import org.graalvm.compiler.lir.gen.LIRGeneratorTool;
  61 import org.graalvm.compiler.nodes.ConstantNode;
  62 import org.graalvm.compiler.nodes.DeoptimizingNode;
  63 import org.graalvm.compiler.nodes.IfNode;
  64 import org.graalvm.compiler.nodes.NodeView;
  65 import org.graalvm.compiler.nodes.ValueNode;
  66 import org.graalvm.compiler.nodes.calc.CompareNode;
  67 import org.graalvm.compiler.nodes.calc.FloatConvertNode;
  68 import org.graalvm.compiler.nodes.calc.LeftShiftNode;
  69 import org.graalvm.compiler.nodes.calc.NarrowNode;
  70 import org.graalvm.compiler.nodes.calc.ReinterpretNode;
  71 import org.graalvm.compiler.nodes.calc.SignExtendNode;
  72 import org.graalvm.compiler.nodes.calc.UnsignedRightShiftNode;
  73 import org.graalvm.compiler.nodes.calc.ZeroExtendNode;
  74 import org.graalvm.compiler.nodes.java.LogicCompareAndSwapNode;
  75 import org.graalvm.compiler.nodes.java.ValueCompareAndSwapNode;
  76 import org.graalvm.compiler.nodes.memory.Access;
  77 import org.graalvm.compiler.nodes.memory.LIRLowerableAccess;
  78 import org.graalvm.compiler.nodes.memory.WriteNode;
  79 import org.graalvm.compiler.nodes.util.GraphUtil;
  80 
  81 import jdk.vm.ci.amd64.AMD64;
  82 import jdk.vm.ci.amd64.AMD64.CPUFeature;
  83 import jdk.vm.ci.amd64.AMD64Kind;
  84 import jdk.vm.ci.meta.AllocatableValue;
  85 import jdk.vm.ci.meta.JavaConstant;
  86 import jdk.vm.ci.meta.JavaKind;
  87 import jdk.vm.ci.meta.PlatformKind;
  88 import jdk.vm.ci.meta.Value;
  89 import jdk.vm.ci.meta.ValueKind;
  90 
  91 public class AMD64NodeMatchRules extends NodeMatchRules {
  92 
  93     public AMD64NodeMatchRules(LIRGeneratorTool gen) {
  94         super(gen);
  95     }
  96 
  97     protected LIRFrameState getState(Access access) {
  98         if (access instanceof DeoptimizingNode) {
  99             return state((DeoptimizingNode) access);
 100         }
 101         return null;
 102     }
 103 
 104     protected AMD64Kind getMemoryKind(LIRLowerableAccess access) {
 105         return (AMD64Kind) getLirKind(access).getPlatformKind();
 106     }
 107 
 108     protected LIRKind getLirKind(LIRLowerableAccess access) {
 109         return gen.getLIRKind(access.getAccessStamp());
 110     }
 111 
 112     protected OperandSize getMemorySize(LIRLowerableAccess access) {
 113         switch (getMemoryKind(access)) {
 114             case BYTE:
 115                 return OperandSize.BYTE;
 116             case WORD:
 117                 return OperandSize.WORD;
 118             case DWORD:
 119                 return OperandSize.DWORD;
 120             case QWORD:
 121                 return OperandSize.QWORD;
 122             case SINGLE:
 123                 return OperandSize.SS;
 124             case DOUBLE:
 125                 return OperandSize.SD;
 126             default:
 127                 throw GraalError.shouldNotReachHere("unsupported memory access type " + getMemoryKind(access));
 128         }
 129     }
 130 
 131     protected ComplexMatchResult emitCompareBranchMemory(IfNode ifNode, CompareNode compare, ValueNode value, LIRLowerableAccess access) {
 132         Condition cond = compare.condition().asCondition();
 133         AMD64Kind kind = getMemoryKind(access);
 134         boolean matchedAsConstant = false; // For assertion checking
 135 
 136         if (value.isConstant()) {
 137             JavaConstant constant = value.asJavaConstant();
 138             if (constant != null) {
 139                 if (kind == AMD64Kind.QWORD && !constant.getJavaKind().isObject() && !NumUtil.isInt(constant.asLong())) {
 140                     // Only imm32 as long
 141                     return null;
 142                 }
 143                 // A QWORD that can be encoded as int can be embedded as a constant
 144                 matchedAsConstant = kind == AMD64Kind.QWORD && !constant.getJavaKind().isObject() && NumUtil.isInt(constant.asLong());
 145             }
 146             if (kind == AMD64Kind.DWORD) {
 147                 // Any DWORD value should be embeddable as a constant
 148                 matchedAsConstant = true;
 149             }
 150             if (kind.isXMM()) {
 151                 ifNode.getDebug().log("Skipping constant compares for float kinds");
 152                 return null;
 153             }
 154         }
 155         boolean matchedAsConstantFinal = matchedAsConstant;
 156 
 157         /*
 158          * emitCompareBranchMemory expects the memory on the right, so mirror the condition if
 159          * that's not true. It might be mirrored again the actual compare is emitted but that's ok.
 160          */
 161         Condition finalCondition = GraphUtil.unproxify(compare.getX()) == access ? cond.mirror() : cond;
 162         return new ComplexMatchResult() {
 163             @Override
 164             public Value evaluate(NodeLIRBuilder builder) {
 165                 LabelRef trueLabel = getLIRBlock(ifNode.trueSuccessor());
 166                 LabelRef falseLabel = getLIRBlock(ifNode.falseSuccessor());
 167                 boolean unorderedIsTrue = compare.unorderedIsTrue();
 168                 double trueLabelProbability = ifNode.probability(ifNode.trueSuccessor());
 169                 Value other = operand(value);
 170                 /*
 171                  * Check that patterns which were matched as a constant actually end up seeing a
 172                  * constant in the LIR.
 173                  */
 174                 assert !matchedAsConstantFinal || !LIRValueUtil.isVariable(other) : "expected constant value " + value;
 175                 AMD64AddressValue address = (AMD64AddressValue) operand(access.getAddress());
 176                 getLIRGeneratorTool().emitCompareBranchMemory(kind, other, address, getState(access), finalCondition, unorderedIsTrue, trueLabel, falseLabel, trueLabelProbability);
 177                 return null;
 178             }
 179         };
 180     }
 181 
 182     private ComplexMatchResult emitIntegerTestBranchMemory(IfNode x, ValueNode value, LIRLowerableAccess access) {
 183         LabelRef trueLabel = getLIRBlock(x.trueSuccessor());
 184         LabelRef falseLabel = getLIRBlock(x.falseSuccessor());
 185         double trueLabelProbability = x.probability(x.trueSuccessor());
 186         AMD64Kind kind = getMemoryKind(access);
 187         OperandSize size = kind == AMD64Kind.QWORD ? QWORD : DWORD;
 188         if (value.isConstant()) {
 189             JavaConstant constant = value.asJavaConstant();
 190             if (constant != null && kind == AMD64Kind.QWORD && !NumUtil.isInt(constant.asLong())) {
 191                 // Only imm32 as long
 192                 return null;
 193             }
 194             return builder -> {
 195                 AMD64AddressValue address = (AMD64AddressValue) operand(access.getAddress());
 196                 gen.append(new AMD64BinaryConsumer.MemoryConstOp(AMD64MIOp.TEST, size, address, (int) constant.asLong(), getState(access)));
 197                 gen.append(new BranchOp(Condition.EQ, trueLabel, falseLabel, trueLabelProbability));
 198                 return null;
 199             };
 200         } else {
 201             return builder -> {
 202                 AMD64AddressValue address = (AMD64AddressValue) operand(access.getAddress());
 203                 gen.append(new AMD64BinaryConsumer.MemoryRMOp(AMD64RMOp.TEST, size, gen.asAllocatable(operand(value)), address, getState(access)));
 204                 gen.append(new BranchOp(Condition.EQ, trueLabel, falseLabel, trueLabelProbability));
 205                 return null;
 206             };
 207         }
 208     }
 209 
 210     protected ComplexMatchResult emitConvertMemoryOp(PlatformKind kind, AMD64RMOp op, OperandSize size, Access access, ValueKind<?> addressKind) {
 211         return builder -> {
 212             AMD64AddressValue address = (AMD64AddressValue) operand(access.getAddress());
 213             LIRFrameState state = getState(access);
 214             if (addressKind != null) {
 215                 address = address.withKind(addressKind);
 216             }
 217             return getArithmeticLIRGenerator().emitConvertMemoryOp(kind, op, size, address, state);
 218         };
 219     }
 220 
 221     protected ComplexMatchResult emitConvertMemoryOp(PlatformKind kind, AMD64RMOp op, OperandSize size, Access access) {
 222         return emitConvertMemoryOp(kind, op, size, access, null);
 223     }
 224 
 225     private ComplexMatchResult emitSignExtendMemory(Access access, int fromBits, int toBits, ValueKind<?> addressKind) {
 226         assert fromBits <= toBits && toBits <= 64;
 227         AMD64Kind kind = null;
 228         AMD64RMOp op;
 229         OperandSize size;
 230         if (fromBits == toBits) {
 231             return null;
 232         } else if (toBits > 32) {
 233             kind = AMD64Kind.QWORD;
 234             size = OperandSize.QWORD;
 235             // sign extend to 64 bits
 236             switch (fromBits) {
 237                 case 8:
 238                     op = MOVSXB;
 239                     break;
 240                 case 16:
 241                     op = MOVSX;
 242                     break;
 243                 case 32:
 244                     op = MOVSXD;
 245                     break;
 246                 default:
 247                     throw GraalError.unimplemented("unsupported sign extension (" + fromBits + " bit -> " + toBits + " bit)");
 248             }
 249         } else {
 250             kind = AMD64Kind.DWORD;
 251             size = OperandSize.DWORD;
 252             // sign extend to 32 bits (smaller values are internally represented as 32 bit values)
 253             switch (fromBits) {
 254                 case 8:
 255                     op = MOVSXB;
 256                     break;
 257                 case 16:
 258                     op = MOVSX;
 259                     break;
 260                 case 32:
 261                     return null;
 262                 default:
 263                     throw GraalError.unimplemented("unsupported sign extension (" + fromBits + " bit -> " + toBits + " bit)");
 264             }
 265         }
 266         if (kind != null && op != null) {
 267             return emitConvertMemoryOp(kind, op, size, access, addressKind);
 268         }
 269         return null;
 270     }
 271 
 272     private Value emitReinterpretMemory(LIRKind to, Access access) {
 273         AMD64AddressValue address = (AMD64AddressValue) operand(access.getAddress());
 274         LIRFrameState state = getState(access);
 275         return getArithmeticLIRGenerator().emitLoad(to, address, state);
 276     }
 277 
 278     private boolean supports(CPUFeature feature) {
 279         return ((AMD64) getLIRGeneratorTool().target().arch).getFeatures().contains(feature);
 280     }
 281 
 282     @MatchRule("(And (Not a) b)")
 283     public ComplexMatchResult logicalAndNot(ValueNode a, ValueNode b) {
 284         if (!supports(CPUFeature.BMI1)) {
 285             return null;
 286         }
 287         return builder -> getArithmeticLIRGenerator().emitLogicalAndNot(operand(a), operand(b));
 288     }
 289 
 290     @MatchRule("(And a (Negate a))")
 291     public ComplexMatchResult lowestSetIsolatedBit(ValueNode a) {
 292         if (!supports(CPUFeature.BMI1)) {
 293             return null;
 294         }
 295         return builder -> getArithmeticLIRGenerator().emitLowestSetIsolatedBit(operand(a));
 296     }
 297 
 298     @MatchRule("(Xor a (Add a b))")
 299     public ComplexMatchResult getMaskUpToLowestSetBit(ValueNode a, ValueNode b) {
 300         if (!supports(CPUFeature.BMI1)) {
 301             return null;
 302         }
 303 
 304         // Make sure that the pattern matches a subtraction by one.
 305         if (!b.isJavaConstant()) {
 306             return null;
 307         }
 308 
 309         JavaConstant bCst = b.asJavaConstant();
 310         long bValue;
 311         if (bCst.getJavaKind() == JavaKind.Int) {
 312             bValue = bCst.asInt();
 313         } else if (bCst.getJavaKind() == JavaKind.Long) {
 314             bValue = bCst.asLong();
 315         } else {
 316             return null;
 317         }
 318 
 319         if (bValue == -1) {
 320             return builder -> getArithmeticLIRGenerator().emitGetMaskUpToLowestSetBit(operand(a));
 321         } else {
 322             return null;
 323         }
 324     }
 325 
 326     @MatchRule("(And a (Add a b))")
 327     public ComplexMatchResult resetLowestSetBit(ValueNode a, ValueNode b) {
 328         if (!supports(CPUFeature.BMI1)) {
 329             return null;
 330         }
 331         // Make sure that the pattern matches a subtraction by one.
 332         if (!b.isJavaConstant()) {
 333             return null;
 334         }
 335 
 336         JavaConstant bCst = b.asJavaConstant();
 337         long bValue;
 338         if (bCst.getJavaKind() == JavaKind.Int) {
 339             bValue = bCst.asInt();
 340         } else if (bCst.getJavaKind() == JavaKind.Long) {
 341             bValue = bCst.asLong();
 342         } else {
 343             return null;
 344         }
 345 
 346         if (bValue == -1) {
 347             return builder -> getArithmeticLIRGenerator().emitResetLowestSetBit(operand(a));
 348         } else {
 349             return null;
 350         }
 351     }
 352 
 353     @MatchRule("(If (IntegerTest Read=access value))")
 354     @MatchRule("(If (IntegerTest FloatingRead=access value))")
 355     public ComplexMatchResult integerTestBranchMemory(IfNode root, LIRLowerableAccess access, ValueNode value) {
 356         return emitIntegerTestBranchMemory(root, value, access);
 357     }
 358 
 359     @MatchRule("(If (IntegerEquals=compare value Read=access))")
 360     @MatchRule("(If (IntegerLessThan=compare value Read=access))")
 361     @MatchRule("(If (IntegerBelow=compare value Read=access))")
 362     @MatchRule("(If (IntegerEquals=compare value FloatingRead=access))")
 363     @MatchRule("(If (IntegerLessThan=compare value FloatingRead=access))")
 364     @MatchRule("(If (IntegerBelow=compare value FloatingRead=access))")
 365     @MatchRule("(If (FloatEquals=compare value Read=access))")
 366     @MatchRule("(If (FloatEquals=compare value FloatingRead=access))")
 367     @MatchRule("(If (FloatLessThan=compare value Read=access))")
 368     @MatchRule("(If (FloatLessThan=compare value FloatingRead=access))")
 369     @MatchRule("(If (PointerEquals=compare value Read=access))")
 370     @MatchRule("(If (PointerEquals=compare value FloatingRead=access))")
 371     @MatchRule("(If (ObjectEquals=compare value Read=access))")
 372     @MatchRule("(If (ObjectEquals=compare value FloatingRead=access))")
 373     public ComplexMatchResult ifCompareMemory(IfNode root, CompareNode compare, ValueNode value, LIRLowerableAccess access) {
 374         return emitCompareBranchMemory(root, compare, value, access);
 375     }
 376 
 377     @MatchRule("(If (ObjectEquals=compare value ValueCompareAndSwap=cas))")
 378     @MatchRule("(If (PointerEquals=compare value ValueCompareAndSwap=cas))")
 379     @MatchRule("(If (FloatEquals=compare value ValueCompareAndSwap=cas))")
 380     @MatchRule("(If (IntegerEquals=compare value ValueCompareAndSwap=cas))")
 381     public ComplexMatchResult ifCompareValueCas(IfNode root, CompareNode compare, ValueNode value, ValueCompareAndSwapNode cas) {
 382         assert compare.condition() == CanonicalCondition.EQ;
 383         if (value == cas.getExpectedValue() && cas.usages().count() == 1) {
 384             return builder -> {
 385                 LIRKind kind = getLirKind(cas);
 386                 LabelRef trueLabel = getLIRBlock(root.trueSuccessor());
 387                 LabelRef falseLabel = getLIRBlock(root.falseSuccessor());
 388                 double trueLabelProbability = root.probability(root.trueSuccessor());
 389                 Value expectedValue = operand(cas.getExpectedValue());
 390                 Value newValue = operand(cas.getNewValue());
 391                 AMD64AddressValue address = (AMD64AddressValue) operand(cas.getAddress());
 392                 getLIRGeneratorTool().emitCompareAndSwapBranch(kind, address, expectedValue, newValue, Condition.EQ, trueLabel, falseLabel, trueLabelProbability);
 393                 return null;
 394             };
 395         }
 396         return null;
 397     }
 398 
 399     @MatchRule("(If (ObjectEquals=compare value LogicCompareAndSwap=cas))")
 400     @MatchRule("(If (PointerEquals=compare value LogicCompareAndSwap=cas))")
 401     @MatchRule("(If (FloatEquals=compare value LogicCompareAndSwap=cas))")
 402     @MatchRule("(If (IntegerEquals=compare value LogicCompareAndSwap=cas))")
 403     public ComplexMatchResult ifCompareLogicCas(IfNode root, CompareNode compare, ValueNode value, LogicCompareAndSwapNode cas) {
 404         JavaConstant constant = value.asJavaConstant();
 405         assert compare.condition() == CanonicalCondition.EQ;
 406         if (constant != null && cas.usages().count() == 1) {
 407             long constantValue = constant.asLong();
 408             boolean successIsTrue;
 409             if (constantValue == 0) {
 410                 successIsTrue = false;
 411             } else if (constantValue == 1) {
 412                 successIsTrue = true;
 413             } else {
 414                 return null;
 415             }
 416             return builder -> {
 417                 LIRKind kind = getLirKind(cas);
 418                 LabelRef trueLabel = getLIRBlock(root.trueSuccessor());
 419                 LabelRef falseLabel = getLIRBlock(root.falseSuccessor());
 420                 double trueLabelProbability = root.probability(root.trueSuccessor());
 421                 Value expectedValue = operand(cas.getExpectedValue());
 422                 Value newValue = operand(cas.getNewValue());
 423                 AMD64AddressValue address = (AMD64AddressValue) operand(cas.getAddress());
 424                 Condition condition = successIsTrue ? Condition.EQ : Condition.NE;
 425                 getLIRGeneratorTool().emitCompareAndSwapBranch(kind, address, expectedValue, newValue, condition, trueLabel, falseLabel, trueLabelProbability);
 426                 return null;
 427             };
 428         }
 429         return null;
 430     }
 431 
 432     @MatchRule("(If (ObjectEquals=compare value FloatingRead=access))")
 433     public ComplexMatchResult ifLogicCas(IfNode root, CompareNode compare, ValueNode value, LIRLowerableAccess access) {
 434         return emitCompareBranchMemory(root, compare, value, access);
 435     }
 436 
 437     @MatchRule("(Or (LeftShift=lshift value Constant) (UnsignedRightShift=rshift value Constant))")
 438     public ComplexMatchResult rotateLeftConstant(LeftShiftNode lshift, UnsignedRightShiftNode rshift) {
 439         if ((lshift.getShiftAmountMask() & (lshift.getY().asJavaConstant().asInt() + rshift.getY().asJavaConstant().asInt())) == 0) {
 440             return builder -> getArithmeticLIRGenerator().emitRol(operand(lshift.getX()), operand(lshift.getY()));
 441         }
 442         return null;
 443     }
 444 
 445     @MatchRule("(Or (LeftShift value (Sub Constant=delta shiftAmount)) (UnsignedRightShift value shiftAmount))")
 446     public ComplexMatchResult rotateRightVariable(ValueNode value, ConstantNode delta, ValueNode shiftAmount) {
 447         if (delta.asJavaConstant().asLong() == 0 || delta.asJavaConstant().asLong() == 32) {
 448             return builder -> getArithmeticLIRGenerator().emitRor(operand(value), operand(shiftAmount));
 449         }
 450         return null;
 451     }
 452 
 453     @MatchRule("(Or (LeftShift value shiftAmount) (UnsignedRightShift value (Sub Constant=delta shiftAmount)))")
 454     public ComplexMatchResult rotateLeftVariable(ValueNode value, ValueNode shiftAmount, ConstantNode delta) {
 455         if (delta.asJavaConstant().asLong() == 0 || delta.asJavaConstant().asLong() == 32) {
 456             return builder -> getArithmeticLIRGenerator().emitRol(operand(value), operand(shiftAmount));
 457         }
 458         return null;
 459     }
 460 
 461     private ComplexMatchResult binaryRead(AMD64RMOp op, OperandSize size, ValueNode value, LIRLowerableAccess access) {
 462         return builder -> getArithmeticLIRGenerator().emitBinaryMemory(op, size, getLIRGeneratorTool().asAllocatable(operand(value)), (AMD64AddressValue) operand(access.getAddress()),
 463                         getState(access));
 464     }
 465 
 466     @MatchRule("(Add value Read=access)")
 467     @MatchRule("(Add value FloatingRead=access)")
 468     public ComplexMatchResult addMemory(ValueNode value, LIRLowerableAccess access) {
 469         OperandSize size = getMemorySize(access);
 470         if (size.isXmmType()) {
 471             return binaryRead(SSEOp.ADD, size, value, access);
 472         } else {
 473             return binaryRead(ADD.getRMOpcode(size), size, value, access);
 474         }
 475     }
 476 
 477     @MatchRule("(Sub value Read=access)")
 478     @MatchRule("(Sub value FloatingRead=access)")
 479     public ComplexMatchResult subMemory(ValueNode value, LIRLowerableAccess access) {
 480         OperandSize size = getMemorySize(access);
 481         if (size.isXmmType()) {
 482             return binaryRead(SSEOp.SUB, size, value, access);
 483         } else {
 484             return binaryRead(SUB.getRMOpcode(size), size, value, access);
 485         }
 486     }
 487 
 488     @MatchRule("(Mul value Read=access)")
 489     @MatchRule("(Mul value FloatingRead=access)")
 490     public ComplexMatchResult mulMemory(ValueNode value, LIRLowerableAccess access) {
 491         OperandSize size = getMemorySize(access);
 492         if (size.isXmmType()) {
 493             return binaryRead(SSEOp.MUL, size, value, access);
 494         } else {
 495             return binaryRead(AMD64RMOp.IMUL, size, value, access);
 496         }
 497     }
 498 
 499     @MatchRule("(And value Read=access)")
 500     @MatchRule("(And value FloatingRead=access)")
 501     public ComplexMatchResult andMemory(ValueNode value, LIRLowerableAccess access) {
 502         OperandSize size = getMemorySize(access);
 503         if (size.isXmmType()) {
 504             return null;
 505         } else {
 506             return binaryRead(AND.getRMOpcode(size), size, value, access);
 507         }
 508     }
 509 
 510     @MatchRule("(Or value Read=access)")
 511     @MatchRule("(Or value FloatingRead=access)")
 512     public ComplexMatchResult orMemory(ValueNode value, LIRLowerableAccess access) {
 513         OperandSize size = getMemorySize(access);
 514         if (size.isXmmType()) {
 515             return null;
 516         } else {
 517             return binaryRead(OR.getRMOpcode(size), size, value, access);
 518         }
 519     }
 520 
 521     @MatchRule("(Xor value Read=access)")
 522     @MatchRule("(Xor value FloatingRead=access)")
 523     public ComplexMatchResult xorMemory(ValueNode value, LIRLowerableAccess access) {
 524         OperandSize size = getMemorySize(access);
 525         if (size.isXmmType()) {
 526             return null;
 527         } else {
 528             return binaryRead(XOR.getRMOpcode(size), size, value, access);
 529         }
 530     }
 531 
 532     @MatchRule("(Write object Narrow=narrow)")
 533     public ComplexMatchResult writeNarrow(WriteNode root, NarrowNode narrow) {
 534         return builder -> {
 535             LIRKind writeKind = getLIRGeneratorTool().getLIRKind(root.value().stamp(NodeView.DEFAULT));
 536             getArithmeticLIRGenerator().emitStore(writeKind, operand(root.getAddress()), operand(narrow.getValue()), state(root));
 537             return null;
 538         };
 539     }
 540 
 541     @MatchRule("(SignExtend Read=access)")
 542     @MatchRule("(SignExtend FloatingRead=access)")
 543     public ComplexMatchResult signExtend(SignExtendNode root, LIRLowerableAccess access) {
 544         return emitSignExtendMemory(access, root.getInputBits(), root.getResultBits(), null);
 545     }
 546 
 547     @MatchRule("(ZeroExtend Read=access)")
 548     @MatchRule("(ZeroExtend FloatingRead=access)")
 549     public ComplexMatchResult zeroExtend(ZeroExtendNode root, LIRLowerableAccess access) {
 550         AMD64Kind memoryKind = getMemoryKind(access);
 551         return builder -> getArithmeticLIRGenerator().emitZeroExtendMemory(memoryKind, root.getResultBits(), (AMD64AddressValue) operand(access.getAddress()), getState(access));
 552     }
 553 
 554     @MatchRule("(Narrow Read=access)")
 555     @MatchRule("(Narrow FloatingRead=access)")
 556     public ComplexMatchResult narrowRead(NarrowNode root, LIRLowerableAccess access) {
 557         return new ComplexMatchResult() {
 558             @Override
 559             public Value evaluate(NodeLIRBuilder builder) {
 560                 AMD64AddressValue address = (AMD64AddressValue) operand(access.getAddress());
 561                 LIRKind addressKind = LIRKind.combineDerived(getLIRGeneratorTool().getLIRKind(root.asNode().stamp(NodeView.DEFAULT)),
 562                                 address.getBase(), address.getIndex());
 563                 AMD64AddressValue newAddress = address.withKind(addressKind);
 564                 LIRKind readKind = getLIRGeneratorTool().getLIRKind(root.stamp(NodeView.DEFAULT));
 565                 return getArithmeticLIRGenerator().emitZeroExtendMemory((AMD64Kind) readKind.getPlatformKind(),
 566                                 root.getResultBits(), newAddress, getState(access));
 567             }
 568         };
 569     }
 570 
 571     @MatchRule("(SignExtend (Narrow=narrow Read=access))")
 572     @MatchRule("(SignExtend (Narrow=narrow FloatingRead=access))")
 573     public ComplexMatchResult signExtendNarrowRead(SignExtendNode root, NarrowNode narrow, LIRLowerableAccess access) {
 574         LIRKind kind = getLIRGeneratorTool().getLIRKind(narrow.stamp(NodeView.DEFAULT));
 575         return emitSignExtendMemory(access, narrow.getResultBits(), root.getResultBits(), kind);
 576     }
 577 
 578     @MatchRule("(FloatConvert Read=access)")
 579     @MatchRule("(FloatConvert FloatingRead=access)")
 580     public ComplexMatchResult floatConvert(FloatConvertNode root, LIRLowerableAccess access) {
 581         switch (root.getFloatConvert()) {
 582             case D2F:
 583                 return emitConvertMemoryOp(AMD64Kind.SINGLE, SSEOp.CVTSD2SS, SD, access);
 584             case D2I:
 585                 return emitConvertMemoryOp(AMD64Kind.DWORD, SSEOp.CVTTSD2SI, DWORD, access);
 586             case D2L:
 587                 return emitConvertMemoryOp(AMD64Kind.QWORD, SSEOp.CVTTSD2SI, QWORD, access);
 588             case F2D:
 589                 return emitConvertMemoryOp(AMD64Kind.DOUBLE, SSEOp.CVTSS2SD, SS, access);
 590             case F2I:
 591                 return emitConvertMemoryOp(AMD64Kind.DWORD, SSEOp.CVTTSS2SI, DWORD, access);
 592             case F2L:
 593                 return emitConvertMemoryOp(AMD64Kind.QWORD, SSEOp.CVTTSS2SI, QWORD, access);
 594             case I2D:
 595                 return emitConvertMemoryOp(AMD64Kind.DOUBLE, SSEOp.CVTSI2SD, DWORD, access);
 596             case I2F:
 597                 return emitConvertMemoryOp(AMD64Kind.SINGLE, SSEOp.CVTSI2SS, DWORD, access);
 598             case L2D:
 599                 return emitConvertMemoryOp(AMD64Kind.DOUBLE, SSEOp.CVTSI2SD, QWORD, access);
 600             case L2F:
 601                 return emitConvertMemoryOp(AMD64Kind.SINGLE, SSEOp.CVTSI2SS, QWORD, access);
 602             default:
 603                 throw GraalError.shouldNotReachHere();
 604         }
 605     }
 606 
 607     @MatchRule("(Reinterpret Read=access)")
 608     @MatchRule("(Reinterpret FloatingRead=access)")
 609     public ComplexMatchResult reinterpret(ReinterpretNode root, LIRLowerableAccess access) {
 610         return builder -> {
 611             LIRKind kind = getLIRGeneratorTool().getLIRKind(root.stamp(NodeView.DEFAULT));
 612             return emitReinterpretMemory(kind, access);
 613         };
 614 
 615     }
 616 
 617     @MatchRule("(Write object Reinterpret=reinterpret)")
 618     public ComplexMatchResult writeReinterpret(WriteNode root, ReinterpretNode reinterpret) {
 619         return builder -> {
 620             LIRKind kind = getLIRGeneratorTool().getLIRKind(reinterpret.getValue().stamp(NodeView.DEFAULT));
 621             AllocatableValue value = getLIRGeneratorTool().asAllocatable(operand(reinterpret.getValue()));
 622 
 623             AMD64AddressValue address = (AMD64AddressValue) operand(root.getAddress());
 624             getArithmeticLIRGenerator().emitStore((AMD64Kind) kind.getPlatformKind(), address, value, getState(root));
 625             return null;
 626         };
 627     }
 628 
 629     @Override
 630     public AMD64LIRGenerator getLIRGeneratorTool() {
 631         return (AMD64LIRGenerator) gen;
 632     }
 633 
 634     protected AMD64ArithmeticLIRGenerator getArithmeticLIRGenerator() {
 635         return (AMD64ArithmeticLIRGenerator) getLIRGeneratorTool().getArithmetic();
 636     }
 637 }