1 /*
   2  * Copyright (c) 2011, 2019, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  */
  23 
  24 
  25 package org.graalvm.compiler.replacements;
  26 
  27 import static jdk.vm.ci.code.MemoryBarriers.JMM_POST_VOLATILE_READ;
  28 import static jdk.vm.ci.code.MemoryBarriers.JMM_POST_VOLATILE_WRITE;
  29 import static jdk.vm.ci.code.MemoryBarriers.JMM_PRE_VOLATILE_READ;
  30 import static jdk.vm.ci.code.MemoryBarriers.JMM_PRE_VOLATILE_WRITE;
  31 import static jdk.vm.ci.meta.DeoptimizationAction.InvalidateReprofile;
  32 import static jdk.vm.ci.meta.DeoptimizationReason.BoundsCheckException;
  33 import static jdk.vm.ci.meta.DeoptimizationReason.NullCheckException;
  34 import static org.graalvm.compiler.core.common.SpeculativeExecutionAttacksMitigations.Options.UseIndexMasking;
  35 import static org.graalvm.compiler.nodes.NamedLocationIdentity.ARRAY_LENGTH_LOCATION;
  36 import static org.graalvm.compiler.nodes.calc.BinaryArithmeticNode.branchlessMax;
  37 import static org.graalvm.compiler.nodes.calc.BinaryArithmeticNode.branchlessMin;
  38 import static org.graalvm.compiler.nodes.java.ArrayLengthNode.readArrayLength;
  39 import static org.graalvm.compiler.nodes.util.GraphUtil.skipPiWhileNonNull;
  40 
  41 import java.nio.ByteOrder;
  42 import java.util.ArrayList;
  43 import java.util.BitSet;
  44 import java.util.List;
  45 
  46 import jdk.vm.ci.meta.JavaConstant;
  47 import org.graalvm.compiler.api.directives.GraalDirectives;
  48 import org.graalvm.compiler.api.replacements.Snippet;
  49 import org.graalvm.compiler.api.replacements.SnippetReflectionProvider;
  50 import org.graalvm.compiler.core.common.LIRKind;
  51 import org.graalvm.compiler.core.common.spi.ForeignCallsProvider;
  52 import org.graalvm.compiler.core.common.type.AbstractPointerStamp;
  53 import org.graalvm.compiler.core.common.type.IntegerStamp;
  54 import org.graalvm.compiler.core.common.type.ObjectStamp;
  55 import org.graalvm.compiler.core.common.type.Stamp;
  56 import org.graalvm.compiler.core.common.type.StampFactory;
  57 import org.graalvm.compiler.core.common.type.TypeReference;
  58 import org.graalvm.compiler.debug.DebugCloseable;
  59 import org.graalvm.compiler.debug.DebugHandlersFactory;
  60 import org.graalvm.compiler.debug.GraalError;
  61 import org.graalvm.compiler.graph.Node;
  62 import org.graalvm.compiler.nodeinfo.InputType;
  63 import org.graalvm.compiler.nodes.CompressionNode.CompressionOp;
  64 import org.graalvm.compiler.nodes.ConstantNode;
  65 import org.graalvm.compiler.nodes.EndNode;
  66 import org.graalvm.compiler.nodes.FieldLocationIdentity;
  67 import org.graalvm.compiler.nodes.FixedNode;
  68 import org.graalvm.compiler.nodes.FixedWithNextNode;
  69 import org.graalvm.compiler.nodes.IfNode;
  70 import org.graalvm.compiler.nodes.LogicNode;
  71 import org.graalvm.compiler.nodes.MergeNode;
  72 import org.graalvm.compiler.nodes.NamedLocationIdentity;
  73 import org.graalvm.compiler.nodes.NodeView;
  74 import org.graalvm.compiler.nodes.PhiNode;
  75 import org.graalvm.compiler.nodes.PiNode;
  76 import org.graalvm.compiler.nodes.StructuredGraph;
  77 import org.graalvm.compiler.nodes.ValueNode;
  78 import org.graalvm.compiler.nodes.ValuePhiNode;
  79 import org.graalvm.compiler.nodes.calc.AddNode;
  80 import org.graalvm.compiler.nodes.calc.ConditionalNode;
  81 import org.graalvm.compiler.nodes.calc.IntegerBelowNode;
  82 import org.graalvm.compiler.nodes.calc.IntegerConvertNode;
  83 import org.graalvm.compiler.nodes.calc.IntegerEqualsNode;
  84 import org.graalvm.compiler.nodes.calc.IsNullNode;
  85 import org.graalvm.compiler.nodes.calc.LeftShiftNode;
  86 import org.graalvm.compiler.nodes.calc.NarrowNode;
  87 import org.graalvm.compiler.nodes.calc.RightShiftNode;
  88 import org.graalvm.compiler.nodes.calc.SignExtendNode;
  89 import org.graalvm.compiler.nodes.calc.SubNode;
  90 import org.graalvm.compiler.nodes.calc.UnpackEndianHalfNode;
  91 import org.graalvm.compiler.nodes.calc.ZeroExtendNode;
  92 import org.graalvm.compiler.nodes.debug.VerifyHeapNode;
  93 import org.graalvm.compiler.nodes.extended.BoxNode;
  94 import org.graalvm.compiler.nodes.extended.FixedValueAnchorNode;
  95 import org.graalvm.compiler.nodes.extended.ForeignCallNode;
  96 import org.graalvm.compiler.nodes.extended.GuardedUnsafeLoadNode;
  97 import org.graalvm.compiler.nodes.extended.GuardingNode;
  98 import org.graalvm.compiler.nodes.extended.JavaReadNode;
  99 import org.graalvm.compiler.nodes.extended.JavaWriteNode;
 100 import org.graalvm.compiler.nodes.extended.LoadArrayComponentHubNode;
 101 import org.graalvm.compiler.nodes.extended.LoadHubNode;
 102 import org.graalvm.compiler.nodes.extended.LoadHubOrNullNode;
 103 import org.graalvm.compiler.nodes.extended.MembarNode;
 104 import org.graalvm.compiler.nodes.extended.RawLoadNode;
 105 import org.graalvm.compiler.nodes.extended.RawStoreNode;
 106 import org.graalvm.compiler.nodes.extended.UnboxNode;
 107 import org.graalvm.compiler.nodes.extended.UnsafeMemoryLoadNode;
 108 import org.graalvm.compiler.nodes.extended.UnsafeMemoryStoreNode;
 109 import org.graalvm.compiler.nodes.java.AbstractNewObjectNode;
 110 import org.graalvm.compiler.nodes.java.AccessIndexedNode;
 111 import org.graalvm.compiler.nodes.java.ArrayLengthNode;
 112 import org.graalvm.compiler.nodes.java.AtomicReadAndWriteNode;
 113 import org.graalvm.compiler.nodes.java.FinalFieldBarrierNode;
 114 import org.graalvm.compiler.nodes.java.InstanceOfDynamicNode;
 115 import org.graalvm.compiler.nodes.java.InstanceOfNode;
 116 import org.graalvm.compiler.nodes.java.LoadFieldNode;
 117 import org.graalvm.compiler.nodes.java.LoadIndexedNode;
 118 import org.graalvm.compiler.nodes.java.LogicCompareAndSwapNode;
 119 import org.graalvm.compiler.nodes.java.LoweredAtomicReadAndWriteNode;
 120 import org.graalvm.compiler.nodes.java.MonitorEnterNode;
 121 import org.graalvm.compiler.nodes.java.MonitorIdNode;
 122 import org.graalvm.compiler.nodes.java.NewArrayNode;
 123 import org.graalvm.compiler.nodes.java.NewInstanceNode;
 124 import org.graalvm.compiler.nodes.java.RawMonitorEnterNode;
 125 import org.graalvm.compiler.nodes.java.StoreFieldNode;
 126 import org.graalvm.compiler.nodes.java.StoreIndexedNode;
 127 import org.graalvm.compiler.nodes.java.UnsafeCompareAndExchangeNode;
 128 import org.graalvm.compiler.nodes.java.UnsafeCompareAndSwapNode;
 129 import org.graalvm.compiler.nodes.java.ValueCompareAndSwapNode;
 130 import org.graalvm.compiler.nodes.memory.HeapAccess.BarrierType;
 131 import org.graalvm.compiler.nodes.memory.ReadNode;
 132 import org.graalvm.compiler.nodes.memory.WriteNode;
 133 import org.graalvm.compiler.nodes.memory.address.AddressNode;
 134 import org.graalvm.compiler.nodes.memory.address.IndexAddressNode;
 135 import org.graalvm.compiler.nodes.memory.address.OffsetAddressNode;
 136 import org.graalvm.compiler.nodes.spi.Lowerable;
 137 import org.graalvm.compiler.nodes.spi.LoweringProvider;
 138 import org.graalvm.compiler.nodes.spi.LoweringTool;
 139 import org.graalvm.compiler.nodes.type.StampTool;
 140 import org.graalvm.compiler.nodes.util.GraphUtil;
 141 import org.graalvm.compiler.nodes.virtual.AllocatedObjectNode;
 142 import org.graalvm.compiler.nodes.virtual.CommitAllocationNode;
 143 import org.graalvm.compiler.nodes.virtual.VirtualArrayNode;
 144 import org.graalvm.compiler.nodes.virtual.VirtualInstanceNode;
 145 import org.graalvm.compiler.nodes.virtual.VirtualObjectNode;
 146 import org.graalvm.compiler.options.OptionValues;
 147 import org.graalvm.compiler.phases.util.Providers;
 148 import org.graalvm.compiler.replacements.SnippetLowerableMemoryNode.SnippetLowering;
 149 import org.graalvm.compiler.replacements.nodes.BinaryMathIntrinsicNode;
 150 import org.graalvm.compiler.replacements.nodes.UnaryMathIntrinsicNode;
 151 import jdk.internal.vm.compiler.word.LocationIdentity;
 152 
 153 import jdk.vm.ci.code.CodeUtil;
 154 import jdk.vm.ci.code.MemoryBarriers;
 155 import jdk.vm.ci.code.TargetDescription;
 156 import jdk.vm.ci.meta.DeoptimizationAction;
 157 import jdk.vm.ci.meta.DeoptimizationReason;
 158 import jdk.vm.ci.meta.JavaKind;
 159 import jdk.vm.ci.meta.MetaAccessProvider;
 160 import jdk.vm.ci.meta.ResolvedJavaField;
 161 import jdk.vm.ci.meta.ResolvedJavaMethod;
 162 import jdk.vm.ci.meta.ResolvedJavaType;
 163 import jdk.vm.ci.meta.SpeculationLog;
 164 
 165 /**
 166  * VM-independent lowerings for standard Java nodes. VM-specific methods are abstract and must be
 167  * implemented by VM-specific subclasses.
 168  */
 169 public abstract class DefaultJavaLoweringProvider implements LoweringProvider {
 170 
 171     protected final MetaAccessProvider metaAccess;
 172     protected final ForeignCallsProvider foreignCalls;
 173     protected final TargetDescription target;
 174     private final boolean useCompressedOops;
 175     private final ResolvedJavaType objectArrayType;
 176 
 177     private BoxingSnippets.Templates boxingSnippets;
 178     private ConstantStringIndexOfSnippets.Templates indexOfSnippets;
 179 
 180     public DefaultJavaLoweringProvider(MetaAccessProvider metaAccess, ForeignCallsProvider foreignCalls, TargetDescription target, boolean useCompressedOops) {
 181         this.metaAccess = metaAccess;
 182         this.foreignCalls = foreignCalls;
 183         this.target = target;
 184         this.useCompressedOops = useCompressedOops;
 185         this.objectArrayType = metaAccess.lookupJavaType(Object[].class);
 186     }
 187 
 188     public void initialize(OptionValues options, Iterable<DebugHandlersFactory> factories, SnippetCounter.Group.Factory factory, Providers providers, SnippetReflectionProvider snippetReflection) {
 189         boxingSnippets = new BoxingSnippets.Templates(options, factories, factory, providers, snippetReflection, target);
 190         indexOfSnippets = new ConstantStringIndexOfSnippets.Templates(options, factories, providers, snippetReflection, target);
 191         providers.getReplacements().registerSnippetTemplateCache(new SnippetCounterNode.SnippetCounterSnippets.Templates(options, factories, providers, snippetReflection, target));
 192     }
 193 
 194     public final TargetDescription getTarget() {
 195         return target;
 196     }
 197 
 198     public MetaAccessProvider getMetaAccess() {
 199         return metaAccess;
 200     }
 201 
 202     @Override
 203     @SuppressWarnings("try")
 204     public void lower(Node n, LoweringTool tool) {
 205         assert n instanceof Lowerable;
 206         StructuredGraph graph = (StructuredGraph) n.graph();
 207         try (DebugCloseable context = n.withNodeSourcePosition()) {
 208             if (n instanceof LoadFieldNode) {
 209                 lowerLoadFieldNode((LoadFieldNode) n, tool);
 210             } else if (n instanceof StoreFieldNode) {
 211                 lowerStoreFieldNode((StoreFieldNode) n, tool);
 212             } else if (n instanceof LoadIndexedNode) {
 213                 lowerLoadIndexedNode((LoadIndexedNode) n, tool);
 214             } else if (n instanceof StoreIndexedNode) {
 215                 lowerStoreIndexedNode((StoreIndexedNode) n, tool);
 216             } else if (n instanceof IndexAddressNode) {
 217                 lowerIndexAddressNode((IndexAddressNode) n);
 218             } else if (n instanceof ArrayLengthNode) {
 219                 lowerArrayLengthNode((ArrayLengthNode) n, tool);
 220             } else if (n instanceof LoadHubNode) {
 221                 lowerLoadHubNode((LoadHubNode) n, tool);
 222             } else if (n instanceof LoadHubOrNullNode) {
 223                 lowerLoadHubOrNullNode((LoadHubOrNullNode) n, tool);
 224             } else if (n instanceof LoadArrayComponentHubNode) {
 225                 lowerLoadArrayComponentHubNode((LoadArrayComponentHubNode) n);
 226             } else if (n instanceof MonitorEnterNode) {
 227                 lowerMonitorEnterNode((MonitorEnterNode) n, tool, graph);
 228             } else if (n instanceof UnsafeCompareAndSwapNode) {
 229                 lowerCompareAndSwapNode((UnsafeCompareAndSwapNode) n);
 230             } else if (n instanceof UnsafeCompareAndExchangeNode) {
 231                 lowerCompareAndExchangeNode((UnsafeCompareAndExchangeNode) n);
 232             } else if (n instanceof AtomicReadAndWriteNode) {
 233                 lowerAtomicReadAndWriteNode((AtomicReadAndWriteNode) n);
 234             } else if (n instanceof RawLoadNode) {
 235                 lowerUnsafeLoadNode((RawLoadNode) n, tool);
 236             } else if (n instanceof UnsafeMemoryLoadNode) {
 237                 lowerUnsafeMemoryLoadNode((UnsafeMemoryLoadNode) n);
 238             } else if (n instanceof RawStoreNode) {
 239                 lowerUnsafeStoreNode((RawStoreNode) n);
 240             } else if (n instanceof UnsafeMemoryStoreNode) {
 241                 lowerUnsafeMemoryStoreNode((UnsafeMemoryStoreNode) n);
 242             } else if (n instanceof JavaReadNode) {
 243                 lowerJavaReadNode((JavaReadNode) n);
 244             } else if (n instanceof JavaWriteNode) {
 245                 lowerJavaWriteNode((JavaWriteNode) n);
 246             } else if (n instanceof CommitAllocationNode) {
 247                 lowerCommitAllocationNode((CommitAllocationNode) n, tool);
 248             } else if (n instanceof BoxNode) {
 249                 boxingSnippets.lower((BoxNode) n, tool);
 250             } else if (n instanceof UnboxNode) {
 251                 boxingSnippets.lower((UnboxNode) n, tool);
 252             } else if (n instanceof VerifyHeapNode) {
 253                 lowerVerifyHeap((VerifyHeapNode) n);
 254             } else if (n instanceof UnaryMathIntrinsicNode) {
 255                 lowerUnaryMath((UnaryMathIntrinsicNode) n, tool);
 256             } else if (n instanceof BinaryMathIntrinsicNode) {
 257                 lowerBinaryMath((BinaryMathIntrinsicNode) n, tool);
 258             } else if (n instanceof StringIndexOfNode) {
 259                 lowerIndexOf((StringIndexOfNode) n);
 260             } else if (n instanceof StringLatin1IndexOfNode) {
 261                 lowerLatin1IndexOf((StringLatin1IndexOfNode) n);
 262             } else if (n instanceof StringUTF16IndexOfNode) {
 263                 lowerUTF16IndexOf((StringUTF16IndexOfNode) n);
 264             } else if (n instanceof UnpackEndianHalfNode) {
 265                 lowerSecondHalf((UnpackEndianHalfNode) n);
 266             } else {
 267                 throw GraalError.shouldNotReachHere("Node implementing Lowerable not handled: " + n);
 268             }
 269         }
 270     }
 271 
 272     private void lowerSecondHalf(UnpackEndianHalfNode n) {
 273         ByteOrder byteOrder = target.arch.getByteOrder();
 274         n.lower(byteOrder);
 275     }
 276 
 277     private void lowerIndexOf(StringIndexOfNode n) {
 278         if (n.getArgument(3).isConstant()) {
 279             SnippetLowering lowering = new SnippetLowering() {
 280                 @Override
 281                 public void lower(SnippetLowerableMemoryNode node, LoweringTool tool) {
 282                     if (tool.getLoweringStage() != LoweringTool.StandardLoweringStage.LOW_TIER) {
 283                         return;
 284                     }
 285                     indexOfSnippets.lower(node, tool);
 286                 }
 287             };
 288             SnippetLowerableMemoryNode snippetLower = new SnippetLowerableMemoryNode(lowering, NamedLocationIdentity.getArrayLocation(JavaKind.Char), n.stamp(NodeView.DEFAULT), n.toArgumentArray());
 289             n.graph().add(snippetLower);
 290             n.graph().replaceFixedWithFixed(n, snippetLower);
 291         }
 292     }
 293 
 294     private void lowerLatin1IndexOf(StringLatin1IndexOfNode n) {
 295         if (n.getArgument(2).isConstant()) {
 296             SnippetLowering lowering = new SnippetLowering() {
 297                 @Override
 298                 public void lower(SnippetLowerableMemoryNode node, LoweringTool tool) {
 299                     if (tool.getLoweringStage() != LoweringTool.StandardLoweringStage.LOW_TIER) {
 300                         return;
 301                     }
 302                     indexOfSnippets.lowerLatin1(node, tool);
 303                 }
 304             };
 305             SnippetLowerableMemoryNode snippetLower = new SnippetLowerableMemoryNode(lowering, NamedLocationIdentity.getArrayLocation(JavaKind.Byte), n.stamp(NodeView.DEFAULT), n.toArgumentArray());
 306             n.graph().add(snippetLower);
 307             n.graph().replaceFixedWithFixed(n, snippetLower);
 308         }
 309     }
 310 
 311     private void lowerUTF16IndexOf(StringUTF16IndexOfNode n) {
 312         if (n.getArgument(2).isConstant()) {
 313             SnippetLowering lowering = new SnippetLowering() {
 314                 @Override
 315                 public void lower(SnippetLowerableMemoryNode node, LoweringTool tool) {
 316                     if (tool.getLoweringStage() != LoweringTool.StandardLoweringStage.LOW_TIER) {
 317                         return;
 318                     }
 319                     indexOfSnippets.lowerUTF16(node, tool);
 320                 }
 321             };
 322             SnippetLowerableMemoryNode snippetLower = new SnippetLowerableMemoryNode(lowering, NamedLocationIdentity.getArrayLocation(JavaKind.Byte), n.stamp(NodeView.DEFAULT), n.toArgumentArray());
 323             n.graph().add(snippetLower);
 324             n.graph().replaceFixedWithFixed(n, snippetLower);
 325         }
 326     }
 327 
 328     private void lowerBinaryMath(BinaryMathIntrinsicNode math, LoweringTool tool) {
 329         if (tool.getLoweringStage() == LoweringTool.StandardLoweringStage.HIGH_TIER) {
 330             return;
 331         }
 332         ResolvedJavaMethod method = math.graph().method();
 333         if (method != null) {
 334             if (method.getAnnotation(Snippet.class) != null) {
 335                 // In the context of SnippetStub, i.e., Graal-generated stubs, use the LIR
 336                 // lowering to emit the stub assembly code instead of the Node lowering.
 337                 return;
 338             }
 339             if (method.getName().equalsIgnoreCase(math.getOperation().name()) && tool.getMetaAccess().lookupJavaType(Math.class).equals(method.getDeclaringClass())) {
 340                 // A root compilation of the intrinsic method should emit the full assembly
 341                 // implementation.
 342                 return;
 343             }
 344         }
 345         StructuredGraph graph = math.graph();
 346         ForeignCallNode call = graph.add(new ForeignCallNode(foreignCalls, math.getOperation().foreignCallDescriptor, math.getX(), math.getY()));
 347         graph.addAfterFixed(tool.lastFixedNode(), call);
 348         math.replaceAtUsages(call);
 349     }
 350 
 351     private void lowerUnaryMath(UnaryMathIntrinsicNode math, LoweringTool tool) {
 352         if (tool.getLoweringStage() == LoweringTool.StandardLoweringStage.HIGH_TIER) {
 353             return;
 354         }
 355         ResolvedJavaMethod method = math.graph().method();
 356         if (method != null) {
 357             if (method.getName().equalsIgnoreCase(math.getOperation().name()) && tool.getMetaAccess().lookupJavaType(Math.class).equals(method.getDeclaringClass())) {
 358                 // A root compilation of the intrinsic method should emit the full assembly
 359                 // implementation.
 360                 return;
 361             }
 362         }
 363         StructuredGraph graph = math.graph();
 364         ForeignCallNode call = math.graph().add(new ForeignCallNode(foreignCalls, math.getOperation().foreignCallDescriptor, math.getValue()));
 365         graph.addAfterFixed(tool.lastFixedNode(), call);
 366         math.replaceAtUsages(call);
 367     }
 368 
 369     protected void lowerVerifyHeap(VerifyHeapNode n) {
 370         GraphUtil.removeFixedWithUnusedInputs(n);
 371     }
 372 
 373     protected AddressNode createOffsetAddress(StructuredGraph graph, ValueNode object, long offset) {
 374         ValueNode o = ConstantNode.forIntegerKind(target.wordJavaKind, offset, graph);
 375         return graph.unique(new OffsetAddressNode(object, o));
 376     }
 377 
 378     protected AddressNode createFieldAddress(StructuredGraph graph, ValueNode object, ResolvedJavaField field) {
 379         int offset = fieldOffset(field);
 380         if (offset >= 0) {
 381             return createOffsetAddress(graph, object, offset);
 382         } else {
 383             return null;
 384         }
 385     }
 386 
 387     protected abstract JavaKind getStorageKind(ResolvedJavaField field);
 388 
 389     protected void lowerLoadFieldNode(LoadFieldNode loadField, LoweringTool tool) {
 390         assert loadField.getStackKind() != JavaKind.Illegal;
 391         StructuredGraph graph = loadField.graph();
 392         ResolvedJavaField field = loadField.field();
 393         ValueNode object = loadField.isStatic() ? staticFieldBase(graph, field) : loadField.object();
 394         object = createNullCheckedValue(object, loadField, tool);
 395         Stamp loadStamp = loadStamp(loadField.stamp(NodeView.DEFAULT), getStorageKind(field));
 396 
 397         AddressNode address = createFieldAddress(graph, object, field);
 398         assert address != null : "Field that is loaded must not be eliminated: " + field.getDeclaringClass().toJavaName(true) + "." + field.getName();
 399 
 400         ReadNode memoryRead = graph.add(new ReadNode(address, fieldLocationIdentity(field), loadStamp, fieldLoadBarrierType(field)));
 401         ValueNode readValue = implicitLoadConvert(graph, getStorageKind(field), memoryRead);
 402         loadField.replaceAtUsages(readValue);
 403         graph.replaceFixed(loadField, memoryRead);
 404 
 405         if (loadField.isVolatile()) {
 406             MembarNode preMembar = graph.add(new MembarNode(JMM_PRE_VOLATILE_READ));
 407             graph.addBeforeFixed(memoryRead, preMembar);
 408             MembarNode postMembar = graph.add(new MembarNode(JMM_POST_VOLATILE_READ));
 409             graph.addAfterFixed(memoryRead, postMembar);
 410         }
 411     }
 412 
 413     protected void lowerStoreFieldNode(StoreFieldNode storeField, LoweringTool tool) {
 414         StructuredGraph graph = storeField.graph();
 415         ResolvedJavaField field = storeField.field();
 416         ValueNode object = storeField.isStatic() ? staticFieldBase(graph, field) : storeField.object();
 417         object = createNullCheckedValue(object, storeField, tool);
 418         ValueNode value = implicitStoreConvert(graph, getStorageKind(storeField.field()), storeField.value());
 419         AddressNode address = createFieldAddress(graph, object, field);
 420         assert address != null;
 421 
 422         WriteNode memoryWrite = graph.add(new WriteNode(address, fieldLocationIdentity(field), value, fieldStoreBarrierType(storeField.field())));
 423         memoryWrite.setStateAfter(storeField.stateAfter());
 424         graph.replaceFixedWithFixed(storeField, memoryWrite);
 425 
 426         if (storeField.isVolatile()) {
 427             MembarNode preMembar = graph.add(new MembarNode(JMM_PRE_VOLATILE_WRITE));
 428             graph.addBeforeFixed(memoryWrite, preMembar);
 429             MembarNode postMembar = graph.add(new MembarNode(JMM_POST_VOLATILE_WRITE));
 430             graph.addAfterFixed(memoryWrite, postMembar);
 431         }
 432     }
 433 
 434     public static final IntegerStamp POSITIVE_ARRAY_INDEX_STAMP = StampFactory.forInteger(32, 0, Integer.MAX_VALUE - 1);
 435 
 436     /**
 437      * Create a PiNode on the index proving that the index is positive. On some platforms this is
 438      * important to allow the index to be used as an int in the address mode.
 439      */
 440     public AddressNode createArrayIndexAddress(StructuredGraph graph, ValueNode array, JavaKind elementKind, ValueNode index, GuardingNode boundsCheck) {
 441         ValueNode positiveIndex = graph.maybeAddOrUnique(PiNode.create(index, POSITIVE_ARRAY_INDEX_STAMP, boundsCheck != null ? boundsCheck.asNode() : null));
 442         return createArrayAddress(graph, array, elementKind, positiveIndex);
 443     }
 444 
 445     public AddressNode createArrayAddress(StructuredGraph graph, ValueNode array, JavaKind elementKind, ValueNode index) {
 446         return createArrayAddress(graph, array, elementKind, elementKind, index);
 447     }
 448 
 449     public AddressNode createArrayAddress(StructuredGraph graph, ValueNode array, JavaKind arrayKind, JavaKind elementKind, ValueNode index) {
 450         ValueNode wordIndex;
 451         if (target.wordSize > 4) {
 452             wordIndex = graph.unique(new SignExtendNode(index, target.wordSize * 8));
 453         } else {
 454             assert target.wordSize == 4 : "unsupported word size";
 455             wordIndex = index;
 456         }
 457 
 458         int shift = CodeUtil.log2(metaAccess.getArrayIndexScale(elementKind));
 459         ValueNode scaledIndex = graph.unique(new LeftShiftNode(wordIndex, ConstantNode.forInt(shift, graph)));
 460 
 461         int base = metaAccess.getArrayBaseOffset(arrayKind);
 462         ValueNode offset = graph.unique(new AddNode(scaledIndex, ConstantNode.forIntegerKind(target.wordJavaKind, base, graph)));
 463 
 464         return graph.unique(new OffsetAddressNode(array, offset));
 465     }
 466 
 467     protected void lowerIndexAddressNode(IndexAddressNode indexAddress) {
 468         AddressNode lowered = createArrayAddress(indexAddress.graph(), indexAddress.getArray(), indexAddress.getArrayKind(), indexAddress.getElementKind(), indexAddress.getIndex());
 469         indexAddress.replaceAndDelete(lowered);
 470     }
 471 
 472     protected void lowerLoadIndexedNode(LoadIndexedNode loadIndexed, LoweringTool tool) {
 473         StructuredGraph graph = loadIndexed.graph();
 474         ValueNode array = loadIndexed.array();
 475         array = createNullCheckedValue(array, loadIndexed, tool);
 476         JavaKind elementKind = loadIndexed.elementKind();
 477         Stamp loadStamp = loadStamp(loadIndexed.stamp(NodeView.DEFAULT), elementKind);
 478 
 479         GuardingNode boundsCheck = getBoundsCheck(loadIndexed, array, tool);
 480         ValueNode index = loadIndexed.index();
 481         if (UseIndexMasking.getValue(graph.getOptions())) {
 482             index = proxyIndex(loadIndexed, index, array, tool);
 483         }
 484         AddressNode address = createArrayIndexAddress(graph, array, elementKind, index, boundsCheck);
 485 
 486         ReadNode memoryRead = graph.add(new ReadNode(address, NamedLocationIdentity.getArrayLocation(elementKind), loadStamp, BarrierType.NONE));
 487         memoryRead.setGuard(boundsCheck);
 488         ValueNode readValue = implicitLoadConvert(graph, elementKind, memoryRead);
 489 
 490         loadIndexed.replaceAtUsages(readValue);
 491         graph.replaceFixed(loadIndexed, memoryRead);
 492     }
 493 
 494     protected void lowerStoreIndexedNode(StoreIndexedNode storeIndexed, LoweringTool tool) {
 495         StructuredGraph graph = storeIndexed.graph();
 496 
 497         ValueNode value = storeIndexed.value();
 498         ValueNode array = storeIndexed.array();
 499 
 500         array = this.createNullCheckedValue(array, storeIndexed, tool);
 501 
 502         GuardingNode boundsCheck = getBoundsCheck(storeIndexed, array, tool);
 503 
 504         JavaKind elementKind = storeIndexed.elementKind();
 505 
 506         LogicNode condition = null;
 507         if (storeIndexed.getStoreCheck() == null && elementKind == JavaKind.Object && !StampTool.isPointerAlwaysNull(value)) {
 508             /* Array store check. */
 509             TypeReference arrayType = StampTool.typeReferenceOrNull(array);
 510             if (arrayType != null && arrayType.isExact()) {
 511                 ResolvedJavaType elementType = arrayType.getType().getComponentType();
 512                 if (!elementType.isJavaLangObject()) {
 513                     TypeReference typeReference = TypeReference.createTrusted(storeIndexed.graph().getAssumptions(), elementType);
 514                     LogicNode typeTest = graph.addOrUniqueWithInputs(InstanceOfNode.create(typeReference, value));
 515                     condition = LogicNode.or(graph.unique(IsNullNode.create(value)), typeTest, GraalDirectives.UNLIKELY_PROBABILITY);
 516                 }
 517             } else {
 518                 /*
 519                  * The guard on the read hub should be the null check of the array that was
 520                  * introduced earlier.
 521                  */
 522                 ValueNode arrayClass = createReadHub(graph, array, tool);
 523                 ValueNode componentHub = createReadArrayComponentHub(graph, arrayClass, storeIndexed);
 524                 LogicNode typeTest = graph.unique(InstanceOfDynamicNode.create(graph.getAssumptions(), tool.getConstantReflection(), componentHub, value, false));
 525                 condition = LogicNode.or(graph.unique(IsNullNode.create(value)), typeTest, GraalDirectives.UNLIKELY_PROBABILITY);
 526             }
 527         }
 528 
 529         AddressNode address = createArrayIndexAddress(graph, array, elementKind, storeIndexed.index(), boundsCheck);
 530         WriteNode memoryWrite = graph.add(new WriteNode(address, NamedLocationIdentity.getArrayLocation(elementKind), implicitStoreConvert(graph, elementKind, value),
 531                         arrayStoreBarrierType(storeIndexed.elementKind())));
 532         memoryWrite.setGuard(boundsCheck);
 533         if (condition != null) {
 534             tool.createGuard(storeIndexed, condition, DeoptimizationReason.ArrayStoreException, DeoptimizationAction.InvalidateReprofile);
 535         }
 536         memoryWrite.setStateAfter(storeIndexed.stateAfter());
 537         graph.replaceFixedWithFixed(storeIndexed, memoryWrite);
 538     }
 539 
 540     protected void lowerArrayLengthNode(ArrayLengthNode arrayLengthNode, LoweringTool tool) {
 541         arrayLengthNode.replaceAtUsages(createReadArrayLength(arrayLengthNode.array(), arrayLengthNode, tool));
 542         StructuredGraph graph = arrayLengthNode.graph();
 543         graph.removeFixed(arrayLengthNode);
 544     }
 545 
 546     /**
 547      * Creates a read node that read the array length and is guarded by a null-check.
 548      *
 549      * The created node is placed before {@code before} in the CFG.
 550      */
 551     protected ReadNode createReadArrayLength(ValueNode array, FixedNode before, LoweringTool tool) {
 552         StructuredGraph graph = array.graph();
 553         ValueNode canonicalArray = this.createNullCheckedValue(skipPiWhileNonNull(array), before, tool);
 554         AddressNode address = createOffsetAddress(graph, canonicalArray, arrayLengthOffset());
 555         ReadNode readArrayLength = graph.add(new ReadNode(address, ARRAY_LENGTH_LOCATION, StampFactory.positiveInt(), BarrierType.NONE));
 556         graph.addBeforeFixed(before, readArrayLength);
 557         return readArrayLength;
 558     }
 559 
 560     protected void lowerLoadHubNode(LoadHubNode loadHub, LoweringTool tool) {
 561         StructuredGraph graph = loadHub.graph();
 562         if (tool.getLoweringStage() != LoweringTool.StandardLoweringStage.LOW_TIER) {
 563             return;
 564         }
 565         if (graph.getGuardsStage().allowsFloatingGuards()) {
 566             return;
 567         }
 568         ValueNode hub = createReadHub(graph, loadHub.getValue(), tool);
 569         loadHub.replaceAtUsagesAndDelete(hub);
 570     }
 571 
 572     protected void lowerLoadHubOrNullNode(LoadHubOrNullNode loadHubOrNullNode, LoweringTool tool) {
 573         StructuredGraph graph = loadHubOrNullNode.graph();
 574         if (tool.getLoweringStage() != LoweringTool.StandardLoweringStage.LOW_TIER) {
 575             return;
 576         }
 577         if (graph.getGuardsStage().allowsFloatingGuards()) {
 578             return;
 579         }
 580         final FixedWithNextNode predecessor = tool.lastFixedNode();
 581         final ValueNode value = loadHubOrNullNode.getValue();
 582         AbstractPointerStamp stamp = (AbstractPointerStamp) value.stamp(NodeView.DEFAULT);
 583         final LogicNode isNull = graph.addOrUniqueWithInputs(IsNullNode.create(value));
 584         final EndNode trueEnd = graph.add(new EndNode());
 585         final EndNode falseEnd = graph.add(new EndNode());
 586         final IfNode ifNode = graph.add(new IfNode(isNull, trueEnd, falseEnd, 0.5));
 587         final MergeNode merge = graph.add(new MergeNode());
 588         merge.addForwardEnd(trueEnd);
 589         merge.addForwardEnd(falseEnd);
 590         final AbstractPointerStamp hubStamp = (AbstractPointerStamp) loadHubOrNullNode.stamp(NodeView.DEFAULT);
 591         ValueNode nullHub = ConstantNode.forConstant(hubStamp.asAlwaysNull(), JavaConstant.NULL_POINTER, tool.getMetaAccess(), graph);
 592         final ValueNode nonNullValue = graph.addOrUniqueWithInputs(PiNode.create(value, stamp.asNonNull(), ifNode.falseSuccessor()));
 593         ValueNode hub = createReadHub(graph, nonNullValue, tool);
 594         ValueNode[] values = new ValueNode[]{nullHub, hub};
 595         final PhiNode hubPhi = graph.unique(new ValuePhiNode(hubStamp, merge, values));
 596         final FixedNode oldNext = predecessor.next();
 597         predecessor.setNext(ifNode);
 598         merge.setNext(oldNext);
 599         loadHubOrNullNode.replaceAtUsagesAndDelete(hubPhi);
 600     }
 601 
 602     protected void lowerLoadArrayComponentHubNode(LoadArrayComponentHubNode loadHub) {
 603         StructuredGraph graph = loadHub.graph();
 604         ValueNode hub = createReadArrayComponentHub(graph, loadHub.getValue(), loadHub);
 605         graph.replaceFixed(loadHub, hub);
 606     }
 607 
 608     protected void lowerMonitorEnterNode(MonitorEnterNode monitorEnter, LoweringTool tool, StructuredGraph graph) {
 609         ValueNode object = createNullCheckedValue(monitorEnter.object(), monitorEnter, tool);
 610         ValueNode hub = graph.addOrUnique(LoadHubNode.create(object, tool.getStampProvider(), tool.getMetaAccess(), tool.getConstantReflection()));
 611         RawMonitorEnterNode rawMonitorEnter = graph.add(new RawMonitorEnterNode(object, hub, monitorEnter.getMonitorId()));
 612         rawMonitorEnter.setStateBefore(monitorEnter.stateBefore());
 613         rawMonitorEnter.setStateAfter(monitorEnter.stateAfter());
 614         graph.replaceFixedWithFixed(monitorEnter, rawMonitorEnter);
 615     }
 616 
 617     protected void lowerCompareAndSwapNode(UnsafeCompareAndSwapNode cas) {
 618         StructuredGraph graph = cas.graph();
 619         JavaKind valueKind = cas.getValueKind();
 620 
 621         ValueNode expectedValue = implicitStoreConvert(graph, valueKind, cas.expected());
 622         ValueNode newValue = implicitStoreConvert(graph, valueKind, cas.newValue());
 623 
 624         AddressNode address = graph.unique(new OffsetAddressNode(cas.object(), cas.offset()));
 625         BarrierType barrierType = guessStoreBarrierType(cas.object(), expectedValue);
 626         LogicCompareAndSwapNode atomicNode = graph.add(new LogicCompareAndSwapNode(address, cas.getLocationIdentity(), expectedValue, newValue, barrierType));
 627         atomicNode.setStateAfter(cas.stateAfter());
 628         graph.replaceFixedWithFixed(cas, atomicNode);
 629     }
 630 
 631     protected void lowerCompareAndExchangeNode(UnsafeCompareAndExchangeNode cas) {
 632         StructuredGraph graph = cas.graph();
 633         JavaKind valueKind = cas.getValueKind();
 634 
 635         ValueNode expectedValue = implicitStoreConvert(graph, valueKind, cas.expected());
 636         ValueNode newValue = implicitStoreConvert(graph, valueKind, cas.newValue());
 637 
 638         AddressNode address = graph.unique(new OffsetAddressNode(cas.object(), cas.offset()));
 639         BarrierType barrierType = guessStoreBarrierType(cas.object(), expectedValue);
 640         ValueCompareAndSwapNode atomicNode = graph.add(new ValueCompareAndSwapNode(address, expectedValue, newValue, cas.getLocationIdentity(), barrierType));
 641         ValueNode coercedNode = implicitLoadConvert(graph, valueKind, atomicNode, true);
 642         atomicNode.setStateAfter(cas.stateAfter());
 643         cas.replaceAtUsages(coercedNode);
 644         graph.replaceFixedWithFixed(cas, atomicNode);
 645     }
 646 
 647     protected void lowerAtomicReadAndWriteNode(AtomicReadAndWriteNode n) {
 648         StructuredGraph graph = n.graph();
 649         JavaKind valueKind = n.getValueKind();
 650 
 651         ValueNode newValue = implicitStoreConvert(graph, valueKind, n.newValue());
 652 
 653         AddressNode address = graph.unique(new OffsetAddressNode(n.object(), n.offset()));
 654         BarrierType barrierType = guessStoreBarrierType(n.object(), n.newValue());
 655         LIRKind lirAccessKind = LIRKind.fromJavaKind(target.arch, valueKind);
 656         LoweredAtomicReadAndWriteNode memoryRead = graph.add(new LoweredAtomicReadAndWriteNode(address, n.getLocationIdentity(), newValue, lirAccessKind, barrierType));
 657         memoryRead.setStateAfter(n.stateAfter());
 658 
 659         ValueNode readValue = implicitLoadConvert(graph, valueKind, memoryRead);
 660         n.stateAfter().replaceFirstInput(n, memoryRead);
 661         n.replaceAtUsages(readValue);
 662         graph.replaceFixedWithFixed(n, memoryRead);
 663     }
 664 
 665     /**
 666      * @param tool utility for performing the lowering
 667      */
 668     protected void lowerUnsafeLoadNode(RawLoadNode load, LoweringTool tool) {
 669         StructuredGraph graph = load.graph();
 670         if (load instanceof GuardedUnsafeLoadNode) {
 671             GuardedUnsafeLoadNode guardedLoad = (GuardedUnsafeLoadNode) load;
 672             GuardingNode guard = guardedLoad.getGuard();
 673             if (guard == null) {
 674                 // can float freely if the guard folded away
 675                 ReadNode memoryRead = createUnsafeRead(graph, load, null);
 676                 memoryRead.setForceFixed(false);
 677                 graph.replaceFixedWithFixed(load, memoryRead);
 678             } else {
 679                 // must be guarded, but flows below the guard
 680                 ReadNode memoryRead = createUnsafeRead(graph, load, guard);
 681                 graph.replaceFixedWithFixed(load, memoryRead);
 682             }
 683         } else {
 684             // never had a guarding condition so it must be fixed, creation of the read will force
 685             // it to be fixed
 686             ReadNode memoryRead = createUnsafeRead(graph, load, null);
 687             graph.replaceFixedWithFixed(load, memoryRead);
 688         }
 689     }
 690 
 691     protected AddressNode createUnsafeAddress(StructuredGraph graph, ValueNode object, ValueNode offset) {
 692         if (object.isConstant() && object.asConstant().isDefaultForKind()) {
 693             return graph.addOrUniqueWithInputs(OffsetAddressNode.create(offset));
 694         } else {
 695             return graph.unique(new OffsetAddressNode(object, offset));
 696         }
 697     }
 698 
 699     protected ReadNode createUnsafeRead(StructuredGraph graph, RawLoadNode load, GuardingNode guard) {
 700         boolean compressible = load.accessKind() == JavaKind.Object;
 701         JavaKind readKind = load.accessKind();
 702         Stamp loadStamp = loadStamp(load.stamp(NodeView.DEFAULT), readKind, compressible);
 703         AddressNode address = createUnsafeAddress(graph, load.object(), load.offset());
 704         ReadNode memoryRead = graph.add(new ReadNode(address, load.getLocationIdentity(), loadStamp, BarrierType.NONE));
 705         if (guard == null) {
 706             // An unsafe read must not float otherwise it may float above
 707             // a test guaranteeing the read is safe.
 708             memoryRead.setForceFixed(true);
 709         } else {
 710             memoryRead.setGuard(guard);
 711         }
 712         ValueNode readValue = performBooleanCoercionIfNecessary(implicitLoadConvert(graph, readKind, memoryRead, compressible), readKind);
 713         load.replaceAtUsages(readValue);
 714         return memoryRead;
 715     }
 716 
 717     protected void lowerUnsafeMemoryLoadNode(UnsafeMemoryLoadNode load) {
 718         StructuredGraph graph = load.graph();
 719         JavaKind readKind = load.getKind();
 720         assert readKind != JavaKind.Object;
 721         Stamp loadStamp = loadStamp(load.stamp(NodeView.DEFAULT), readKind, false);
 722         AddressNode address = graph.addOrUniqueWithInputs(OffsetAddressNode.create(load.getAddress()));
 723         ReadNode memoryRead = graph.add(new ReadNode(address, load.getLocationIdentity(), loadStamp, BarrierType.NONE));
 724         // An unsafe read must not float otherwise it may float above
 725         // a test guaranteeing the read is safe.
 726         memoryRead.setForceFixed(true);
 727         ValueNode readValue = performBooleanCoercionIfNecessary(implicitLoadConvert(graph, readKind, memoryRead, false), readKind);
 728         load.replaceAtUsages(readValue);
 729         graph.replaceFixedWithFixed(load, memoryRead);
 730     }
 731 
 732     private static ValueNode performBooleanCoercionIfNecessary(ValueNode readValue, JavaKind readKind) {
 733         if (readKind == JavaKind.Boolean) {
 734             StructuredGraph graph = readValue.graph();
 735             IntegerEqualsNode eq = graph.addOrUnique(new IntegerEqualsNode(readValue, ConstantNode.forInt(0, graph)));
 736             return graph.addOrUnique(new ConditionalNode(eq, ConstantNode.forBoolean(false, graph), ConstantNode.forBoolean(true, graph)));
 737         }
 738         return readValue;
 739     }
 740 
 741     protected void lowerUnsafeStoreNode(RawStoreNode store) {
 742         StructuredGraph graph = store.graph();
 743         boolean compressible = store.value().getStackKind() == JavaKind.Object;
 744         JavaKind valueKind = store.accessKind();
 745         ValueNode value = implicitStoreConvert(graph, valueKind, store.value(), compressible);
 746         AddressNode address = createUnsafeAddress(graph, store.object(), store.offset());
 747         WriteNode write = graph.add(new WriteNode(address, store.getLocationIdentity(), value, unsafeStoreBarrierType(store)));
 748         write.setStateAfter(store.stateAfter());
 749         graph.replaceFixedWithFixed(store, write);
 750     }
 751 
 752     protected void lowerUnsafeMemoryStoreNode(UnsafeMemoryStoreNode store) {
 753         StructuredGraph graph = store.graph();
 754         assert store.getValue().getStackKind() != JavaKind.Object;
 755         JavaKind valueKind = store.getKind();
 756         ValueNode value = implicitStoreConvert(graph, valueKind, store.getValue(), false);
 757         AddressNode address = graph.addOrUniqueWithInputs(OffsetAddressNode.create(store.getAddress()));
 758         WriteNode write = graph.add(new WriteNode(address, store.getLocationIdentity(), value, BarrierType.NONE));
 759         write.setStateAfter(store.stateAfter());
 760         graph.replaceFixedWithFixed(store, write);
 761     }
 762 
 763     protected void lowerJavaReadNode(JavaReadNode read) {
 764         StructuredGraph graph = read.graph();
 765         JavaKind valueKind = read.getReadKind();
 766         Stamp loadStamp = loadStamp(read.stamp(NodeView.DEFAULT), valueKind, read.isCompressible());
 767 
 768         ReadNode memoryRead = graph.add(new ReadNode(read.getAddress(), read.getLocationIdentity(), loadStamp, read.getBarrierType()));
 769         GuardingNode guard = read.getGuard();
 770         ValueNode readValue = implicitLoadConvert(graph, valueKind, memoryRead, read.isCompressible());
 771         if (guard == null) {
 772             // An unsafe read must not float otherwise it may float above
 773             // a test guaranteeing the read is safe.
 774             memoryRead.setForceFixed(true);
 775         } else {
 776             memoryRead.setGuard(guard);
 777         }
 778         read.replaceAtUsages(readValue);
 779         graph.replaceFixed(read, memoryRead);
 780     }
 781 
 782     protected void lowerJavaWriteNode(JavaWriteNode write) {
 783         StructuredGraph graph = write.graph();
 784         ValueNode value = implicitStoreConvert(graph, write.getWriteKind(), write.value(), write.isCompressible());
 785         WriteNode memoryWrite = graph.add(new WriteNode(write.getAddress(), write.getLocationIdentity(), value, write.getBarrierType()));
 786         memoryWrite.setStateAfter(write.stateAfter());
 787         graph.replaceFixedWithFixed(write, memoryWrite);
 788         memoryWrite.setGuard(write.getGuard());
 789     }
 790 
 791     @SuppressWarnings("try")
 792     protected void lowerCommitAllocationNode(CommitAllocationNode commit, LoweringTool tool) {
 793         StructuredGraph graph = commit.graph();
 794         if (graph.getGuardsStage() == StructuredGraph.GuardsStage.FIXED_DEOPTS) {
 795             List<AbstractNewObjectNode> recursiveLowerings = new ArrayList<>();
 796 
 797             ValueNode[] allocations = new ValueNode[commit.getVirtualObjects().size()];
 798             BitSet omittedValues = new BitSet();
 799             int valuePos = 0;
 800             for (int objIndex = 0; objIndex < commit.getVirtualObjects().size(); objIndex++) {
 801                 VirtualObjectNode virtual = commit.getVirtualObjects().get(objIndex);
 802                 try (DebugCloseable nsp = graph.withNodeSourcePosition(virtual)) {
 803                     int entryCount = virtual.entryCount();
 804                     AbstractNewObjectNode newObject;
 805                     if (virtual instanceof VirtualInstanceNode) {
 806                         newObject = graph.add(createNewInstanceFromVirtual(virtual));
 807                     } else {
 808                         newObject = graph.add(createNewArrayFromVirtual(virtual, ConstantNode.forInt(entryCount, graph)));
 809                     }
 810                     // The final STORE_STORE barrier will be emitted by finishAllocatedObjects
 811                     newObject.clearEmitMemoryBarrier();
 812 
 813                     recursiveLowerings.add(newObject);
 814                     graph.addBeforeFixed(commit, newObject);
 815                     allocations[objIndex] = newObject;
 816                     for (int i = 0; i < entryCount; i++) {
 817                         ValueNode value = commit.getValues().get(valuePos);
 818                         if (value instanceof VirtualObjectNode) {
 819                             value = allocations[commit.getVirtualObjects().indexOf(value)];
 820                         }
 821                         if (value == null) {
 822                             omittedValues.set(valuePos);
 823                         } else if (!(value.isConstant() && value.asConstant().isDefaultForKind())) {
 824                             // Constant.illegal is always the defaultForKind, so it is skipped
 825                             JavaKind valueKind = value.getStackKind();
 826                             JavaKind entryKind = virtual.entryKind(i);
 827 
 828                             // Truffle requires some leniency in terms of what can be put where:
 829                             assert valueKind.getStackKind() == entryKind.getStackKind() ||
 830                                             (valueKind == JavaKind.Long || valueKind == JavaKind.Double || (valueKind == JavaKind.Int && virtual instanceof VirtualArrayNode));
 831                             AddressNode address = null;
 832                             BarrierType barrierType = null;
 833                             if (virtual instanceof VirtualInstanceNode) {
 834                                 ResolvedJavaField field = ((VirtualInstanceNode) virtual).field(i);
 835                                 long offset = fieldOffset(field);
 836                                 if (offset >= 0) {
 837                                     address = createOffsetAddress(graph, newObject, offset);
 838                                     barrierType = fieldInitializationBarrier(entryKind);
 839                                 }
 840                             } else {
 841                                 address = createOffsetAddress(graph, newObject, metaAccess.getArrayBaseOffset(entryKind) + i * metaAccess.getArrayIndexScale(entryKind));
 842                                 barrierType = arrayInitializationBarrier(entryKind);
 843                             }
 844                             if (address != null) {
 845                                 WriteNode write = new WriteNode(address, LocationIdentity.init(), implicitStoreConvert(graph, entryKind, value), barrierType);
 846                                 graph.addAfterFixed(newObject, graph.add(write));
 847                             }
 848                         }
 849                         valuePos++;
 850                     }
 851                 }
 852             }
 853             valuePos = 0;
 854 
 855             for (int objIndex = 0; objIndex < commit.getVirtualObjects().size(); objIndex++) {
 856                 VirtualObjectNode virtual = commit.getVirtualObjects().get(objIndex);
 857                 try (DebugCloseable nsp = graph.withNodeSourcePosition(virtual)) {
 858                     int entryCount = virtual.entryCount();
 859                     ValueNode newObject = allocations[objIndex];
 860                     for (int i = 0; i < entryCount; i++) {
 861                         if (omittedValues.get(valuePos)) {
 862                             ValueNode value = commit.getValues().get(valuePos);
 863                             assert value instanceof VirtualObjectNode;
 864                             ValueNode allocValue = allocations[commit.getVirtualObjects().indexOf(value)];
 865                             if (!(allocValue.isConstant() && allocValue.asConstant().isDefaultForKind())) {
 866                                 assert virtual.entryKind(i) == JavaKind.Object && allocValue.getStackKind() == JavaKind.Object;
 867                                 AddressNode address;
 868                                 BarrierType barrierType;
 869                                 if (virtual instanceof VirtualInstanceNode) {
 870                                     VirtualInstanceNode virtualInstance = (VirtualInstanceNode) virtual;
 871                                     address = createFieldAddress(graph, newObject, virtualInstance.field(i));
 872                                     barrierType = fieldStoreBarrierType(virtualInstance.field(i));
 873                                 } else {
 874                                     address = createArrayAddress(graph, newObject, virtual.entryKind(i), ConstantNode.forInt(i, graph));
 875                                     barrierType = arrayStoreBarrierType(virtual.entryKind(i));
 876                                 }
 877                                 if (address != null) {
 878                                     WriteNode write = new WriteNode(address, LocationIdentity.init(), implicitStoreConvert(graph, JavaKind.Object, allocValue), barrierType);
 879                                     graph.addBeforeFixed(commit, graph.add(write));
 880                                 }
 881                             }
 882                         }
 883                         valuePos++;
 884                     }
 885                 }
 886             }
 887 
 888             finishAllocatedObjects(tool, commit, allocations);
 889             graph.removeFixed(commit);
 890 
 891             for (AbstractNewObjectNode recursiveLowering : recursiveLowerings) {
 892                 recursiveLowering.lower(tool);
 893             }
 894         }
 895 
 896     }
 897 
 898     public NewInstanceNode createNewInstanceFromVirtual(VirtualObjectNode virtual) {
 899         return new NewInstanceNode(virtual.type(), true);
 900     }
 901 
 902     protected NewArrayNode createNewArrayFromVirtual(VirtualObjectNode virtual, ValueNode length) {
 903         return new NewArrayNode(((VirtualArrayNode) virtual).componentType(), length, true);
 904     }
 905 
 906     public void finishAllocatedObjects(LoweringTool tool, CommitAllocationNode commit, ValueNode[] allocations) {
 907         StructuredGraph graph = commit.graph();
 908         for (int objIndex = 0; objIndex < commit.getVirtualObjects().size(); objIndex++) {
 909             FixedValueAnchorNode anchor = graph.add(new FixedValueAnchorNode(allocations[objIndex]));
 910             allocations[objIndex] = anchor;
 911             graph.addBeforeFixed(commit, anchor);
 912         }
 913         /*
 914          * Note that the FrameState that is assigned to these MonitorEnterNodes isn't the correct
 915          * state. It will be the state from before the allocation occurred instead of a valid state
 916          * after the locking is performed. In practice this should be fine since these are newly
 917          * allocated objects. The bytecodes themselves permit allocating an object, doing a
 918          * monitorenter and then dropping all references to the object which would produce the same
 919          * state, though that would normally produce an IllegalMonitorStateException. In HotSpot
 920          * some form of fast path locking should always occur so the FrameState should never
 921          * actually be used.
 922          */
 923         ArrayList<MonitorEnterNode> enters = null;
 924         for (int objIndex = 0; objIndex < commit.getVirtualObjects().size(); objIndex++) {
 925             List<MonitorIdNode> locks = commit.getLocks(objIndex);
 926             if (locks.size() > 1) {
 927                 // Ensure that the lock operations are performed in lock depth order
 928                 ArrayList<MonitorIdNode> newList = new ArrayList<>(locks);
 929                 newList.sort((a, b) -> Integer.compare(a.getLockDepth(), b.getLockDepth()));
 930                 locks = newList;
 931             }
 932             int lastDepth = -1;
 933             for (MonitorIdNode monitorId : locks) {
 934                 assert lastDepth < monitorId.getLockDepth();
 935                 lastDepth = monitorId.getLockDepth();
 936                 MonitorEnterNode enter = graph.add(new MonitorEnterNode(allocations[objIndex], monitorId));
 937                 graph.addBeforeFixed(commit, enter);
 938                 if (enters == null) {
 939                     enters = new ArrayList<>();
 940                 }
 941                 enters.add(enter);
 942             }
 943         }
 944         for (Node usage : commit.usages().snapshot()) {
 945             if (usage instanceof AllocatedObjectNode) {
 946                 AllocatedObjectNode addObject = (AllocatedObjectNode) usage;
 947                 int index = commit.getVirtualObjects().indexOf(addObject.getVirtualObject());
 948                 addObject.replaceAtUsagesAndDelete(allocations[index]);
 949             } else {
 950                 assert enters != null;
 951                 commit.replaceAtUsages(InputType.Memory, enters.get(enters.size() - 1));
 952             }
 953         }
 954         if (enters != null) {
 955             for (MonitorEnterNode enter : enters) {
 956                 enter.lower(tool);
 957             }
 958         }
 959         assert commit.hasNoUsages();
 960         insertAllocationBarrier(commit, graph);
 961     }
 962 
 963     /**
 964      * Insert the required {@link MemoryBarriers#STORE_STORE} barrier for an allocation and also
 965      * include the {@link MemoryBarriers#LOAD_STORE} required for final fields if any final fields
 966      * are being written, as if {@link FinalFieldBarrierNode} were emitted.
 967      */
 968     private static void insertAllocationBarrier(CommitAllocationNode commit, StructuredGraph graph) {
 969         int barrier = MemoryBarriers.STORE_STORE;
 970         outer: for (VirtualObjectNode vobj : commit.getVirtualObjects()) {
 971             for (ResolvedJavaField field : vobj.type().getInstanceFields(true)) {
 972                 if (field.isFinal()) {
 973                     barrier = barrier | MemoryBarriers.LOAD_STORE;
 974                     break outer;
 975                 }
 976             }
 977         }
 978         graph.addAfterFixed(commit, graph.add(new MembarNode(barrier, LocationIdentity.init())));
 979     }
 980 
 981     /**
 982      * @param field the field whose barrier type should be returned
 983      */
 984     protected BarrierType fieldLoadBarrierType(ResolvedJavaField field) {
 985         return BarrierType.NONE;
 986     }
 987 
 988     protected BarrierType fieldStoreBarrierType(ResolvedJavaField field) {
 989         if (getStorageKind(field) == JavaKind.Object) {
 990             return BarrierType.FIELD;
 991         }
 992         return BarrierType.NONE;
 993     }
 994 
 995     protected BarrierType arrayStoreBarrierType(JavaKind elementKind) {
 996         if (elementKind == JavaKind.Object) {
 997             return BarrierType.ARRAY;
 998         }
 999         return BarrierType.NONE;
1000     }
1001 
1002     public BarrierType fieldInitializationBarrier(JavaKind entryKind) {
1003         return entryKind == JavaKind.Object ? BarrierType.FIELD : BarrierType.NONE;
1004     }
1005 
1006     public BarrierType arrayInitializationBarrier(JavaKind entryKind) {
1007         return entryKind == JavaKind.Object ? BarrierType.ARRAY : BarrierType.NONE;
1008     }
1009 
1010     private BarrierType unsafeStoreBarrierType(RawStoreNode store) {
1011         if (!store.needsBarrier()) {
1012             return BarrierType.NONE;
1013         }
1014         return guessStoreBarrierType(store.object(), store.value());
1015     }
1016 
1017     private BarrierType guessStoreBarrierType(ValueNode object, ValueNode value) {
1018         if (value.getStackKind() == JavaKind.Object && object.getStackKind() == JavaKind.Object) {
1019             ResolvedJavaType type = StampTool.typeOrNull(object);
1020             // Array types must use a precise barrier, so if the type is unknown or is a supertype
1021             // of Object[] then treat it as an array.
1022             if (type != null && type.isArray()) {
1023                 return BarrierType.ARRAY;
1024             } else if (type == null || type.isAssignableFrom(objectArrayType)) {
1025                 return BarrierType.UNKNOWN;
1026             } else {
1027                 return BarrierType.FIELD;
1028             }
1029         }
1030         return BarrierType.NONE;
1031     }
1032 
1033     public abstract int fieldOffset(ResolvedJavaField field);
1034 
1035     public FieldLocationIdentity fieldLocationIdentity(ResolvedJavaField field) {
1036         return new FieldLocationIdentity(field);
1037     }
1038 
1039     public abstract ValueNode staticFieldBase(StructuredGraph graph, ResolvedJavaField field);
1040 
1041     public abstract int arrayLengthOffset();
1042 
1043     public Stamp loadStamp(Stamp stamp, JavaKind kind) {
1044         return loadStamp(stamp, kind, true);
1045     }
1046 
1047     private boolean useCompressedOops(JavaKind kind, boolean compressible) {
1048         return kind == JavaKind.Object && compressible && useCompressedOops;
1049     }
1050 
1051     protected abstract Stamp loadCompressedStamp(ObjectStamp stamp);
1052 
1053     /**
1054      * @param compressible whether the stamp should be compressible
1055      */
1056     protected Stamp loadStamp(Stamp stamp, JavaKind kind, boolean compressible) {
1057         if (useCompressedOops(kind, compressible)) {
1058             return loadCompressedStamp((ObjectStamp) stamp);
1059         }
1060 
1061         switch (kind) {
1062             case Boolean:
1063             case Byte:
1064                 return IntegerStamp.OPS.getNarrow().foldStamp(32, 8, stamp);
1065             case Char:
1066             case Short:
1067                 return IntegerStamp.OPS.getNarrow().foldStamp(32, 16, stamp);
1068         }
1069         return stamp;
1070     }
1071 
1072     public final ValueNode implicitLoadConvert(StructuredGraph graph, JavaKind kind, ValueNode value) {
1073         return implicitLoadConvert(graph, kind, value, true);
1074     }
1075 
1076     public ValueNode implicitLoadConvert(JavaKind kind, ValueNode value) {
1077         return implicitLoadConvert(kind, value, true);
1078     }
1079 
1080     protected final ValueNode implicitLoadConvert(StructuredGraph graph, JavaKind kind, ValueNode value, boolean compressible) {
1081         ValueNode ret = implicitLoadConvert(kind, value, compressible);
1082         if (!ret.isAlive()) {
1083             ret = graph.addOrUnique(ret);
1084         }
1085         return ret;
1086     }
1087 
1088     protected abstract ValueNode newCompressionNode(CompressionOp op, ValueNode value);
1089 
1090     /**
1091      * @param compressible whether the convert should be compressible
1092      */
1093     protected ValueNode implicitLoadConvert(JavaKind kind, ValueNode value, boolean compressible) {
1094         if (useCompressedOops(kind, compressible)) {
1095             return newCompressionNode(CompressionOp.Uncompress, value);
1096         }
1097 
1098         switch (kind) {
1099             case Byte:
1100             case Short:
1101                 return new SignExtendNode(value, 32);
1102             case Boolean:
1103             case Char:
1104                 return new ZeroExtendNode(value, 32);
1105         }
1106         return value;
1107     }
1108 
1109     public final ValueNode implicitStoreConvert(StructuredGraph graph, JavaKind kind, ValueNode value) {
1110         return implicitStoreConvert(graph, kind, value, true);
1111     }
1112 
1113     public ValueNode implicitStoreConvert(JavaKind kind, ValueNode value) {
1114         return implicitStoreConvert(kind, value, true);
1115     }
1116 
1117     protected final ValueNode implicitStoreConvert(StructuredGraph graph, JavaKind kind, ValueNode value, boolean compressible) {
1118         ValueNode ret = implicitStoreConvert(kind, value, compressible);
1119         if (!ret.isAlive()) {
1120             ret = graph.addOrUnique(ret);
1121         }
1122         return ret;
1123     }
1124 
1125     /**
1126      * @param compressible whether the covert should be compressible
1127      */
1128     protected ValueNode implicitStoreConvert(JavaKind kind, ValueNode value, boolean compressible) {
1129         if (useCompressedOops(kind, compressible)) {
1130             return newCompressionNode(CompressionOp.Compress, value);
1131         }
1132 
1133         switch (kind) {
1134             case Boolean:
1135             case Byte:
1136                 return new NarrowNode(value, 8);
1137             case Char:
1138             case Short:
1139                 return new NarrowNode(value, 16);
1140         }
1141         return value;
1142     }
1143 
1144     protected abstract ValueNode createReadHub(StructuredGraph graph, ValueNode object, LoweringTool tool);
1145 
1146     protected abstract ValueNode createReadArrayComponentHub(StructuredGraph graph, ValueNode arrayHub, FixedNode anchor);
1147 
1148     protected ValueNode proxyIndex(AccessIndexedNode n, ValueNode index, ValueNode array, LoweringTool tool) {
1149         StructuredGraph graph = index.graph();
1150         ValueNode arrayLength = readOrCreateArrayLength(n, array, tool, graph);
1151         ValueNode lengthMinusOne = SubNode.create(arrayLength, ConstantNode.forInt(1), NodeView.DEFAULT);
1152         return branchlessMax(branchlessMin(index, lengthMinusOne, NodeView.DEFAULT), ConstantNode.forInt(0), NodeView.DEFAULT);
1153     }
1154 
1155     protected GuardingNode getBoundsCheck(AccessIndexedNode n, ValueNode array, LoweringTool tool) {
1156         if (n.getBoundsCheck() != null) {
1157             return n.getBoundsCheck();
1158         }
1159 
1160         StructuredGraph graph = n.graph();
1161         ValueNode arrayLength = readOrCreateArrayLength(n, array, tool, graph);
1162 
1163         LogicNode boundsCheck = IntegerBelowNode.create(n.index(), arrayLength, NodeView.DEFAULT);
1164         if (boundsCheck.isTautology()) {
1165             return null;
1166         }
1167         return tool.createGuard(n, graph.addOrUniqueWithInputs(boundsCheck), BoundsCheckException, InvalidateReprofile);
1168     }
1169 
1170     private ValueNode readOrCreateArrayLength(AccessIndexedNode n, ValueNode array, LoweringTool tool, StructuredGraph graph) {
1171         ValueNode arrayLength = readArrayLength(array, tool.getConstantReflection());
1172         if (arrayLength == null) {
1173             arrayLength = createReadArrayLength(array, n, tool);
1174         } else {
1175             arrayLength = arrayLength.isAlive() ? arrayLength : graph.addOrUniqueWithInputs(arrayLength);
1176         }
1177         return arrayLength;
1178     }
1179 
1180     protected GuardingNode createNullCheck(ValueNode object, FixedNode before, LoweringTool tool) {
1181         if (StampTool.isPointerNonNull(object)) {
1182             return null;
1183         }
1184         return tool.createGuard(before, before.graph().unique(IsNullNode.create(object)), NullCheckException, InvalidateReprofile, SpeculationLog.NO_SPECULATION, true, null);
1185     }
1186 
1187     protected ValueNode createNullCheckedValue(ValueNode object, FixedNode before, LoweringTool tool) {
1188         GuardingNode nullCheck = createNullCheck(object, before, tool);
1189         if (nullCheck == null) {
1190             return object;
1191         }
1192         return before.graph().maybeAddOrUnique(PiNode.create(object, (object.stamp(NodeView.DEFAULT)).join(StampFactory.objectNonNull()), (ValueNode) nullCheck));
1193     }
1194 
1195     @Override
1196     public ValueNode reconstructArrayIndex(JavaKind elementKind, AddressNode address) {
1197         StructuredGraph graph = address.graph();
1198         ValueNode offset = ((OffsetAddressNode) address).getOffset();
1199 
1200         int base = metaAccess.getArrayBaseOffset(elementKind);
1201         ValueNode scaledIndex = graph.unique(new SubNode(offset, ConstantNode.forIntegerStamp(offset.stamp(NodeView.DEFAULT), base, graph)));
1202 
1203         int shift = CodeUtil.log2(metaAccess.getArrayIndexScale(elementKind));
1204         ValueNode ret = graph.unique(new RightShiftNode(scaledIndex, ConstantNode.forInt(shift, graph)));
1205         return IntegerConvertNode.convert(ret, StampFactory.forKind(JavaKind.Int), graph, NodeView.DEFAULT);
1206     }
1207 }