1 /*
   2  * Copyright (c) 2012, 2016, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  */
  23 package org.graalvm.compiler.hotspot.replacements;
  24 
  25 import static jdk.vm.ci.hotspot.HotSpotJVMCIRuntimeProvider.getArrayBaseOffset;
  26 import static org.graalvm.compiler.core.common.GraalOptions.GeneratePIC;
  27 import static org.graalvm.compiler.core.common.calc.UnsignedMath.belowThan;
  28 import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfig.INJECTED_VMCONFIG;
  29 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.CLASS_ARRAY_KLASS_LOCATION;
  30 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.HUB_WRITE_LOCATION;
  31 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.MARK_WORD_LOCATION;
  32 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.PROTOTYPE_MARK_WORD_LOCATION;
  33 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_END_LOCATION;
  34 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_TOP_LOCATION;
  35 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayAllocationSize;
  36 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayKlassOffset;
  37 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayLengthOffset;
  38 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.config;
  39 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.initializeObjectHeader;
  40 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.instanceHeaderSize;
  41 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.isInstanceKlassFullyInitialized;
  42 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperHeaderSizeMask;
  43 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperHeaderSizeShift;
  44 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperLog2ElementSizeMask;
  45 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperLog2ElementSizeShift;
  46 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.loadKlassFromObject;
  47 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.prototypeMarkWordOffset;
  48 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readLayoutHelper;
  49 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readTlabEnd;
  50 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readTlabTop;
  51 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.registerAsWord;
  52 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.useBiasedLocking;
  53 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.useTLAB;
  54 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.verifyOop;
  55 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.writeTlabTop;
  56 import static org.graalvm.compiler.hotspot.replacements.HotspotSnippetsOptions.ProfileAllocations;
  57 import static org.graalvm.compiler.hotspot.replacements.HotspotSnippetsOptions.ProfileAllocationsContext;
  58 import static org.graalvm.compiler.nodes.PiArrayNode.piArrayCastToSnippetReplaceeStamp;
  59 import static org.graalvm.compiler.nodes.PiNode.piCastToSnippetReplaceeStamp;
  60 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.FAST_PATH_PROBABILITY;
  61 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.FREQUENT_PROBABILITY;
  62 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.SLOW_PATH_PROBABILITY;
  63 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.probability;
  64 import static org.graalvm.compiler.replacements.ReplacementsUtil.REPLACEMENTS_ASSERTIONS_ENABLED;
  65 import static org.graalvm.compiler.replacements.ReplacementsUtil.staticAssert;
  66 import static org.graalvm.compiler.replacements.SnippetTemplate.DEFAULT_REPLACER;
  67 import static org.graalvm.compiler.replacements.nodes.CStringConstant.cstring;
  68 import static org.graalvm.compiler.replacements.nodes.ExplodeLoopNode.explodeLoop;
  69 
  70 import org.graalvm.compiler.api.replacements.Fold;
  71 import org.graalvm.compiler.api.replacements.Snippet;
  72 import org.graalvm.compiler.api.replacements.Snippet.ConstantParameter;
  73 import org.graalvm.compiler.api.replacements.Snippet.VarargsParameter;
  74 import org.graalvm.compiler.core.common.spi.ForeignCallDescriptor;
  75 import org.graalvm.compiler.core.common.type.StampFactory;
  76 import org.graalvm.compiler.debug.DebugHandlersFactory;
  77 import org.graalvm.compiler.debug.GraalError;
  78 import org.graalvm.compiler.graph.Node.ConstantNodeParameter;
  79 import org.graalvm.compiler.graph.Node.NodeIntrinsic;
  80 import org.graalvm.compiler.hotspot.GraalHotSpotVMConfig;
  81 import org.graalvm.compiler.hotspot.HotSpotBackend;
  82 import org.graalvm.compiler.hotspot.meta.HotSpotProviders;
  83 import org.graalvm.compiler.hotspot.meta.HotSpotRegistersProvider;
  84 import org.graalvm.compiler.hotspot.nodes.DimensionsNode;
  85 import org.graalvm.compiler.hotspot.nodes.aot.LoadConstantIndirectlyFixedNode;
  86 import org.graalvm.compiler.hotspot.nodes.aot.LoadConstantIndirectlyNode;
  87 import org.graalvm.compiler.hotspot.nodes.type.KlassPointerStamp;
  88 import org.graalvm.compiler.hotspot.word.KlassPointer;
  89 import org.graalvm.compiler.nodes.ConstantNode;
  90 import org.graalvm.compiler.nodes.DeoptimizeNode;
  91 import org.graalvm.compiler.nodes.PiNode;
  92 import org.graalvm.compiler.nodes.PrefetchAllocateNode;
  93 import org.graalvm.compiler.nodes.SnippetAnchorNode;
  94 import org.graalvm.compiler.nodes.StructuredGraph;
  95 import org.graalvm.compiler.nodes.ValueNode;
  96 import org.graalvm.compiler.nodes.debug.DynamicCounterNode;
  97 import org.graalvm.compiler.nodes.debug.VerifyHeapNode;
  98 import org.graalvm.compiler.nodes.extended.ForeignCallNode;
  99 import org.graalvm.compiler.nodes.extended.MembarNode;
 100 import org.graalvm.compiler.nodes.java.DynamicNewArrayNode;
 101 import org.graalvm.compiler.nodes.java.DynamicNewInstanceNode;
 102 import org.graalvm.compiler.nodes.java.NewArrayNode;
 103 import org.graalvm.compiler.nodes.java.NewInstanceNode;
 104 import org.graalvm.compiler.nodes.java.NewMultiArrayNode;
 105 import org.graalvm.compiler.nodes.memory.address.OffsetAddressNode;
 106 import org.graalvm.compiler.nodes.spi.LoweringTool;
 107 import org.graalvm.compiler.nodes.util.GraphUtil;
 108 import org.graalvm.compiler.options.OptionValues;
 109 import org.graalvm.compiler.replacements.ReplacementsUtil;
 110 import org.graalvm.compiler.replacements.SnippetCounter;
 111 import org.graalvm.compiler.replacements.SnippetCounter.Group;
 112 import org.graalvm.compiler.replacements.SnippetTemplate;
 113 import org.graalvm.compiler.replacements.SnippetTemplate.AbstractTemplates;
 114 import org.graalvm.compiler.replacements.SnippetTemplate.Arguments;
 115 import org.graalvm.compiler.replacements.SnippetTemplate.SnippetInfo;
 116 import org.graalvm.compiler.replacements.Snippets;
 117 import org.graalvm.compiler.replacements.nodes.ExplodeLoopNode;
 118 import org.graalvm.compiler.word.Word;
 119 import org.graalvm.word.LocationIdentity;
 120 import org.graalvm.word.WordFactory;
 121 
 122 import jdk.vm.ci.code.CodeUtil;
 123 import jdk.vm.ci.code.MemoryBarriers;
 124 import jdk.vm.ci.code.Register;
 125 import jdk.vm.ci.code.TargetDescription;
 126 import jdk.vm.ci.hotspot.HotSpotJVMCIRuntimeProvider;
 127 import jdk.vm.ci.hotspot.HotSpotResolvedObjectType;
 128 import jdk.vm.ci.meta.DeoptimizationAction;
 129 import jdk.vm.ci.meta.DeoptimizationReason;
 130 import jdk.vm.ci.meta.JavaKind;
 131 import jdk.vm.ci.meta.ResolvedJavaType;
 132 
 133 /**
 134  * Snippets used for implementing NEW, ANEWARRAY and NEWARRAY.
 135  */
 136 public class NewObjectSnippets implements Snippets {
 137 
 138     enum ProfileContext {
 139         AllocatingMethod,
 140         InstanceOrArray,
 141         AllocatedType,
 142         AllocatedTypesInMethod,
 143         Total
 144     }
 145 
 146     @Fold
 147     static String createName(String path, String typeContext, OptionValues options) {
 148         switch (ProfileAllocationsContext.getValue(options)) {
 149             case AllocatingMethod:
 150                 return "";
 151             case InstanceOrArray:
 152                 return path;
 153             case AllocatedType:
 154             case AllocatedTypesInMethod:
 155                 return typeContext;
 156             case Total:
 157                 return "bytes";
 158             default:
 159                 throw GraalError.shouldNotReachHere();
 160         }
 161     }
 162 
 163     @Fold
 164     static boolean doProfile(OptionValues options) {
 165         return ProfileAllocations.getValue(options);
 166     }
 167 
 168     @Fold
 169     static boolean withContext(OptionValues options) {
 170         ProfileContext context = ProfileAllocationsContext.getValue(options);
 171         return context == ProfileContext.AllocatingMethod || context == ProfileContext.AllocatedTypesInMethod;
 172     }
 173 
 174     protected static void profileAllocation(String path, long size, String typeContext, OptionValues options) {
 175         if (doProfile(options)) {
 176             String name = createName(path, typeContext, options);
 177 
 178             boolean context = withContext(options);
 179             DynamicCounterNode.counter(name, "number of bytes allocated", size, context);
 180             DynamicCounterNode.counter(name, "number of allocations", 1, context);
 181         }
 182     }
 183 
 184     public static void emitPrefetchAllocate(Word address, boolean isArray) {
 185         GraalHotSpotVMConfig config = config(INJECTED_VMCONFIG);
 186         if (config.allocatePrefetchStyle > 0) {
 187             // Insert a prefetch for each allocation only on the fast-path
 188             // Generate several prefetch instructions.
 189             int lines = isArray ? config.allocatePrefetchLines : config.allocateInstancePrefetchLines;
 190             int stepSize = config.allocatePrefetchStepSize;
 191             int distance = config.allocatePrefetchDistance;
 192             ExplodeLoopNode.explodeLoop();
 193             for (int i = 0; i < lines; i++) {
 194                 PrefetchAllocateNode.prefetch(OffsetAddressNode.address(address, distance));
 195                 distance += stepSize;
 196             }
 197         }
 198     }
 199 
 200     @Snippet
 201     public static Object allocateInstance(@ConstantParameter int size, KlassPointer hub, Word prototypeMarkWord, @ConstantParameter boolean fillContents,
 202                     @ConstantParameter Register threadRegister, @ConstantParameter boolean constantSize, @ConstantParameter String typeContext, @ConstantParameter OptionValues options,
 203                     @ConstantParameter Counters counters) {
 204         return piCastToSnippetReplaceeStamp(allocateInstanceHelper(size, hub, prototypeMarkWord, fillContents, threadRegister, constantSize, typeContext, options, counters));
 205     }
 206 
 207     public static Object allocateInstanceHelper(int size, KlassPointer hub, Word prototypeMarkWord, boolean fillContents,
 208                     Register threadRegister, boolean constantSize, String typeContext, OptionValues options, Counters counters) {
 209         Object result;
 210         Word thread = registerAsWord(threadRegister);
 211         Word top = readTlabTop(thread);
 212         Word end = readTlabEnd(thread);
 213         Word newTop = top.add(size);
 214         if (useTLAB(INJECTED_VMCONFIG) && probability(FAST_PATH_PROBABILITY, newTop.belowOrEqual(end))) {
 215             writeTlabTop(thread, newTop);
 216             emitPrefetchAllocate(newTop, false);
 217             result = formatObject(hub, size, top, prototypeMarkWord, fillContents, constantSize, counters);
 218         } else {
 219             if (counters != null && counters.stub != null) {
 220                 counters.stub.inc();
 221             }
 222             result = newInstance(HotSpotBackend.NEW_INSTANCE, hub);
 223         }
 224         profileAllocation("instance", size, typeContext, options);
 225         return verifyOop(result);
 226     }
 227 
 228     @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = true)
 229     public static native Object newInstance(@ConstantNodeParameter ForeignCallDescriptor descriptor, KlassPointer hub);
 230 
 231     @Snippet
 232     public static Object allocateInstancePIC(@ConstantParameter int size, KlassPointer hub, Word prototypeMarkWord, @ConstantParameter boolean fillContents,
 233                     @ConstantParameter Register threadRegister, @ConstantParameter boolean constantSize, @ConstantParameter String typeContext, @ConstantParameter OptionValues options,
 234                     @ConstantParameter Counters counters) {
 235         // Klass must be initialized by the time the first instance is allocated, therefore we can
 236         // just load it from the corresponding cell and avoid the resolution check. We have to use a
 237         // fixed load though, to prevent it from floating above the initialization.
 238         KlassPointer picHub = LoadConstantIndirectlyFixedNode.loadKlass(hub);
 239         return piCastToSnippetReplaceeStamp(allocateInstanceHelper(size, picHub, prototypeMarkWord, fillContents, threadRegister, constantSize, typeContext, options, counters));
 240     }
 241 
 242     @Snippet
 243     public static Object allocateInstanceDynamic(Class<?> type, Class<?> classClass, @ConstantParameter boolean fillContents, @ConstantParameter Register threadRegister,
 244                     @ConstantParameter OptionValues options, @ConstantParameter Counters counters) {
 245         if (probability(SLOW_PATH_PROBABILITY, type == null)) {
 246             DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
 247         }
 248         Class<?> nonNullType = PiNode.piCastNonNullClass(type, SnippetAnchorNode.anchor());
 249 
 250         if (probability(SLOW_PATH_PROBABILITY, DynamicNewInstanceNode.throwsInstantiationException(type, classClass))) {
 251             DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
 252         }
 253 
 254         return PiNode.piCastToSnippetReplaceeStamp(allocateInstanceDynamicHelper(type, fillContents, threadRegister, options, counters, nonNullType));
 255     }
 256 
 257     private static Object allocateInstanceDynamicHelper(Class<?> type, boolean fillContents, Register threadRegister, OptionValues options, Counters counters, Class<?> nonNullType) {
 258         KlassPointer hub = ClassGetHubNode.readClass(nonNullType);
 259         if (probability(FAST_PATH_PROBABILITY, !hub.isNull())) {
 260             KlassPointer nonNullHub = ClassGetHubNode.piCastNonNull(hub, SnippetAnchorNode.anchor());
 261 
 262             if (probability(FAST_PATH_PROBABILITY, isInstanceKlassFullyInitialized(nonNullHub))) {
 263                 int layoutHelper = readLayoutHelper(nonNullHub);
 264                 /*
 265                  * src/share/vm/oops/klass.hpp: For instances, layout helper is a positive number,
 266                  * the instance size. This size is already passed through align_object_size and
 267                  * scaled to bytes. The low order bit is set if instances of this class cannot be
 268                  * allocated using the fastpath.
 269                  */
 270                 if (probability(FAST_PATH_PROBABILITY, (layoutHelper & 1) == 0)) {
 271                     Word prototypeMarkWord = nonNullHub.readWord(prototypeMarkWordOffset(INJECTED_VMCONFIG), PROTOTYPE_MARK_WORD_LOCATION);
 272                     /*
 273                      * FIXME(je,ds): we should actually pass typeContext instead of "" but late
 274                      * binding of parameters is not yet supported by the GraphBuilderPlugin system.
 275                      */
 276                     return allocateInstanceHelper(layoutHelper, nonNullHub, prototypeMarkWord, fillContents, threadRegister, false, "", options, counters);
 277                 }
 278             }
 279         }
 280         return dynamicNewInstanceStub(type);
 281     }
 282 
 283     /**
 284      * Maximum array length for which fast path allocation is used.
 285      */
 286     public static final int MAX_ARRAY_FAST_PATH_ALLOCATION_LENGTH = 0x00FFFFFF;
 287 
 288     @Snippet
 289     public static Object allocatePrimitiveArrayPIC(KlassPointer hub, int length, Word prototypeMarkWord, @ConstantParameter int headerSize, @ConstantParameter int log2ElementSize,
 290                     @ConstantParameter boolean fillContents, @ConstantParameter Register threadRegister, @ConstantParameter boolean maybeUnroll, @ConstantParameter String typeContext,
 291                     @ConstantParameter OptionValues options, @ConstantParameter Counters counters) {
 292         // Primitive array types are eagerly pre-resolved. We can use a floating load.
 293         KlassPointer picHub = LoadConstantIndirectlyNode.loadKlass(hub);
 294         return allocateArrayImpl(picHub, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents, threadRegister, maybeUnroll, typeContext, false, options, counters);
 295     }
 296 
 297     @Snippet
 298     public static Object allocateArrayPIC(KlassPointer hub, int length, Word prototypeMarkWord, @ConstantParameter int headerSize, @ConstantParameter int log2ElementSize,
 299                     @ConstantParameter boolean fillContents, @ConstantParameter Register threadRegister, @ConstantParameter boolean maybeUnroll, @ConstantParameter String typeContext,
 300                     @ConstantParameter OptionValues options, @ConstantParameter Counters counters) {
 301         // Array type would be resolved by dominating resolution.
 302         KlassPointer picHub = LoadConstantIndirectlyFixedNode.loadKlass(hub);
 303         return allocateArrayImpl(picHub, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents, threadRegister, maybeUnroll, typeContext, false, options, counters);
 304     }
 305 
 306     @Snippet
 307     public static Object allocateArray(KlassPointer hub, int length, Word prototypeMarkWord, @ConstantParameter int headerSize, @ConstantParameter int log2ElementSize,
 308                     @ConstantParameter boolean fillContents, @ConstantParameter Register threadRegister, @ConstantParameter boolean maybeUnroll, @ConstantParameter String typeContext,
 309                     @ConstantParameter OptionValues options, @ConstantParameter Counters counters) {
 310         Object result = allocateArrayImpl(hub, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents, threadRegister, maybeUnroll, typeContext, false, options, counters);
 311         return piArrayCastToSnippetReplaceeStamp(verifyOop(result), length);
 312     }
 313 
 314     private static Object allocateArrayImpl(KlassPointer hub, int length, Word prototypeMarkWord, int headerSize, int log2ElementSize, boolean fillContents, Register threadRegister,
 315                     boolean maybeUnroll, String typeContext, boolean skipNegativeCheck, OptionValues options, Counters counters) {
 316         Object result;
 317         int allocationSize = arrayAllocationSize(length, headerSize, log2ElementSize);
 318         Word thread = registerAsWord(threadRegister);
 319         Word top = readTlabTop(thread);
 320         Word end = readTlabEnd(thread);
 321         Word newTop = top.add(allocationSize);
 322         if (probability(FREQUENT_PROBABILITY, skipNegativeCheck || belowThan(length, MAX_ARRAY_FAST_PATH_ALLOCATION_LENGTH)) && useTLAB(INJECTED_VMCONFIG) &&
 323                         probability(FAST_PATH_PROBABILITY, newTop.belowOrEqual(end))) {
 324             writeTlabTop(thread, newTop);
 325             emitPrefetchAllocate(newTop, true);
 326             if (counters != null && counters.arrayLoopInit != null) {
 327                 counters.arrayLoopInit.inc();
 328             }
 329             result = formatArray(hub, allocationSize, length, headerSize, top, prototypeMarkWord, fillContents, maybeUnroll, counters);
 330         } else {
 331             result = newArray(HotSpotBackend.NEW_ARRAY, hub, length, fillContents);
 332         }
 333         profileAllocation("array", allocationSize, typeContext, options);
 334         return result;
 335     }
 336 
 337     @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = true)
 338     public static native Object newArray(@ConstantNodeParameter ForeignCallDescriptor descriptor, KlassPointer hub, int length, boolean fillContents);
 339 
 340     public static final ForeignCallDescriptor DYNAMIC_NEW_ARRAY = new ForeignCallDescriptor("dynamic_new_array", Object.class, Class.class, int.class);
 341     public static final ForeignCallDescriptor DYNAMIC_NEW_INSTANCE = new ForeignCallDescriptor("dynamic_new_instance", Object.class, Class.class);
 342 
 343     @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = true)
 344     public static native Object dynamicNewArrayStub(@ConstantNodeParameter ForeignCallDescriptor descriptor, Class<?> elementType, int length);
 345 
 346     public static Object dynamicNewInstanceStub(Class<?> elementType) {
 347         return dynamicNewInstanceStubCall(DYNAMIC_NEW_INSTANCE, elementType);
 348     }
 349 
 350     @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = true)
 351     public static native Object dynamicNewInstanceStubCall(@ConstantNodeParameter ForeignCallDescriptor descriptor, Class<?> elementType);
 352 
 353     @Snippet
 354     public static Object allocateArrayDynamic(Class<?> elementType, Class<?> voidClass, int length, @ConstantParameter boolean fillContents, @ConstantParameter Register threadRegister,
 355                     @ConstantParameter JavaKind knownElementKind, @ConstantParameter int knownLayoutHelper, Word prototypeMarkWord, @ConstantParameter OptionValues options,
 356                     @ConstantParameter Counters counters) {
 357         Object result = allocateArrayDynamicImpl(elementType, voidClass, length, fillContents, threadRegister, knownElementKind, knownLayoutHelper, prototypeMarkWord, options, counters);
 358         return result;
 359     }
 360 
 361     private static Object allocateArrayDynamicImpl(Class<?> elementType, Class<?> voidClass, int length, boolean fillContents, Register threadRegister, JavaKind knownElementKind,
 362                     int knownLayoutHelper, Word prototypeMarkWord, OptionValues options, Counters counters) {
 363         /*
 364          * We only need the dynamic check for void when we have no static information from
 365          * knownElementKind.
 366          */
 367         staticAssert(knownElementKind != JavaKind.Void, "unsupported knownElementKind");
 368         if (knownElementKind == JavaKind.Illegal && probability(SLOW_PATH_PROBABILITY, elementType == null || DynamicNewArrayNode.throwsIllegalArgumentException(elementType, voidClass))) {
 369             DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
 370         }
 371 
 372         KlassPointer klass = loadKlassFromObject(elementType, arrayKlassOffset(INJECTED_VMCONFIG), CLASS_ARRAY_KLASS_LOCATION);
 373         if (klass.isNull()) {
 374             DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
 375         }
 376         KlassPointer nonNullKlass = ClassGetHubNode.piCastNonNull(klass, SnippetAnchorNode.anchor());
 377 
 378         if (length < 0) {
 379             DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
 380         }
 381         int layoutHelper = knownElementKind != JavaKind.Illegal ? knownLayoutHelper : readLayoutHelper(nonNullKlass);
 382         //@formatter:off
 383         // from src/share/vm/oops/klass.hpp:
 384         //
 385         // For arrays, layout helper is a negative number, containing four
 386         // distinct bytes, as follows:
 387         //    MSB:[tag, hsz, ebt, log2(esz)]:LSB
 388         // where:
 389         //    tag is 0x80 if the elements are oops, 0xC0 if non-oops
 390         //    hsz is array header size in bytes (i.e., offset of first element)
 391         //    ebt is the BasicType of the elements
 392         //    esz is the element size in bytes
 393         //@formatter:on
 394 
 395         int headerSize = (layoutHelper >> layoutHelperHeaderSizeShift(INJECTED_VMCONFIG)) & layoutHelperHeaderSizeMask(INJECTED_VMCONFIG);
 396         int log2ElementSize = (layoutHelper >> layoutHelperLog2ElementSizeShift(INJECTED_VMCONFIG)) & layoutHelperLog2ElementSizeMask(INJECTED_VMCONFIG);
 397 
 398         Object result = allocateArrayImpl(nonNullKlass, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents, threadRegister, false, "dynamic type", true, options, counters);
 399         return piArrayCastToSnippetReplaceeStamp(verifyOop(result), length);
 400     }
 401 
 402     /**
 403      * Calls the runtime stub for implementing MULTIANEWARRAY.
 404      */
 405     @Snippet
 406     public static Object newmultiarray(KlassPointer hub, @ConstantParameter int rank, @VarargsParameter int[] dimensions) {
 407         Word dims = DimensionsNode.allocaDimsArray(rank);
 408         ExplodeLoopNode.explodeLoop();
 409         for (int i = 0; i < rank; i++) {
 410             dims.writeInt(i * 4, dimensions[i], LocationIdentity.init());
 411         }
 412         return newArrayCall(HotSpotBackend.NEW_MULTI_ARRAY, hub, rank, dims);
 413     }
 414 
 415     @Snippet
 416     public static Object newmultiarrayPIC(KlassPointer hub, @ConstantParameter int rank, @VarargsParameter int[] dimensions) {
 417         // Array type would be resolved by dominating resolution.
 418         KlassPointer picHub = LoadConstantIndirectlyFixedNode.loadKlass(hub);
 419         return newmultiarray(picHub, rank, dimensions);
 420     }
 421 
 422     @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = true)
 423     public static native Object newArrayCall(@ConstantNodeParameter ForeignCallDescriptor descriptor, KlassPointer hub, int rank, Word dims);
 424 
 425     /**
 426      * Maximum number of long stores to emit when zeroing an object with a constant size. Larger
 427      * objects have their bodies initialized in a loop.
 428      */
 429     private static final int MAX_UNROLLED_OBJECT_ZEROING_STORES = 8;
 430 
 431     /**
 432      * Zero uninitialized memory in a newly allocated object, unrolling as necessary and ensuring
 433      * that stores are aligned.
 434      *
 435      * @param size number of bytes to zero
 436      * @param memory beginning of object which is being zeroed
 437      * @param constantSize is {@code size} known to be constant in the snippet
 438      * @param startOffset offset to begin zeroing. May not be word aligned.
 439      * @param manualUnroll maximally unroll zeroing
 440      */
 441     private static void zeroMemory(int size, Word memory, boolean constantSize, int startOffset, boolean manualUnroll, Counters counters) {
 442         fillMemory(0, size, memory, constantSize, startOffset, manualUnroll, counters);
 443     }
 444 
 445     private static void fillMemory(long value, int size, Word memory, boolean constantSize, int startOffset, boolean manualUnroll, Counters counters) {
 446         ReplacementsUtil.runtimeAssert((size & 0x7) == 0, "unaligned object size");
 447         int offset = startOffset;
 448         if ((offset & 0x7) != 0) {
 449             memory.writeInt(offset, (int) value, LocationIdentity.init());
 450             offset += 4;
 451         }
 452         ReplacementsUtil.runtimeAssert((offset & 0x7) == 0, "unaligned offset");
 453         if (manualUnroll && ((size - offset) / 8) <= MAX_UNROLLED_OBJECT_ZEROING_STORES) {
 454             ReplacementsUtil.staticAssert(!constantSize, "size shouldn't be constant at instantiation time");
 455             // This case handles arrays of constant length. Instead of having a snippet variant for
 456             // each length, generate a chain of stores of maximum length. Once it's inlined the
 457             // break statement will trim excess stores.
 458             if (counters != null && counters.instanceSeqInit != null) {
 459                 counters.instanceSeqInit.inc();
 460             }
 461 
 462             explodeLoop();
 463             for (int i = 0; i < MAX_UNROLLED_OBJECT_ZEROING_STORES; i++, offset += 8) {
 464                 if (offset == size) {
 465                     break;
 466                 }
 467                 memory.initializeLong(offset, value, LocationIdentity.init());
 468             }
 469         } else {
 470             // Use Word instead of int to avoid extension to long in generated code
 471             Word off = WordFactory.signed(offset);
 472             if (constantSize && ((size - offset) / 8) <= MAX_UNROLLED_OBJECT_ZEROING_STORES) {
 473                 if (counters != null && counters.instanceSeqInit != null) {
 474                     counters.instanceSeqInit.inc();
 475                 }
 476                 explodeLoop();
 477             } else {
 478                 if (counters != null && counters.instanceLoopInit != null) {
 479                     counters.instanceLoopInit.inc();
 480                 }
 481             }
 482             for (; off.rawValue() < size; off = off.add(8)) {
 483                 memory.initializeLong(off, value, LocationIdentity.init());
 484             }
 485         }
 486     }
 487 
 488     /**
 489      * Fill uninitialized memory with garbage value in a newly allocated object, unrolling as
 490      * necessary and ensuring that stores are aligned.
 491      *
 492      * @param size number of bytes to zero
 493      * @param memory beginning of object which is being zeroed
 494      * @param constantSize is {@code  size} known to be constant in the snippet
 495      * @param startOffset offset to begin zeroing. May not be word aligned.
 496      * @param manualUnroll maximally unroll zeroing
 497      */
 498     private static void fillWithGarbage(int size, Word memory, boolean constantSize, int startOffset, boolean manualUnroll, Counters counters) {
 499         fillMemory(0xfefefefefefefefeL, size, memory, constantSize, startOffset, manualUnroll, counters);
 500     }
 501 
 502     /**
 503      * Formats some allocated memory with an object header and zeroes out the rest. Disables asserts
 504      * since they can't be compiled in stubs.
 505      */
 506     public static Object formatObjectForStub(KlassPointer hub, int size, Word memory, Word compileTimePrototypeMarkWord) {
 507         return formatObject(hub, size, memory, compileTimePrototypeMarkWord, true, false, null);
 508     }
 509 
 510     /**
 511      * Formats some allocated memory with an object header and zeroes out the rest.
 512      */
 513     protected static Object formatObject(KlassPointer hub, int size, Word memory, Word compileTimePrototypeMarkWord, boolean fillContents, boolean constantSize, Counters counters) {
 514         Word prototypeMarkWord = useBiasedLocking(INJECTED_VMCONFIG) ? hub.readWord(prototypeMarkWordOffset(INJECTED_VMCONFIG), PROTOTYPE_MARK_WORD_LOCATION) : compileTimePrototypeMarkWord;
 515         initializeObjectHeader(memory, prototypeMarkWord, hub);
 516         if (fillContents) {
 517             zeroMemory(size, memory, constantSize, instanceHeaderSize(INJECTED_VMCONFIG), false, counters);
 518         } else if (REPLACEMENTS_ASSERTIONS_ENABLED) {
 519             fillWithGarbage(size, memory, constantSize, instanceHeaderSize(INJECTED_VMCONFIG), false, counters);
 520         }
 521         MembarNode.memoryBarrier(MemoryBarriers.STORE_STORE, LocationIdentity.init());
 522         return memory.toObjectNonNull();
 523     }
 524 
 525     @Snippet
 526     protected static void verifyHeap(@ConstantParameter Register threadRegister) {
 527         Word thread = registerAsWord(threadRegister);
 528         Word topValue = readTlabTop(thread);
 529         if (!topValue.equal(WordFactory.zero())) {
 530             Word topValueContents = topValue.readWord(0, MARK_WORD_LOCATION);
 531             if (topValueContents.equal(WordFactory.zero())) {
 532                 AssertionSnippets.vmMessageC(AssertionSnippets.ASSERTION_VM_MESSAGE_C, true, cstring("overzeroing of TLAB detected"), 0L, 0L, 0L);
 533             }
 534         }
 535     }
 536 
 537     /**
 538      * Formats some allocated memory with an object header and zeroes out the rest.
 539      */
 540     public static Object formatArray(KlassPointer hub, int allocationSize, int length, int headerSize, Word memory, Word prototypeMarkWord, boolean fillContents, boolean maybeUnroll,
 541                     Counters counters) {
 542         memory.writeInt(arrayLengthOffset(INJECTED_VMCONFIG), length, LocationIdentity.init());
 543         /*
 544          * store hub last as the concurrent garbage collectors assume length is valid if hub field
 545          * is not null
 546          */
 547         initializeObjectHeader(memory, prototypeMarkWord, hub);
 548         if (fillContents) {
 549             zeroMemory(allocationSize, memory, false, headerSize, maybeUnroll, counters);
 550         } else if (REPLACEMENTS_ASSERTIONS_ENABLED) {
 551             fillWithGarbage(allocationSize, memory, false, headerSize, maybeUnroll, counters);
 552         }
 553         MembarNode.memoryBarrier(MemoryBarriers.STORE_STORE, LocationIdentity.init());
 554         return memory.toObjectNonNull();
 555     }
 556 
 557     static class Counters {
 558         Counters(SnippetCounter.Group.Factory factory) {
 559             Group newInstance = factory.createSnippetCounterGroup("NewInstance");
 560             Group newArray = factory.createSnippetCounterGroup("NewArray");
 561             instanceSeqInit = new SnippetCounter(newInstance, "tlabSeqInit", "TLAB alloc with unrolled zeroing");
 562             instanceLoopInit = new SnippetCounter(newInstance, "tlabLoopInit", "TLAB alloc with zeroing in a loop");
 563             arrayLoopInit = new SnippetCounter(newArray, "tlabLoopInit", "TLAB alloc with zeroing in a loop");
 564             stub = new SnippetCounter(newInstance, "stub", "alloc and zeroing via stub");
 565         }
 566 
 567         final SnippetCounter instanceSeqInit;
 568         final SnippetCounter instanceLoopInit;
 569         final SnippetCounter arrayLoopInit;
 570         final SnippetCounter stub;
 571     }
 572 
 573     public static class Templates extends AbstractTemplates {
 574 
 575         private final SnippetInfo allocateInstance = snippet(NewObjectSnippets.class, "allocateInstance", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION, TLAB_END_LOCATION);
 576         private final SnippetInfo allocateInstancePIC = snippet(NewObjectSnippets.class, "allocateInstancePIC", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION,
 577                         TLAB_END_LOCATION);
 578         private final SnippetInfo allocateArray = snippet(NewObjectSnippets.class, "allocateArray", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION, TLAB_END_LOCATION);
 579         private final SnippetInfo allocateArrayPIC = snippet(NewObjectSnippets.class, "allocateArrayPIC", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION, TLAB_END_LOCATION);
 580         private final SnippetInfo allocatePrimitiveArrayPIC = snippet(NewObjectSnippets.class, "allocatePrimitiveArrayPIC", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION,
 581                         TLAB_END_LOCATION);
 582         private final SnippetInfo allocateArrayDynamic = snippet(NewObjectSnippets.class, "allocateArrayDynamic", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION,
 583                         TLAB_END_LOCATION);
 584         private final SnippetInfo allocateInstanceDynamic = snippet(NewObjectSnippets.class, "allocateInstanceDynamic", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION,
 585                         TLAB_END_LOCATION);
 586         private final SnippetInfo newmultiarray = snippet(NewObjectSnippets.class, "newmultiarray", TLAB_TOP_LOCATION, TLAB_END_LOCATION);
 587         private final SnippetInfo newmultiarrayPIC = snippet(NewObjectSnippets.class, "newmultiarrayPIC", TLAB_TOP_LOCATION, TLAB_END_LOCATION);
 588         private final SnippetInfo verifyHeap = snippet(NewObjectSnippets.class, "verifyHeap");
 589         private final GraalHotSpotVMConfig config;
 590         private final Counters counters;
 591 
 592         public Templates(OptionValues options, Iterable<DebugHandlersFactory> factories, SnippetCounter.Group.Factory factory, HotSpotProviders providers, TargetDescription target,
 593                         GraalHotSpotVMConfig config) {
 594             super(options, factories, providers, providers.getSnippetReflection(), target);
 595             this.config = config;
 596             counters = new Counters(factory);
 597         }
 598 
 599         /**
 600          * Lowers a {@link NewInstanceNode}.
 601          */
 602         public void lower(NewInstanceNode newInstanceNode, HotSpotRegistersProvider registers, LoweringTool tool) {
 603             StructuredGraph graph = newInstanceNode.graph();
 604             HotSpotResolvedObjectType type = (HotSpotResolvedObjectType) newInstanceNode.instanceClass();
 605             assert !type.isArray();
 606             ConstantNode hub = ConstantNode.forConstant(KlassPointerStamp.klassNonNull(), type.klass(), providers.getMetaAccess(), graph);
 607             int size = instanceSize(type);
 608 
 609             OptionValues localOptions = graph.getOptions();
 610             SnippetInfo snippet = GeneratePIC.getValue(localOptions) ? allocateInstancePIC : allocateInstance;
 611             Arguments args = new Arguments(snippet, graph.getGuardsStage(), tool.getLoweringStage());
 612             args.addConst("size", size);
 613             args.add("hub", hub);
 614             args.add("prototypeMarkWord", type.prototypeMarkWord());
 615             args.addConst("fillContents", newInstanceNode.fillContents());
 616             args.addConst("threadRegister", registers.getThreadRegister());
 617             args.addConst("constantSize", true);
 618             args.addConst("typeContext", ProfileAllocations.getValue(localOptions) ? type.toJavaName(false) : "");
 619             args.addConst("options", localOptions);
 620             args.addConst("counters", counters);
 621 
 622             SnippetTemplate template = template(graph.getDebug(), args);
 623             graph.getDebug().log("Lowering allocateInstance in %s: node=%s, template=%s, arguments=%s", graph, newInstanceNode, template, args);
 624             template.instantiate(providers.getMetaAccess(), newInstanceNode, DEFAULT_REPLACER, args);
 625         }
 626 
 627         /**
 628          * Lowers a {@link NewArrayNode}.
 629          */
 630         public void lower(NewArrayNode newArrayNode, HotSpotRegistersProvider registers, LoweringTool tool) {
 631             StructuredGraph graph = newArrayNode.graph();
 632             ResolvedJavaType elementType = newArrayNode.elementType();
 633             HotSpotResolvedObjectType arrayType = (HotSpotResolvedObjectType) elementType.getArrayClass();
 634             JavaKind elementKind = elementType.getJavaKind();
 635             ConstantNode hub = ConstantNode.forConstant(KlassPointerStamp.klassNonNull(), arrayType.klass(), providers.getMetaAccess(), graph);
 636             final int headerSize = getArrayBaseOffset(elementKind);
 637             int log2ElementSize = CodeUtil.log2(HotSpotJVMCIRuntimeProvider.getArrayIndexScale(elementKind));
 638 
 639             OptionValues localOptions = graph.getOptions();
 640             SnippetInfo snippet;
 641             if (GeneratePIC.getValue(localOptions)) {
 642                 if (elementType.isPrimitive()) {
 643                     snippet = allocatePrimitiveArrayPIC;
 644                 } else {
 645                     snippet = allocateArrayPIC;
 646                 }
 647             } else {
 648                 snippet = allocateArray;
 649             }
 650 
 651             Arguments args = new Arguments(snippet, graph.getGuardsStage(), tool.getLoweringStage());
 652             args.add("hub", hub);
 653             ValueNode length = newArrayNode.length();
 654             args.add("length", length.isAlive() ? length : graph.addOrUniqueWithInputs(length));
 655             assert arrayType.prototypeMarkWord() == lookupArrayClass(tool, JavaKind.Object).prototypeMarkWord() : "all array types are assumed to have the same prototypeMarkWord";
 656             args.add("prototypeMarkWord", arrayType.prototypeMarkWord());
 657             args.addConst("headerSize", headerSize);
 658             args.addConst("log2ElementSize", log2ElementSize);
 659             args.addConst("fillContents", newArrayNode.fillContents());
 660             args.addConst("threadRegister", registers.getThreadRegister());
 661             args.addConst("maybeUnroll", length.isConstant());
 662             args.addConst("typeContext", ProfileAllocations.getValue(localOptions) ? arrayType.toJavaName(false) : "");
 663             args.addConst("options", localOptions);
 664             args.addConst("counters", counters);
 665             SnippetTemplate template = template(graph.getDebug(), args);
 666             graph.getDebug().log("Lowering allocateArray in %s: node=%s, template=%s, arguments=%s", graph, newArrayNode, template, args);
 667             template.instantiate(providers.getMetaAccess(), newArrayNode, DEFAULT_REPLACER, args);
 668         }
 669 
 670         public void lower(DynamicNewInstanceNode newInstanceNode, HotSpotRegistersProvider registers, LoweringTool tool) {
 671             Arguments args = new Arguments(allocateInstanceDynamic, newInstanceNode.graph().getGuardsStage(), tool.getLoweringStage());
 672             OptionValues localOptions = newInstanceNode.getOptions();
 673             args.add("type", newInstanceNode.getInstanceType());
 674             ValueNode classClass = newInstanceNode.getClassClass();
 675             assert classClass != null;
 676             args.add("classClass", classClass);
 677             args.addConst("fillContents", newInstanceNode.fillContents());
 678             args.addConst("threadRegister", registers.getThreadRegister());
 679             args.addConst("options", localOptions);
 680             args.addConst("counters", counters);
 681 
 682             SnippetTemplate template = template(newInstanceNode.getDebug(), args);
 683             template.instantiate(providers.getMetaAccess(), newInstanceNode, DEFAULT_REPLACER, args);
 684         }
 685 
 686         public void lower(DynamicNewArrayNode newArrayNode, HotSpotRegistersProvider registers, LoweringTool tool) {
 687             StructuredGraph graph = newArrayNode.graph();
 688             OptionValues localOptions = graph.getOptions();
 689             Arguments args = new Arguments(allocateArrayDynamic, newArrayNode.graph().getGuardsStage(), tool.getLoweringStage());
 690             args.add("elementType", newArrayNode.getElementType());
 691             ValueNode voidClass = newArrayNode.getVoidClass();
 692             assert voidClass != null;
 693             args.add("voidClass", voidClass);
 694             ValueNode length = newArrayNode.length();
 695             args.add("length", length.isAlive() ? length : graph.addOrUniqueWithInputs(length));
 696             args.addConst("fillContents", newArrayNode.fillContents());
 697             args.addConst("threadRegister", registers.getThreadRegister());
 698             /*
 699              * We use Kind.Illegal as a marker value instead of null because constant snippet
 700              * parameters cannot be null.
 701              */
 702             args.addConst("knownElementKind", newArrayNode.getKnownElementKind() == null ? JavaKind.Illegal : newArrayNode.getKnownElementKind());
 703             if (newArrayNode.getKnownElementKind() != null) {
 704                 args.addConst("knownLayoutHelper", lookupArrayClass(tool, newArrayNode.getKnownElementKind()).layoutHelper());
 705             } else {
 706                 args.addConst("knownLayoutHelper", 0);
 707             }
 708             args.add("prototypeMarkWord", lookupArrayClass(tool, JavaKind.Object).prototypeMarkWord());
 709             args.addConst("options", localOptions);
 710             args.addConst("counters", counters);
 711             SnippetTemplate template = template(graph.getDebug(), args);
 712             template.instantiate(providers.getMetaAccess(), newArrayNode, DEFAULT_REPLACER, args);
 713         }
 714 
 715         private static HotSpotResolvedObjectType lookupArrayClass(LoweringTool tool, JavaKind kind) {
 716             return (HotSpotResolvedObjectType) tool.getMetaAccess().lookupJavaType(kind == JavaKind.Object ? Object.class : kind.toJavaClass()).getArrayClass();
 717         }
 718 
 719         public void lower(NewMultiArrayNode newmultiarrayNode, LoweringTool tool) {
 720             StructuredGraph graph = newmultiarrayNode.graph();
 721             OptionValues localOptions = graph.getOptions();
 722             int rank = newmultiarrayNode.dimensionCount();
 723             ValueNode[] dims = new ValueNode[rank];
 724             for (int i = 0; i < newmultiarrayNode.dimensionCount(); i++) {
 725                 dims[i] = newmultiarrayNode.dimension(i);
 726             }
 727             HotSpotResolvedObjectType type = (HotSpotResolvedObjectType) newmultiarrayNode.type();
 728             ConstantNode hub = ConstantNode.forConstant(KlassPointerStamp.klassNonNull(), type.klass(), providers.getMetaAccess(), graph);
 729 
 730             SnippetInfo snippet = GeneratePIC.getValue(localOptions) ? newmultiarrayPIC : newmultiarray;
 731             Arguments args = new Arguments(snippet, graph.getGuardsStage(), tool.getLoweringStage());
 732             args.add("hub", hub);
 733             args.addConst("rank", rank);
 734             args.addVarargs("dimensions", int.class, StampFactory.forKind(JavaKind.Int), dims);
 735             template(newmultiarrayNode.getDebug(), args).instantiate(providers.getMetaAccess(), newmultiarrayNode, DEFAULT_REPLACER, args);
 736         }
 737 
 738         private static int instanceSize(HotSpotResolvedObjectType type) {
 739             int size = type.instanceSize();
 740             assert size >= 0;
 741             return size;
 742         }
 743 
 744         public void lower(VerifyHeapNode verifyHeapNode, HotSpotRegistersProvider registers, LoweringTool tool) {
 745             if (config.cAssertions) {
 746                 Arguments args = new Arguments(verifyHeap, verifyHeapNode.graph().getGuardsStage(), tool.getLoweringStage());
 747                 args.addConst("threadRegister", registers.getThreadRegister());
 748 
 749                 SnippetTemplate template = template(verifyHeapNode.getDebug(), args);
 750                 template.instantiate(providers.getMetaAccess(), verifyHeapNode, DEFAULT_REPLACER, args);
 751             } else {
 752                 GraphUtil.removeFixedWithUnusedInputs(verifyHeapNode);
 753             }
 754         }
 755     }
 756 }