1 /*
   2  * Copyright (c) 2012, 2018, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  */
  23 
  24 
  25 package org.graalvm.compiler.hotspot.replacements;
  26 
  27 import static jdk.vm.ci.meta.DeoptimizationAction.None;
  28 import static jdk.vm.ci.meta.DeoptimizationReason.RuntimeConstraint;
  29 import static org.graalvm.compiler.core.common.GraalOptions.GeneratePIC;
  30 import static org.graalvm.compiler.core.common.calc.UnsignedMath.belowThan;
  31 import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfigBase.INJECTED_OPTIONVALUES;
  32 import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfigBase.INJECTED_VMCONFIG;
  33 import static org.graalvm.compiler.hotspot.HotSpotBackend.NEW_ARRAY;
  34 import static org.graalvm.compiler.hotspot.HotSpotBackend.NEW_ARRAY_OR_NULL;
  35 import static org.graalvm.compiler.hotspot.HotSpotBackend.NEW_INSTANCE;
  36 import static org.graalvm.compiler.hotspot.HotSpotBackend.NEW_INSTANCE_OR_NULL;
  37 import static org.graalvm.compiler.hotspot.HotSpotBackend.NEW_MULTI_ARRAY;
  38 import static org.graalvm.compiler.hotspot.HotSpotBackend.NEW_MULTI_ARRAY_OR_NULL;
  39 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.CLASS_ARRAY_KLASS_LOCATION;
  40 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.HUB_WRITE_LOCATION;
  41 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.MARK_WORD_LOCATION;
  42 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.PROTOTYPE_MARK_WORD_LOCATION;
  43 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_END_LOCATION;
  44 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_TOP_LOCATION;
  45 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.allocateInstancePrefetchLines;
  46 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.allocatePrefetchDistance;
  47 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.allocatePrefetchLines;
  48 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.allocatePrefetchStepSize;
  49 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.allocatePrefetchStyle;
  50 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayAllocationSize;
  51 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayKlassOffset;
  52 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayLengthOffset;
  53 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.initializeObjectHeader;
  54 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.instanceHeaderSize;
  55 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.isInstanceKlassFullyInitialized;
  56 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperHeaderSizeMask;
  57 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperHeaderSizeShift;
  58 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperLog2ElementSizeMask;
  59 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperLog2ElementSizeShift;
  60 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.loadKlassFromObject;
  61 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.prototypeMarkWordOffset;
  62 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readLayoutHelper;
  63 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readTlabEnd;
  64 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readTlabTop;
  65 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.registerAsWord;
  66 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.useBiasedLocking;
  67 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.useTLAB;
  68 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.verifyOop;
  69 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.writeTlabTop;
  70 import static org.graalvm.compiler.hotspot.replacements.HotspotSnippetsOptions.ProfileAllocations;
  71 import static org.graalvm.compiler.hotspot.replacements.HotspotSnippetsOptions.ProfileAllocationsContext;
  72 import static org.graalvm.compiler.nodes.PiArrayNode.piArrayCastToSnippetReplaceeStamp;
  73 import static org.graalvm.compiler.nodes.PiNode.piCastToSnippetReplaceeStamp;
  74 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.FAST_PATH_PROBABILITY;
  75 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.FREQUENT_PROBABILITY;
  76 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.SLOW_PATH_PROBABILITY;
  77 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.probability;
  78 import static org.graalvm.compiler.replacements.ReplacementsUtil.REPLACEMENTS_ASSERTIONS_ENABLED;
  79 import static org.graalvm.compiler.replacements.ReplacementsUtil.runtimeAssert;
  80 import static org.graalvm.compiler.replacements.ReplacementsUtil.staticAssert;
  81 import static org.graalvm.compiler.replacements.SnippetTemplate.DEFAULT_REPLACER;
  82 import static org.graalvm.compiler.replacements.nodes.CStringConstant.cstring;
  83 import static org.graalvm.compiler.replacements.nodes.ExplodeLoopNode.explodeLoop;
  84 
  85 import org.graalvm.compiler.api.replacements.Fold;
  86 import org.graalvm.compiler.api.replacements.Fold.InjectedParameter;
  87 import org.graalvm.compiler.api.replacements.Snippet;
  88 import org.graalvm.compiler.api.replacements.Snippet.ConstantParameter;
  89 import org.graalvm.compiler.api.replacements.Snippet.VarargsParameter;
  90 import org.graalvm.compiler.core.common.spi.ForeignCallDescriptor;
  91 import org.graalvm.compiler.core.common.type.StampFactory;
  92 import org.graalvm.compiler.debug.DebugHandlersFactory;
  93 import org.graalvm.compiler.debug.GraalError;
  94 import org.graalvm.compiler.graph.Node.ConstantNodeParameter;
  95 import org.graalvm.compiler.graph.Node.NodeIntrinsic;
  96 import org.graalvm.compiler.hotspot.GraalHotSpotVMConfig;
  97 import org.graalvm.compiler.hotspot.meta.HotSpotProviders;
  98 import org.graalvm.compiler.hotspot.meta.HotSpotRegistersProvider;
  99 import org.graalvm.compiler.hotspot.nodes.DimensionsNode;
 100 import org.graalvm.compiler.hotspot.nodes.aot.LoadConstantIndirectlyFixedNode;
 101 import org.graalvm.compiler.hotspot.nodes.aot.LoadConstantIndirectlyNode;
 102 import org.graalvm.compiler.hotspot.nodes.type.KlassPointerStamp;
 103 import org.graalvm.compiler.hotspot.word.KlassPointer;
 104 import org.graalvm.compiler.nodes.ConstantNode;
 105 import org.graalvm.compiler.nodes.DeoptimizeNode;
 106 import org.graalvm.compiler.nodes.PiNode;
 107 import org.graalvm.compiler.nodes.PrefetchAllocateNode;
 108 import org.graalvm.compiler.nodes.SnippetAnchorNode;
 109 import org.graalvm.compiler.nodes.StructuredGraph;
 110 import org.graalvm.compiler.nodes.ValueNode;
 111 import org.graalvm.compiler.nodes.debug.DynamicCounterNode;
 112 import org.graalvm.compiler.nodes.debug.VerifyHeapNode;
 113 import org.graalvm.compiler.nodes.extended.ForeignCallNode;
 114 import org.graalvm.compiler.nodes.extended.MembarNode;
 115 import org.graalvm.compiler.nodes.java.DynamicNewArrayNode;
 116 import org.graalvm.compiler.nodes.java.DynamicNewInstanceNode;
 117 import org.graalvm.compiler.nodes.java.NewArrayNode;
 118 import org.graalvm.compiler.nodes.java.NewInstanceNode;
 119 import org.graalvm.compiler.nodes.java.NewMultiArrayNode;
 120 import org.graalvm.compiler.nodes.memory.address.OffsetAddressNode;
 121 import org.graalvm.compiler.nodes.spi.LoweringTool;
 122 import org.graalvm.compiler.nodes.util.GraphUtil;
 123 import org.graalvm.compiler.options.OptionValues;
 124 import org.graalvm.compiler.replacements.ReplacementsUtil;
 125 import org.graalvm.compiler.replacements.SnippetCounter;
 126 import org.graalvm.compiler.replacements.SnippetCounter.Group;
 127 import org.graalvm.compiler.replacements.SnippetTemplate;
 128 import org.graalvm.compiler.replacements.SnippetTemplate.AbstractTemplates;
 129 import org.graalvm.compiler.replacements.SnippetTemplate.Arguments;
 130 import org.graalvm.compiler.replacements.SnippetTemplate.SnippetInfo;
 131 import org.graalvm.compiler.replacements.Snippets;
 132 import org.graalvm.compiler.replacements.nodes.ExplodeLoopNode;
 133 import org.graalvm.compiler.word.Word;
 134 import jdk.internal.vm.compiler.word.LocationIdentity;
 135 import jdk.internal.vm.compiler.word.WordFactory;
 136 
 137 import jdk.vm.ci.code.CodeUtil;
 138 import jdk.vm.ci.code.MemoryBarriers;
 139 import jdk.vm.ci.code.Register;
 140 import jdk.vm.ci.code.TargetDescription;
 141 import jdk.vm.ci.hotspot.HotSpotResolvedObjectType;
 142 import jdk.vm.ci.meta.JavaKind;
 143 import jdk.vm.ci.meta.ResolvedJavaType;
 144 
 145 /**
 146  * Snippets used for implementing NEW, ANEWARRAY and NEWARRAY.
 147  */
 148 public class NewObjectSnippets implements Snippets {
 149 
 150     enum ProfileContext {
 151         AllocatingMethod,
 152         InstanceOrArray,
 153         AllocatedType,
 154         AllocatedTypesInMethod,
 155         Total
 156     }
 157 
 158     @Fold
 159     static String createName(@Fold.InjectedParameter OptionValues options, String path, String typeContext) {
 160         switch (ProfileAllocationsContext.getValue(options)) {
 161             case AllocatingMethod:
 162                 return "";
 163             case InstanceOrArray:
 164                 return path;
 165             case AllocatedType:
 166             case AllocatedTypesInMethod:
 167                 return typeContext;
 168             case Total:
 169                 return "bytes";
 170             default:
 171                 throw GraalError.shouldNotReachHere();
 172         }
 173     }
 174 
 175     @Fold
 176     static boolean doProfile(@Fold.InjectedParameter OptionValues options) {
 177         return ProfileAllocations.getValue(options);
 178     }
 179 
 180     @Fold
 181     static boolean withContext(@Fold.InjectedParameter OptionValues options) {
 182         ProfileContext context = ProfileAllocationsContext.getValue(options);
 183         return context == ProfileContext.AllocatingMethod || context == ProfileContext.AllocatedTypesInMethod;
 184     }
 185 
 186     protected static void profileAllocation(String path, long size, String typeContext) {
 187         if (doProfile(INJECTED_OPTIONVALUES)) {
 188             String name = createName(INJECTED_OPTIONVALUES, path, typeContext);
 189 
 190             boolean context = withContext(INJECTED_OPTIONVALUES);
 191             DynamicCounterNode.counter("number of bytes allocated", name, size, context);
 192             DynamicCounterNode.counter("number of allocations", name, 1, context);
 193         }
 194     }
 195 
 196     public static void emitPrefetchAllocate(Word address, boolean isArray) {
 197         if (allocatePrefetchStyle(INJECTED_VMCONFIG) > 0) {
 198             // Insert a prefetch for each allocation only on the fast-path
 199             // Generate several prefetch instructions.
 200             int lines = isArray ? allocatePrefetchLines(INJECTED_VMCONFIG) : allocateInstancePrefetchLines(INJECTED_VMCONFIG);
 201             int stepSize = allocatePrefetchStepSize(INJECTED_VMCONFIG);
 202             int distance = allocatePrefetchDistance(INJECTED_VMCONFIG);
 203             ExplodeLoopNode.explodeLoop();
 204             for (int i = 0; i < lines; i++) {
 205                 PrefetchAllocateNode.prefetch(OffsetAddressNode.address(address, distance));
 206                 distance += stepSize;
 207             }
 208         }
 209     }
 210 
 211     @Snippet
 212     public static Object allocateInstance(@ConstantParameter long size, KlassPointer hub, Word prototypeMarkWord, @ConstantParameter boolean fillContents,
 213                     @ConstantParameter Register threadRegister, @ConstantParameter boolean constantSize, @ConstantParameter String typeContext,
 214                     @ConstantParameter Counters counters) {
 215         return piCastToSnippetReplaceeStamp(allocateInstanceHelper(size, hub, prototypeMarkWord, fillContents, threadRegister, constantSize, typeContext, counters));
 216     }
 217 
 218     public static Object allocateInstanceHelper(long size, KlassPointer hub, Word prototypeMarkWord, boolean fillContents,
 219                     Register threadRegister, boolean constantSize, String typeContext, Counters counters) {
 220         Object result;
 221         Word thread = registerAsWord(threadRegister);
 222         Word top = readTlabTop(thread);
 223         Word end = readTlabEnd(thread);
 224         Word newTop = top.add(WordFactory.unsigned(size));
 225         if (useTLAB(INJECTED_VMCONFIG) && probability(FAST_PATH_PROBABILITY, newTop.belowOrEqual(end))) {
 226             writeTlabTop(thread, newTop);
 227             emitPrefetchAllocate(newTop, false);
 228             result = formatObject(hub, size, top, prototypeMarkWord, fillContents, constantSize, counters);
 229         } else {
 230             Counters theCounters = counters;
 231             if (theCounters != null && theCounters.stub != null) {
 232                 theCounters.stub.inc();
 233             }
 234             result = newInstanceStub(hub);
 235         }
 236         profileAllocation("instance", size, typeContext);
 237         return verifyOop(result);
 238     }
 239 
 240     public static Object newInstanceStub(KlassPointer hub) {
 241         if (useNullAllocationStubs(INJECTED_VMCONFIG)) {
 242             return nonNullOrDeopt(newInstanceOrNull(NEW_INSTANCE_OR_NULL, hub));
 243         } else {
 244             return newInstance(NEW_INSTANCE, hub);
 245         }
 246     }
 247 
 248     @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = true)
 249     private static native Object newInstance(@ConstantNodeParameter ForeignCallDescriptor descriptor, KlassPointer hub);
 250 
 251     @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = false)
 252     private static native Object newInstanceOrNull(@ConstantNodeParameter ForeignCallDescriptor descriptor, KlassPointer hub);
 253 
 254     @Snippet
 255     public static Object allocateInstancePIC(@ConstantParameter long size, KlassPointer hub, Word prototypeMarkWord, @ConstantParameter boolean fillContents,
 256                     @ConstantParameter Register threadRegister, @ConstantParameter boolean constantSize, @ConstantParameter String typeContext,
 257                     @ConstantParameter Counters counters) {
 258         // Klass must be initialized by the time the first instance is allocated, therefore we can
 259         // just load it from the corresponding cell and avoid the resolution check. We have to use a
 260         // fixed load though, to prevent it from floating above the initialization.
 261         KlassPointer picHub = LoadConstantIndirectlyFixedNode.loadKlass(hub);
 262         return piCastToSnippetReplaceeStamp(allocateInstanceHelper(size, picHub, prototypeMarkWord, fillContents, threadRegister, constantSize, typeContext, counters));
 263     }
 264 
 265     @Snippet
 266     public static Object allocateInstanceDynamic(Class<?> type, Class<?> classClass, @ConstantParameter boolean fillContents, @ConstantParameter Register threadRegister,
 267                     @ConstantParameter Counters counters) {
 268         if (probability(SLOW_PATH_PROBABILITY, type == null)) {
 269             DeoptimizeNode.deopt(None, RuntimeConstraint);
 270         }
 271         Class<?> nonNullType = PiNode.piCastNonNullClass(type, SnippetAnchorNode.anchor());
 272 
 273         if (probability(SLOW_PATH_PROBABILITY, DynamicNewInstanceNode.throwsInstantiationException(type, classClass))) {
 274             DeoptimizeNode.deopt(None, RuntimeConstraint);
 275         }
 276 
 277         return PiNode.piCastToSnippetReplaceeStamp(allocateInstanceDynamicHelper(type, fillContents, threadRegister, counters, nonNullType));
 278     }
 279 
 280     private static Object allocateInstanceDynamicHelper(Class<?> type, boolean fillContents, Register threadRegister, Counters counters, Class<?> nonNullType) {
 281         KlassPointer hub = ClassGetHubNode.readClass(nonNullType);
 282         if (probability(FAST_PATH_PROBABILITY, !hub.isNull())) {
 283             KlassPointer nonNullHub = ClassGetHubNode.piCastNonNull(hub, SnippetAnchorNode.anchor());
 284 
 285             if (probability(FAST_PATH_PROBABILITY, isInstanceKlassFullyInitialized(nonNullHub))) {
 286                 int layoutHelper = readLayoutHelper(nonNullHub);
 287                 /*
 288                  * src/share/vm/oops/klass.hpp: For instances, layout helper is a positive number,
 289                  * the instance size. This size is already passed through align_object_size and
 290                  * scaled to bytes. The low order bit is set if instances of this class cannot be
 291                  * allocated using the fastpath.
 292                  */
 293                 if (probability(FAST_PATH_PROBABILITY, (layoutHelper & 1) == 0)) {
 294                     Word prototypeMarkWord = nonNullHub.readWord(prototypeMarkWordOffset(INJECTED_VMCONFIG), PROTOTYPE_MARK_WORD_LOCATION);
 295                     /*
 296                      * FIXME(je,ds): we should actually pass typeContext instead of "" but late
 297                      * binding of parameters is not yet supported by the GraphBuilderPlugin system.
 298                      */
 299                     return allocateInstanceHelper(layoutHelper, nonNullHub, prototypeMarkWord, fillContents, threadRegister, false, "", counters);
 300                 }
 301             } else {
 302                 DeoptimizeNode.deopt(None, RuntimeConstraint);
 303             }
 304         }
 305         return dynamicNewInstanceStub(type);
 306     }
 307 
 308     /**
 309      * Maximum array length for which fast path allocation is used.
 310      */
 311     public static final int MAX_ARRAY_FAST_PATH_ALLOCATION_LENGTH = 0x00FFFFFF;
 312 
 313     @Snippet
 314     public static Object allocatePrimitiveArrayPIC(KlassPointer hub, int length, Word prototypeMarkWord, @ConstantParameter int headerSize, @ConstantParameter int log2ElementSize,
 315                     @ConstantParameter boolean fillContents, @ConstantParameter Register threadRegister, @ConstantParameter boolean maybeUnroll, @ConstantParameter String typeContext,
 316                     @ConstantParameter Counters counters) {
 317         // Primitive array types are eagerly pre-resolved. We can use a floating load.
 318         KlassPointer picHub = LoadConstantIndirectlyNode.loadKlass(hub);
 319         return allocateArrayImpl(picHub, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents, threadRegister, maybeUnroll, typeContext, counters);
 320     }
 321 
 322     @Snippet
 323     public static Object allocateArrayPIC(KlassPointer hub, int length, Word prototypeMarkWord, @ConstantParameter int headerSize, @ConstantParameter int log2ElementSize,
 324                     @ConstantParameter boolean fillContents, @ConstantParameter Register threadRegister, @ConstantParameter boolean maybeUnroll, @ConstantParameter String typeContext,
 325                     @ConstantParameter Counters counters) {
 326         // Array type would be resolved by dominating resolution.
 327         KlassPointer picHub = LoadConstantIndirectlyFixedNode.loadKlass(hub);
 328         return allocateArrayImpl(picHub, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents, threadRegister, maybeUnroll, typeContext, counters);
 329     }
 330 
 331     @Snippet
 332     public static Object allocateArray(KlassPointer hub,
 333                     int length,
 334                     Word prototypeMarkWord,
 335                     @ConstantParameter int headerSize,
 336                     @ConstantParameter int log2ElementSize,
 337                     @ConstantParameter boolean fillContents,
 338                     @ConstantParameter Register threadRegister,
 339                     @ConstantParameter boolean maybeUnroll,
 340                     @ConstantParameter String typeContext,
 341                     @ConstantParameter Counters counters) {
 342         Object result = allocateArrayImpl(hub,
 343                         length,
 344                         prototypeMarkWord,
 345                         headerSize,
 346                         log2ElementSize,
 347                         fillContents,
 348                         threadRegister,
 349                         maybeUnroll,
 350                         typeContext,

 351 
 352                         counters);
 353         return piArrayCastToSnippetReplaceeStamp(verifyOop(result), length);
 354     }
 355 
 356     /**
 357      * When allocating on the slow path, determines whether to use a version of the runtime call
 358      * that returns {@code null} on a failed allocation instead of raising an OutOfMemoryError.
 359      */
 360     @Fold
 361     static boolean useNullAllocationStubs(@InjectedParameter GraalHotSpotVMConfig config) {
 362         return config.areNullAllocationStubsAvailable();
 363     }
 364 
 365     private static Object allocateArrayImpl(KlassPointer hub, int length, Word prototypeMarkWord, int headerSize, int log2ElementSize, boolean fillContents, Register threadRegister,
 366                     boolean maybeUnroll, String typeContext, Counters counters) {
 367         Object result;
 368         long allocationSize = arrayAllocationSize(length, headerSize, log2ElementSize);
 369         Word thread = registerAsWord(threadRegister);
 370         Word top = readTlabTop(thread);
 371         Word end = readTlabEnd(thread);
 372         Word newTop = top.add(WordFactory.unsigned(allocationSize));
 373         if (probability(FREQUENT_PROBABILITY, belowThan(length, MAX_ARRAY_FAST_PATH_ALLOCATION_LENGTH)) && useTLAB(INJECTED_VMCONFIG) &&
 374                         probability(FAST_PATH_PROBABILITY, newTop.belowOrEqual(end))) {
 375             writeTlabTop(thread, newTop);
 376             emitPrefetchAllocate(newTop, true);
 377             Counters theCounters = counters;
 378             if (theCounters != null && theCounters.arrayLoopInit != null) {
 379                 theCounters.arrayLoopInit.inc();
 380             }
 381             result = formatArray(hub, allocationSize, length, headerSize, top, prototypeMarkWord, fillContents, maybeUnroll, counters);
 382         } else {
 383             result = newArrayStub(hub, length);
 384         }
 385         profileAllocation("array", allocationSize, typeContext);
 386         return result;
 387     }
 388 
 389     public static Object newArrayStub(KlassPointer hub, int length) {
 390         if (useNullAllocationStubs(INJECTED_VMCONFIG)) {
 391             return nonNullOrDeopt(newArrayOrNull(NEW_ARRAY_OR_NULL, hub, length));
 392         } else {
 393             return newArray(NEW_ARRAY, hub, length);
 394         }
 395     }
 396 
 397     @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = true)
 398     private static native Object newArray(@ConstantNodeParameter ForeignCallDescriptor descriptor, KlassPointer hub, int length);
 399 
 400     @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = false)
 401     private static native Object newArrayOrNull(@ConstantNodeParameter ForeignCallDescriptor descriptor, KlassPointer hub, int length);
 402 
 403     /**
 404      * New dynamic array stub that throws an {@link OutOfMemoryError} on allocation failure.
 405      */
 406     public static final ForeignCallDescriptor DYNAMIC_NEW_INSTANCE = new ForeignCallDescriptor("dynamic_new_instance", Object.class, Class.class);
 407 
 408     /**
 409      * New dynamic array stub that returns null on allocation failure.
 410      */
 411     public static final ForeignCallDescriptor DYNAMIC_NEW_INSTANCE_OR_NULL = new ForeignCallDescriptor("dynamic_new_instance_or_null", Object.class, Class.class);
 412 
 413     public static Object dynamicNewInstanceStub(Class<?> elementType) {
 414         if (useNullAllocationStubs(INJECTED_VMCONFIG)) {
 415             return nonNullOrDeopt(dynamicNewInstanceOrNull(DYNAMIC_NEW_INSTANCE_OR_NULL, elementType));
 416         } else {
 417             return dynamicNewInstance(DYNAMIC_NEW_INSTANCE, elementType);
 418         }
 419     }
 420 
 421     /**
 422      * Deoptimizes if {@code obj == null} otherwise returns {@code obj}.
 423      */
 424     private static Object nonNullOrDeopt(Object obj) {
 425         if (obj == null) {
 426             DeoptimizeNode.deopt(None, RuntimeConstraint);
 427         }
 428         return obj;
 429     }
 430 
 431     @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = true)
 432     public static native Object dynamicNewInstance(@ConstantNodeParameter ForeignCallDescriptor descriptor, Class<?> elementType);
 433 
 434     @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = false)
 435     public static native Object dynamicNewInstanceOrNull(@ConstantNodeParameter ForeignCallDescriptor descriptor, Class<?> elementType);
 436 
 437     @Snippet
 438     public static Object allocateArrayDynamic(Class<?> elementType, Class<?> voidClass, int length, @ConstantParameter boolean fillContents, @ConstantParameter Register threadRegister,
 439                     @ConstantParameter JavaKind knownElementKind, @ConstantParameter int knownLayoutHelper, Word prototypeMarkWord,
 440                     @ConstantParameter Counters counters) {
 441         Object result = allocateArrayDynamicImpl(elementType, voidClass, length, fillContents, threadRegister, knownElementKind, knownLayoutHelper, prototypeMarkWord, counters);
 442         return result;
 443     }
 444 
 445     private static Object allocateArrayDynamicImpl(Class<?> elementType, Class<?> voidClass, int length, boolean fillContents, Register threadRegister, JavaKind knownElementKind,
 446                     int knownLayoutHelper, Word prototypeMarkWord, Counters counters) {
 447         /*
 448          * We only need the dynamic check for void when we have no static information from
 449          * knownElementKind.
 450          */
 451         staticAssert(knownElementKind != JavaKind.Void, "unsupported knownElementKind");
 452         if (knownElementKind == JavaKind.Illegal && probability(SLOW_PATH_PROBABILITY, elementType == null || DynamicNewArrayNode.throwsIllegalArgumentException(elementType, voidClass))) {
 453             DeoptimizeNode.deopt(None, RuntimeConstraint);
 454         }
 455 
 456         KlassPointer klass = loadKlassFromObject(elementType, arrayKlassOffset(INJECTED_VMCONFIG), CLASS_ARRAY_KLASS_LOCATION);
 457         if (klass.isNull()) {
 458             DeoptimizeNode.deopt(None, RuntimeConstraint);
 459         }
 460         KlassPointer nonNullKlass = ClassGetHubNode.piCastNonNull(klass, SnippetAnchorNode.anchor());
 461 
 462         if (length < 0) {
 463             DeoptimizeNode.deopt(None, RuntimeConstraint);
 464         }
 465         int layoutHelper;
 466         if (knownElementKind == JavaKind.Illegal) {
 467             layoutHelper = readLayoutHelper(nonNullKlass);
 468         } else {
 469             runtimeAssert(knownLayoutHelper == readLayoutHelper(nonNullKlass), "layout mismatch");
 470             layoutHelper = knownLayoutHelper;
 471         }
 472         //@formatter:off
 473         // from src/share/vm/oops/klass.hpp:
 474         //
 475         // For arrays, layout helper is a negative number, containing four
 476         // distinct bytes, as follows:
 477         //    MSB:[tag, hsz, ebt, log2(esz)]:LSB
 478         // where:
 479         //    tag is 0x80 if the elements are oops, 0xC0 if non-oops
 480         //    hsz is array header size in bytes (i.e., offset of first element)
 481         //    ebt is the BasicType of the elements
 482         //    esz is the element size in bytes
 483         //@formatter:on
 484 
 485         int headerSize = (layoutHelper >> layoutHelperHeaderSizeShift(INJECTED_VMCONFIG)) & layoutHelperHeaderSizeMask(INJECTED_VMCONFIG);
 486         int log2ElementSize = (layoutHelper >> layoutHelperLog2ElementSizeShift(INJECTED_VMCONFIG)) & layoutHelperLog2ElementSizeMask(INJECTED_VMCONFIG);
 487 
 488         Object result = allocateArrayImpl(nonNullKlass, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents, threadRegister, false, "dynamic type", counters);
 489         return piArrayCastToSnippetReplaceeStamp(verifyOop(result), length);
 490     }
 491 
 492     /**
 493      * Calls the runtime stub for implementing MULTIANEWARRAY.
 494      */
 495     @Snippet
 496     private static Object newmultiarray(KlassPointer hub, @ConstantParameter int rank, @VarargsParameter int[] dimensions) {
 497         Word dims = DimensionsNode.allocaDimsArray(rank);
 498         ExplodeLoopNode.explodeLoop();
 499         for (int i = 0; i < rank; i++) {
 500             dims.writeInt(i * 4, dimensions[i], LocationIdentity.init());
 501         }
 502         return newMultiArrayStub(hub, rank, dims);
 503     }
 504 
 505     private static Object newMultiArrayStub(KlassPointer hub, int rank, Word dims) {
 506         if (useNullAllocationStubs(INJECTED_VMCONFIG)) {
 507             return nonNullOrDeopt(newMultiArrayOrNull(NEW_MULTI_ARRAY_OR_NULL, hub, rank, dims));
 508         } else {
 509             return newMultiArray(NEW_MULTI_ARRAY, hub, rank, dims);
 510         }
 511     }
 512 
 513     @Snippet
 514     private static Object newmultiarrayPIC(KlassPointer hub, @ConstantParameter int rank, @VarargsParameter int[] dimensions) {
 515         // Array type would be resolved by dominating resolution.
 516         KlassPointer picHub = LoadConstantIndirectlyFixedNode.loadKlass(hub);
 517         return newmultiarray(picHub, rank, dimensions);
 518     }
 519 
 520     @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = true)
 521     private static native Object newMultiArray(@ConstantNodeParameter ForeignCallDescriptor descriptor, KlassPointer hub, int rank, Word dims);
 522 
 523     @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = false)
 524     private static native Object newMultiArrayOrNull(@ConstantNodeParameter ForeignCallDescriptor descriptor, KlassPointer hub, int rank, Word dims);
 525 
 526     /**
 527      * Maximum number of long stores to emit when zeroing an object with a constant size. Larger
 528      * objects have their bodies initialized in a loop.
 529      */
 530     private static final int MAX_UNROLLED_OBJECT_ZEROING_STORES = 8;
 531 
 532     /**
 533      * Zero uninitialized memory in a newly allocated object, unrolling as necessary and ensuring
 534      * that stores are aligned.
 535      *
 536      * @param size number of bytes to zero
 537      * @param memory beginning of object which is being zeroed
 538      * @param constantSize is {@code size} known to be constant in the snippet
 539      * @param startOffset offset to begin zeroing. May not be word aligned.
 540      * @param manualUnroll maximally unroll zeroing
 541      */
 542     private static void zeroMemory(long size, Word memory, boolean constantSize, int startOffset, boolean manualUnroll, Counters counters) {
 543         fillMemory(0, size, memory, constantSize, startOffset, manualUnroll, counters);
 544     }
 545 
 546     private static void fillMemory(long value, long size, Word memory, boolean constantSize, int startOffset, boolean manualUnroll, Counters counters) {
 547         ReplacementsUtil.runtimeAssert((size & 0x7) == 0, "unaligned object size");
 548         int offset = startOffset;
 549         if ((offset & 0x7) != 0) {
 550             memory.writeInt(offset, (int) value, LocationIdentity.init());
 551             offset += 4;
 552         }
 553         ReplacementsUtil.runtimeAssert((offset & 0x7) == 0, "unaligned offset");
 554         Counters theCounters = counters;
 555         if (manualUnroll && ((size - offset) / 8) <= MAX_UNROLLED_OBJECT_ZEROING_STORES) {
 556             ReplacementsUtil.staticAssert(!constantSize, "size shouldn't be constant at instantiation time");
 557             // This case handles arrays of constant length. Instead of having a snippet variant for
 558             // each length, generate a chain of stores of maximum length. Once it's inlined the
 559             // break statement will trim excess stores.
 560             if (theCounters != null && theCounters.instanceSeqInit != null) {
 561                 theCounters.instanceSeqInit.inc();
 562             }
 563 
 564             explodeLoop();
 565             for (int i = 0; i < MAX_UNROLLED_OBJECT_ZEROING_STORES; i++, offset += 8) {
 566                 if (offset == size) {
 567                     break;
 568                 }
 569                 memory.initializeLong(offset, value, LocationIdentity.init());
 570             }
 571         } else {
 572             // Use Word instead of int to avoid extension to long in generated code
 573             Word off = WordFactory.signed(offset);
 574             if (constantSize && ((size - offset) / 8) <= MAX_UNROLLED_OBJECT_ZEROING_STORES) {
 575                 if (theCounters != null && theCounters.instanceSeqInit != null) {
 576                     theCounters.instanceSeqInit.inc();
 577                 }
 578                 explodeLoop();
 579             } else {
 580                 if (theCounters != null && theCounters.instanceLoopInit != null) {
 581                     theCounters.instanceLoopInit.inc();
 582                 }
 583             }
 584             for (; off.rawValue() < size; off = off.add(8)) {
 585                 memory.initializeLong(off, value, LocationIdentity.init());
 586             }
 587         }
 588     }
 589 
 590     /**
 591      * Fill uninitialized memory with garbage value in a newly allocated object, unrolling as
 592      * necessary and ensuring that stores are aligned.
 593      *
 594      * @param size number of bytes to zero
 595      * @param memory beginning of object which is being zeroed
 596      * @param constantSize is {@code  size} known to be constant in the snippet
 597      * @param startOffset offset to begin zeroing. May not be word aligned.
 598      * @param manualUnroll maximally unroll zeroing
 599      */
 600     private static void fillWithGarbage(long size, Word memory, boolean constantSize, int startOffset, boolean manualUnroll, Counters counters) {
 601         fillMemory(0xfefefefefefefefeL, size, memory, constantSize, startOffset, manualUnroll, counters);
 602     }
 603 
 604     /**
 605      * Formats some allocated memory with an object header and zeroes out the rest.
 606      */
 607     private static Object formatObject(KlassPointer hub, long size, Word memory, Word compileTimePrototypeMarkWord, boolean fillContents, boolean constantSize, Counters counters) {
 608         Word prototypeMarkWord = useBiasedLocking(INJECTED_VMCONFIG) ? hub.readWord(prototypeMarkWordOffset(INJECTED_VMCONFIG), PROTOTYPE_MARK_WORD_LOCATION) : compileTimePrototypeMarkWord;
 609         initializeObjectHeader(memory, prototypeMarkWord, hub);
 610         if (fillContents) {
 611             zeroMemory(size, memory, constantSize, instanceHeaderSize(INJECTED_VMCONFIG), false, counters);
 612         } else if (REPLACEMENTS_ASSERTIONS_ENABLED) {
 613             fillWithGarbage(size, memory, constantSize, instanceHeaderSize(INJECTED_VMCONFIG), false, counters);
 614         }
 615         MembarNode.memoryBarrier(MemoryBarriers.STORE_STORE, LocationIdentity.init());
 616         return memory.toObjectNonNull();
 617     }
 618 
 619     @Snippet
 620     private static void verifyHeap(@ConstantParameter Register threadRegister) {
 621         Word thread = registerAsWord(threadRegister);
 622         Word topValue = readTlabTop(thread);
 623         if (!topValue.equal(WordFactory.zero())) {
 624             Word topValueContents = topValue.readWord(0, MARK_WORD_LOCATION);
 625             if (topValueContents.equal(WordFactory.zero())) {
 626                 AssertionSnippets.vmMessageC(AssertionSnippets.ASSERTION_VM_MESSAGE_C, true, cstring("overzeroing of TLAB detected"), 0L, 0L, 0L);
 627             }
 628         }
 629     }
 630 
 631     /**
 632      * Formats some allocated memory with an object header and zeroes out the rest.
 633      */
 634     private static Object formatArray(KlassPointer hub, long allocationSize, int length, int headerSize, Word memory, Word prototypeMarkWord, boolean fillContents, boolean maybeUnroll,
 635                     Counters counters) {
 636         memory.writeInt(arrayLengthOffset(INJECTED_VMCONFIG), length, LocationIdentity.init());
 637         /*
 638          * store hub last as the concurrent garbage collectors assume length is valid if hub field
 639          * is not null
 640          */
 641         initializeObjectHeader(memory, prototypeMarkWord, hub);
 642         if (fillContents) {
 643             zeroMemory(allocationSize, memory, false, headerSize, maybeUnroll, counters);
 644         } else if (REPLACEMENTS_ASSERTIONS_ENABLED) {
 645             fillWithGarbage(allocationSize, memory, false, headerSize, maybeUnroll, counters);
 646         }
 647         MembarNode.memoryBarrier(MemoryBarriers.STORE_STORE, LocationIdentity.init());
 648         return memory.toObjectNonNull();
 649     }
 650 
 651     static class Counters {
 652         Counters(SnippetCounter.Group.Factory factory) {
 653             Group newInstance = factory.createSnippetCounterGroup("NewInstance");
 654             Group newArray = factory.createSnippetCounterGroup("NewArray");
 655             instanceSeqInit = new SnippetCounter(newInstance, "tlabSeqInit", "TLAB alloc with unrolled zeroing");
 656             instanceLoopInit = new SnippetCounter(newInstance, "tlabLoopInit", "TLAB alloc with zeroing in a loop");
 657             arrayLoopInit = new SnippetCounter(newArray, "tlabLoopInit", "TLAB alloc with zeroing in a loop");
 658             stub = new SnippetCounter(newInstance, "stub", "alloc and zeroing via stub");
 659         }
 660 
 661         final SnippetCounter instanceSeqInit;
 662         final SnippetCounter instanceLoopInit;
 663         final SnippetCounter arrayLoopInit;
 664         final SnippetCounter stub;
 665     }
 666 
 667     public static class Templates extends AbstractTemplates {
 668 
 669         private final SnippetInfo allocateInstance = snippet(NewObjectSnippets.class, "allocateInstance", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION, TLAB_END_LOCATION);
 670         private final SnippetInfo allocateInstancePIC = snippet(NewObjectSnippets.class, "allocateInstancePIC", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION,
 671                         TLAB_END_LOCATION);
 672         private final SnippetInfo allocateArray = snippet(NewObjectSnippets.class, "allocateArray", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION, TLAB_END_LOCATION);
 673         private final SnippetInfo allocateArrayPIC = snippet(NewObjectSnippets.class, "allocateArrayPIC", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION, TLAB_END_LOCATION);
 674         private final SnippetInfo allocatePrimitiveArrayPIC = snippet(NewObjectSnippets.class, "allocatePrimitiveArrayPIC", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION,
 675                         TLAB_END_LOCATION);
 676         private final SnippetInfo allocateArrayDynamic = snippet(NewObjectSnippets.class, "allocateArrayDynamic", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION,
 677                         TLAB_END_LOCATION);
 678         private final SnippetInfo allocateInstanceDynamic = snippet(NewObjectSnippets.class, "allocateInstanceDynamic", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION,
 679                         TLAB_END_LOCATION);
 680         private final SnippetInfo newmultiarray = snippet(NewObjectSnippets.class, "newmultiarray", TLAB_TOP_LOCATION, TLAB_END_LOCATION);
 681         private final SnippetInfo newmultiarrayPIC = snippet(NewObjectSnippets.class, "newmultiarrayPIC", TLAB_TOP_LOCATION, TLAB_END_LOCATION);
 682         private final SnippetInfo verifyHeap = snippet(NewObjectSnippets.class, "verifyHeap");
 683         private final GraalHotSpotVMConfig config;
 684         private final Counters counters;
 685 
 686         public Templates(OptionValues options, Iterable<DebugHandlersFactory> factories, SnippetCounter.Group.Factory factory, HotSpotProviders providers, TargetDescription target,
 687                         GraalHotSpotVMConfig config) {
 688             super(options, factories, providers, providers.getSnippetReflection(), target);
 689             this.config = config;
 690             counters = new Counters(factory);
 691         }
 692 
 693         /**
 694          * Lowers a {@link NewInstanceNode}.
 695          */
 696         public void lower(NewInstanceNode newInstanceNode, HotSpotRegistersProvider registers, LoweringTool tool) {
 697             StructuredGraph graph = newInstanceNode.graph();
 698             HotSpotResolvedObjectType type = (HotSpotResolvedObjectType) newInstanceNode.instanceClass();
 699             assert !type.isArray();
 700             ConstantNode hub = ConstantNode.forConstant(KlassPointerStamp.klassNonNull(), type.klass(), providers.getMetaAccess(), graph);
 701             long size = instanceSize(type);
 702 
 703             OptionValues localOptions = graph.getOptions();
 704             SnippetInfo snippet = GeneratePIC.getValue(localOptions) ? allocateInstancePIC : allocateInstance;
 705             Arguments args = new Arguments(snippet, graph.getGuardsStage(), tool.getLoweringStage());
 706             args.addConst("size", size);
 707             args.add("hub", hub);
 708             args.add("prototypeMarkWord", type.prototypeMarkWord());
 709             args.addConst("fillContents", newInstanceNode.fillContents());
 710             args.addConst("threadRegister", registers.getThreadRegister());
 711             args.addConst("constantSize", true);
 712             args.addConst("typeContext", ProfileAllocations.getValue(localOptions) ? type.toJavaName(false) : "");
 713             args.addConst("counters", counters);
 714 
 715             SnippetTemplate template = template(newInstanceNode, args);
 716             graph.getDebug().log("Lowering allocateInstance in %s: node=%s, template=%s, arguments=%s", graph, newInstanceNode, template, args);
 717             template.instantiate(providers.getMetaAccess(), newInstanceNode, DEFAULT_REPLACER, args);
 718         }
 719 
 720         /**
 721          * Lowers a {@link NewArrayNode}.
 722          */
 723         public void lower(NewArrayNode newArrayNode, HotSpotRegistersProvider registers, LoweringTool tool) {
 724             StructuredGraph graph = newArrayNode.graph();
 725             ResolvedJavaType elementType = newArrayNode.elementType();
 726             HotSpotResolvedObjectType arrayType = (HotSpotResolvedObjectType) elementType.getArrayClass();
 727             JavaKind elementKind = elementType.getJavaKind();
 728             ConstantNode hub = ConstantNode.forConstant(KlassPointerStamp.klassNonNull(), arrayType.klass(), providers.getMetaAccess(), graph);
 729             final int headerSize = tool.getMetaAccess().getArrayBaseOffset(elementKind);
 730             int log2ElementSize = CodeUtil.log2(tool.getMetaAccess().getArrayIndexScale(elementKind));
 731 
 732             OptionValues localOptions = graph.getOptions();
 733             SnippetInfo snippet;
 734             if (GeneratePIC.getValue(localOptions)) {
 735                 if (elementType.isPrimitive()) {
 736                     snippet = allocatePrimitiveArrayPIC;
 737                 } else {
 738                     snippet = allocateArrayPIC;
 739                 }
 740             } else {
 741                 snippet = allocateArray;
 742             }
 743 
 744             Arguments args = new Arguments(snippet, graph.getGuardsStage(), tool.getLoweringStage());
 745             args.add("hub", hub);
 746             ValueNode length = newArrayNode.length();
 747             args.add("length", length.isAlive() ? length : graph.addOrUniqueWithInputs(length));
 748             assert arrayType.prototypeMarkWord() == lookupArrayClass(tool, JavaKind.Object).prototypeMarkWord() : "all array types are assumed to have the same prototypeMarkWord";
 749             args.add("prototypeMarkWord", arrayType.prototypeMarkWord());
 750             args.addConst("headerSize", headerSize);
 751             args.addConst("log2ElementSize", log2ElementSize);
 752             args.addConst("fillContents", newArrayNode.fillContents());
 753             args.addConst("threadRegister", registers.getThreadRegister());
 754             args.addConst("maybeUnroll", length.isConstant());
 755             args.addConst("typeContext", ProfileAllocations.getValue(localOptions) ? arrayType.toJavaName(false) : "");
 756             args.addConst("counters", counters);
 757             SnippetTemplate template = template(newArrayNode, args);
 758             graph.getDebug().log("Lowering allocateArray in %s: node=%s, template=%s, arguments=%s", graph, newArrayNode, template, args);
 759             template.instantiate(providers.getMetaAccess(), newArrayNode, DEFAULT_REPLACER, args);
 760         }
 761 
 762         public void lower(DynamicNewInstanceNode newInstanceNode, HotSpotRegistersProvider registers, LoweringTool tool) {
 763             Arguments args = new Arguments(allocateInstanceDynamic, newInstanceNode.graph().getGuardsStage(), tool.getLoweringStage());
 764             args.add("type", newInstanceNode.getInstanceType());
 765             ValueNode classClass = newInstanceNode.getClassClass();
 766             assert classClass != null;
 767             args.add("classClass", classClass);
 768             args.addConst("fillContents", newInstanceNode.fillContents());
 769             args.addConst("threadRegister", registers.getThreadRegister());
 770             args.addConst("counters", counters);
 771 
 772             SnippetTemplate template = template(newInstanceNode, args);
 773             template.instantiate(providers.getMetaAccess(), newInstanceNode, DEFAULT_REPLACER, args);
 774         }
 775 
 776         public void lower(DynamicNewArrayNode newArrayNode, HotSpotRegistersProvider registers, LoweringTool tool) {
 777             StructuredGraph graph = newArrayNode.graph();
 778             Arguments args = new Arguments(allocateArrayDynamic, newArrayNode.graph().getGuardsStage(), tool.getLoweringStage());
 779             args.add("elementType", newArrayNode.getElementType());
 780             ValueNode voidClass = newArrayNode.getVoidClass();
 781             assert voidClass != null;
 782             args.add("voidClass", voidClass);
 783             ValueNode length = newArrayNode.length();
 784             args.add("length", length.isAlive() ? length : graph.addOrUniqueWithInputs(length));
 785             args.addConst("fillContents", newArrayNode.fillContents());
 786             args.addConst("threadRegister", registers.getThreadRegister());
 787             /*
 788              * We use Kind.Illegal as a marker value instead of null because constant snippet
 789              * parameters cannot be null.
 790              */
 791             args.addConst("knownElementKind", newArrayNode.getKnownElementKind() == null ? JavaKind.Illegal : newArrayNode.getKnownElementKind());
 792             if (newArrayNode.getKnownElementKind() != null) {
 793                 args.addConst("knownLayoutHelper", lookupArrayClass(tool, newArrayNode.getKnownElementKind()).layoutHelper());
 794             } else {
 795                 args.addConst("knownLayoutHelper", 0);
 796             }
 797             args.add("prototypeMarkWord", lookupArrayClass(tool, JavaKind.Object).prototypeMarkWord());
 798             args.addConst("counters", counters);
 799             SnippetTemplate template = template(newArrayNode, args);
 800             template.instantiate(providers.getMetaAccess(), newArrayNode, DEFAULT_REPLACER, args);
 801         }
 802 
 803         private static HotSpotResolvedObjectType lookupArrayClass(LoweringTool tool, JavaKind kind) {
 804             return (HotSpotResolvedObjectType) tool.getMetaAccess().lookupJavaType(kind == JavaKind.Object ? Object.class : kind.toJavaClass()).getArrayClass();
 805         }
 806 
 807         public void lower(NewMultiArrayNode newmultiarrayNode, LoweringTool tool) {
 808             StructuredGraph graph = newmultiarrayNode.graph();
 809             OptionValues localOptions = graph.getOptions();
 810             int rank = newmultiarrayNode.dimensionCount();
 811             ValueNode[] dims = new ValueNode[rank];
 812             for (int i = 0; i < newmultiarrayNode.dimensionCount(); i++) {
 813                 dims[i] = newmultiarrayNode.dimension(i);
 814             }
 815             HotSpotResolvedObjectType type = (HotSpotResolvedObjectType) newmultiarrayNode.type();
 816             ConstantNode hub = ConstantNode.forConstant(KlassPointerStamp.klassNonNull(), type.klass(), providers.getMetaAccess(), graph);
 817 
 818             SnippetInfo snippet = GeneratePIC.getValue(localOptions) ? newmultiarrayPIC : newmultiarray;
 819             Arguments args = new Arguments(snippet, graph.getGuardsStage(), tool.getLoweringStage());
 820             args.add("hub", hub);
 821             args.addConst("rank", rank);
 822             args.addVarargs("dimensions", int.class, StampFactory.forKind(JavaKind.Int), dims);
 823             template(newmultiarrayNode, args).instantiate(providers.getMetaAccess(), newmultiarrayNode, DEFAULT_REPLACER, args);
 824         }
 825 
 826         private static long instanceSize(HotSpotResolvedObjectType type) {
 827             long size = type.instanceSize();
 828             assert size >= 0;
 829             return size;
 830         }
 831 
 832         public void lower(VerifyHeapNode verifyHeapNode, HotSpotRegistersProvider registers, LoweringTool tool) {
 833             if (config.cAssertions) {
 834                 Arguments args = new Arguments(verifyHeap, verifyHeapNode.graph().getGuardsStage(), tool.getLoweringStage());
 835                 args.addConst("threadRegister", registers.getThreadRegister());
 836 
 837                 SnippetTemplate template = template(verifyHeapNode, args);
 838                 template.instantiate(providers.getMetaAccess(), verifyHeapNode, DEFAULT_REPLACER, args);
 839             } else {
 840                 GraphUtil.removeFixedWithUnusedInputs(verifyHeapNode);
 841             }
 842         }
 843     }
 844 }
--- EOF ---