1 /*
   2  * Copyright (c) 2012, 2018, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  */
  23 
  24 
  25 package org.graalvm.compiler.hotspot.replacements;
  26 
  27 import static jdk.vm.ci.meta.DeoptimizationAction.None;
  28 import static jdk.vm.ci.meta.DeoptimizationReason.RuntimeConstraint;
  29 import static org.graalvm.compiler.core.common.GraalOptions.GeneratePIC;
  30 import static org.graalvm.compiler.core.common.calc.UnsignedMath.belowThan;
  31 import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfigBase.INJECTED_OPTIONVALUES;
  32 import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfigBase.INJECTED_VMCONFIG;
  33 import static org.graalvm.compiler.hotspot.HotSpotBackend.NEW_ARRAY;
  34 import static org.graalvm.compiler.hotspot.HotSpotBackend.NEW_ARRAY_OR_NULL;
  35 import static org.graalvm.compiler.hotspot.HotSpotBackend.NEW_INSTANCE;
  36 import static org.graalvm.compiler.hotspot.HotSpotBackend.NEW_INSTANCE_OR_NULL;
  37 import static org.graalvm.compiler.hotspot.HotSpotBackend.NEW_MULTI_ARRAY;
  38 import static org.graalvm.compiler.hotspot.HotSpotBackend.NEW_MULTI_ARRAY_OR_NULL;
  39 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.CLASS_ARRAY_KLASS_LOCATION;
  40 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.HUB_WRITE_LOCATION;
  41 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.MARK_WORD_LOCATION;
  42 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.PROTOTYPE_MARK_WORD_LOCATION;
  43 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_END_LOCATION;
  44 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_TOP_LOCATION;
  45 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.allocateInstancePrefetchLines;
  46 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.allocatePrefetchDistance;
  47 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.allocatePrefetchLines;
  48 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.allocatePrefetchStepSize;
  49 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.allocatePrefetchStyle;
  50 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayAllocationSize;
  51 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayKlassOffset;
  52 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayLengthOffset;
  53 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.initializeObjectHeader;
  54 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.instanceHeaderSize;
  55 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.isInstanceKlassFullyInitialized;
  56 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperHeaderSizeMask;
  57 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperHeaderSizeShift;
  58 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperLog2ElementSizeMask;
  59 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperLog2ElementSizeShift;
  60 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.loadKlassFromObject;
  61 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.prototypeMarkWordOffset;
  62 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readLayoutHelper;
  63 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readTlabEnd;
  64 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readTlabTop;
  65 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.registerAsWord;
  66 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.useBiasedLocking;
  67 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.useTLAB;
  68 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.verifyOop;
  69 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.writeTlabTop;
  70 import static org.graalvm.compiler.hotspot.replacements.HotspotSnippetsOptions.ProfileAllocations;
  71 import static org.graalvm.compiler.hotspot.replacements.HotspotSnippetsOptions.ProfileAllocationsContext;
  72 import static org.graalvm.compiler.nodes.PiArrayNode.piArrayCastToSnippetReplaceeStamp;
  73 import static org.graalvm.compiler.nodes.PiNode.piCastToSnippetReplaceeStamp;
  74 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.FAST_PATH_PROBABILITY;
  75 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.FREQUENT_PROBABILITY;
  76 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.SLOW_PATH_PROBABILITY;
  77 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.probability;
  78 import static org.graalvm.compiler.replacements.ReplacementsUtil.REPLACEMENTS_ASSERTIONS_ENABLED;
  79 import static org.graalvm.compiler.replacements.ReplacementsUtil.runtimeAssert;
  80 import static org.graalvm.compiler.replacements.ReplacementsUtil.staticAssert;
  81 import static org.graalvm.compiler.replacements.SnippetTemplate.DEFAULT_REPLACER;
  82 import static org.graalvm.compiler.replacements.nodes.CStringConstant.cstring;
  83 import static org.graalvm.compiler.replacements.nodes.ExplodeLoopNode.explodeLoop;
  84 
  85 import org.graalvm.compiler.api.replacements.Fold;
  86 import org.graalvm.compiler.api.replacements.Fold.InjectedParameter;
  87 import org.graalvm.compiler.api.replacements.Snippet;
  88 import org.graalvm.compiler.api.replacements.Snippet.ConstantParameter;
  89 import org.graalvm.compiler.api.replacements.Snippet.VarargsParameter;
  90 import org.graalvm.compiler.core.common.spi.ForeignCallDescriptor;
  91 import org.graalvm.compiler.core.common.type.StampFactory;
  92 import org.graalvm.compiler.debug.DebugHandlersFactory;
  93 import org.graalvm.compiler.debug.GraalError;
  94 import org.graalvm.compiler.graph.Node.ConstantNodeParameter;
  95 import org.graalvm.compiler.graph.Node.NodeIntrinsic;
  96 import org.graalvm.compiler.hotspot.GraalHotSpotVMConfig;
  97 import org.graalvm.compiler.hotspot.meta.HotSpotProviders;
  98 import org.graalvm.compiler.hotspot.meta.HotSpotRegistersProvider;
  99 import org.graalvm.compiler.hotspot.nodes.DimensionsNode;
 100 import org.graalvm.compiler.hotspot.nodes.aot.LoadConstantIndirectlyFixedNode;
 101 import org.graalvm.compiler.hotspot.nodes.aot.LoadConstantIndirectlyNode;
 102 import org.graalvm.compiler.hotspot.nodes.type.KlassPointerStamp;
 103 import org.graalvm.compiler.hotspot.word.KlassPointer;
 104 import org.graalvm.compiler.nodes.ConstantNode;
 105 import org.graalvm.compiler.nodes.DeoptimizeNode;
 106 import org.graalvm.compiler.nodes.PiNode;
 107 import org.graalvm.compiler.nodes.PrefetchAllocateNode;
 108 import org.graalvm.compiler.nodes.SnippetAnchorNode;
 109 import org.graalvm.compiler.nodes.StructuredGraph;
 110 import org.graalvm.compiler.nodes.ValueNode;
 111 import org.graalvm.compiler.nodes.debug.DynamicCounterNode;
 112 import org.graalvm.compiler.nodes.debug.VerifyHeapNode;
 113 import org.graalvm.compiler.nodes.extended.ForeignCallNode;
 114 import org.graalvm.compiler.nodes.extended.MembarNode;
 115 import org.graalvm.compiler.nodes.java.DynamicNewArrayNode;
 116 import org.graalvm.compiler.nodes.java.DynamicNewInstanceNode;
 117 import org.graalvm.compiler.nodes.java.NewArrayNode;
 118 import org.graalvm.compiler.nodes.java.NewInstanceNode;
 119 import org.graalvm.compiler.nodes.java.NewMultiArrayNode;
 120 import org.graalvm.compiler.nodes.memory.address.OffsetAddressNode;
 121 import org.graalvm.compiler.nodes.spi.LoweringTool;
 122 import org.graalvm.compiler.nodes.util.GraphUtil;
 123 import org.graalvm.compiler.options.OptionValues;
 124 import org.graalvm.compiler.replacements.ReplacementsUtil;
 125 import org.graalvm.compiler.replacements.SnippetCounter;
 126 import org.graalvm.compiler.replacements.SnippetCounter.Group;
 127 import org.graalvm.compiler.replacements.SnippetTemplate;
 128 import org.graalvm.compiler.replacements.SnippetTemplate.AbstractTemplates;
 129 import org.graalvm.compiler.replacements.SnippetTemplate.Arguments;
 130 import org.graalvm.compiler.replacements.SnippetTemplate.SnippetInfo;
 131 import org.graalvm.compiler.replacements.Snippets;
 132 import org.graalvm.compiler.replacements.nodes.ExplodeLoopNode;
 133 import org.graalvm.compiler.word.Word;
 134 import jdk.internal.vm.compiler.word.LocationIdentity;
 135 import jdk.internal.vm.compiler.word.WordFactory;
 136 
 137 import jdk.vm.ci.code.CodeUtil;
 138 import jdk.vm.ci.code.MemoryBarriers;
 139 import jdk.vm.ci.code.Register;
 140 import jdk.vm.ci.code.TargetDescription;
 141 import jdk.vm.ci.hotspot.HotSpotResolvedObjectType;
 142 import jdk.vm.ci.meta.JavaKind;
 143 import jdk.vm.ci.meta.ResolvedJavaType;
 144 
 145 /**
 146  * Snippets used for implementing NEW, ANEWARRAY and NEWARRAY.
 147  */
 148 public class NewObjectSnippets implements Snippets {
 149 
 150     enum ProfileContext {
 151         AllocatingMethod,
 152         InstanceOrArray,
 153         AllocatedType,
 154         AllocatedTypesInMethod,
 155         Total
 156     }
 157 
 158     @Fold
 159     static String createName(@Fold.InjectedParameter OptionValues options, String path, String typeContext) {
 160         switch (ProfileAllocationsContext.getValue(options)) {
 161             case AllocatingMethod:
 162                 return "";
 163             case InstanceOrArray:
 164                 return path;
 165             case AllocatedType:
 166             case AllocatedTypesInMethod:
 167                 return typeContext;
 168             case Total:
 169                 return "bytes";
 170             default:
 171                 throw GraalError.shouldNotReachHere();
 172         }
 173     }
 174 
 175     @Fold
 176     static boolean doProfile(@Fold.InjectedParameter OptionValues options) {
 177         return ProfileAllocations.getValue(options);
 178     }
 179 
 180     @Fold
 181     static boolean withContext(@Fold.InjectedParameter OptionValues options) {
 182         ProfileContext context = ProfileAllocationsContext.getValue(options);
 183         return context == ProfileContext.AllocatingMethod || context == ProfileContext.AllocatedTypesInMethod;
 184     }
 185 
 186     protected static void profileAllocation(String path, long size, String typeContext) {
 187         if (doProfile(INJECTED_OPTIONVALUES)) {
 188             String name = createName(INJECTED_OPTIONVALUES, path, typeContext);
 189 
 190             boolean context = withContext(INJECTED_OPTIONVALUES);
 191             DynamicCounterNode.counter("number of bytes allocated", name, size, context);
 192             DynamicCounterNode.counter("number of allocations", name, 1, context);
 193         }
 194     }
 195 
 196     public static void emitPrefetchAllocate(Word address, boolean isArray) {
 197         if (allocatePrefetchStyle(INJECTED_VMCONFIG) > 0) {
 198             // Insert a prefetch for each allocation only on the fast-path
 199             // Generate several prefetch instructions.
 200             int lines = isArray ? allocatePrefetchLines(INJECTED_VMCONFIG) : allocateInstancePrefetchLines(INJECTED_VMCONFIG);
 201             int stepSize = allocatePrefetchStepSize(INJECTED_VMCONFIG);
 202             int distance = allocatePrefetchDistance(INJECTED_VMCONFIG);
 203             ExplodeLoopNode.explodeLoop();
 204             for (int i = 0; i < lines; i++) {
 205                 PrefetchAllocateNode.prefetch(OffsetAddressNode.address(address, distance));
 206                 distance += stepSize;
 207             }
 208         }
 209     }
 210 
 211     @Snippet
 212     public static Object allocateInstance(@ConstantParameter int size, KlassPointer hub, Word prototypeMarkWord, @ConstantParameter boolean fillContents,
 213                     @ConstantParameter Register threadRegister, @ConstantParameter boolean constantSize, @ConstantParameter String typeContext,
 214                     @ConstantParameter Counters counters) {
 215         return piCastToSnippetReplaceeStamp(allocateInstanceHelper(size, hub, prototypeMarkWord, fillContents, threadRegister, constantSize, typeContext, counters));
 216     }
 217 
 218     public static Object allocateInstanceHelper(int size, KlassPointer hub, Word prototypeMarkWord, boolean fillContents,
 219                     Register threadRegister, boolean constantSize, String typeContext, Counters counters) {
 220         Object result;
 221         Word thread = registerAsWord(threadRegister);
 222         Word top = readTlabTop(thread);
 223         Word end = readTlabEnd(thread);
 224         Word newTop = top.add(size);
 225         if (useTLAB(INJECTED_VMCONFIG) && probability(FAST_PATH_PROBABILITY, newTop.belowOrEqual(end))) {
 226             writeTlabTop(thread, newTop);
 227             emitPrefetchAllocate(newTop, false);
 228             result = formatObject(hub, size, top, prototypeMarkWord, fillContents, constantSize, counters);
 229         } else {
 230             Counters theCounters = counters;
 231             if (theCounters != null && theCounters.stub != null) {
 232                 theCounters.stub.inc();
 233             }
 234             result = newInstanceStub(hub);
 235         }
 236         profileAllocation("instance", size, typeContext);
 237         return verifyOop(result);
 238     }
 239 
 240     public static Object newInstanceStub(KlassPointer hub) {
 241         if (useNullAllocationStubs(INJECTED_VMCONFIG)) {
 242             return nonNullOrDeopt(newInstanceOrNull(NEW_INSTANCE_OR_NULL, hub));
 243         } else {
 244             return newInstance(NEW_INSTANCE, hub);
 245         }
 246     }
 247 
 248     @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = true)
 249     private static native Object newInstance(@ConstantNodeParameter ForeignCallDescriptor descriptor, KlassPointer hub);
 250 
 251     @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = false)
 252     private static native Object newInstanceOrNull(@ConstantNodeParameter ForeignCallDescriptor descriptor, KlassPointer hub);
 253 
 254     @Snippet
 255     public static Object allocateInstancePIC(@ConstantParameter int size, KlassPointer hub, Word prototypeMarkWord, @ConstantParameter boolean fillContents,
 256                     @ConstantParameter Register threadRegister, @ConstantParameter boolean constantSize, @ConstantParameter String typeContext,
 257                     @ConstantParameter Counters counters) {
 258         // Klass must be initialized by the time the first instance is allocated, therefore we can
 259         // just load it from the corresponding cell and avoid the resolution check. We have to use a
 260         // fixed load though, to prevent it from floating above the initialization.
 261         KlassPointer picHub = LoadConstantIndirectlyFixedNode.loadKlass(hub);
 262         return piCastToSnippetReplaceeStamp(allocateInstanceHelper(size, picHub, prototypeMarkWord, fillContents, threadRegister, constantSize, typeContext, counters));
 263     }
 264 
 265     @Snippet
 266     public static Object allocateInstanceDynamic(Class<?> type, Class<?> classClass, @ConstantParameter boolean fillContents, @ConstantParameter Register threadRegister,
 267                     @ConstantParameter Counters counters) {
 268         if (probability(SLOW_PATH_PROBABILITY, type == null)) {
 269             DeoptimizeNode.deopt(None, RuntimeConstraint);
 270         }
 271         Class<?> nonNullType = PiNode.piCastNonNullClass(type, SnippetAnchorNode.anchor());
 272 
 273         if (probability(SLOW_PATH_PROBABILITY, DynamicNewInstanceNode.throwsInstantiationException(type, classClass))) {
 274             DeoptimizeNode.deopt(None, RuntimeConstraint);
 275         }
 276 
 277         return PiNode.piCastToSnippetReplaceeStamp(allocateInstanceDynamicHelper(type, fillContents, threadRegister, counters, nonNullType));
 278     }
 279 
 280     private static Object allocateInstanceDynamicHelper(Class<?> type, boolean fillContents, Register threadRegister, Counters counters, Class<?> nonNullType) {
 281         KlassPointer hub = ClassGetHubNode.readClass(nonNullType);
 282         if (probability(FAST_PATH_PROBABILITY, !hub.isNull())) {
 283             KlassPointer nonNullHub = ClassGetHubNode.piCastNonNull(hub, SnippetAnchorNode.anchor());
 284 
 285             if (probability(FAST_PATH_PROBABILITY, isInstanceKlassFullyInitialized(nonNullHub))) {
 286                 int layoutHelper = readLayoutHelper(nonNullHub);
 287                 /*
 288                  * src/share/vm/oops/klass.hpp: For instances, layout helper is a positive number,
 289                  * the instance size. This size is already passed through align_object_size and
 290                  * scaled to bytes. The low order bit is set if instances of this class cannot be
 291                  * allocated using the fastpath.
 292                  */
 293                 if (probability(FAST_PATH_PROBABILITY, (layoutHelper & 1) == 0)) {
 294                     Word prototypeMarkWord = nonNullHub.readWord(prototypeMarkWordOffset(INJECTED_VMCONFIG), PROTOTYPE_MARK_WORD_LOCATION);
 295                     /*
 296                      * FIXME(je,ds): we should actually pass typeContext instead of "" but late
 297                      * binding of parameters is not yet supported by the GraphBuilderPlugin system.
 298                      */
 299                     return allocateInstanceHelper(layoutHelper, nonNullHub, prototypeMarkWord, fillContents, threadRegister, false, "", counters);
 300                 }
 301             } else {
 302                 DeoptimizeNode.deopt(None, RuntimeConstraint);
 303             }
 304         }
 305         return dynamicNewInstanceStub(type);
 306     }
 307 
 308     /**
 309      * Maximum array length for which fast path allocation is used.
 310      */
 311     public static final int MAX_ARRAY_FAST_PATH_ALLOCATION_LENGTH = 0x00FFFFFF;
 312 
 313     @Snippet
 314     public static Object allocatePrimitiveArrayPIC(KlassPointer hub, int length, Word prototypeMarkWord, @ConstantParameter int headerSize, @ConstantParameter int log2ElementSize,
 315                     @ConstantParameter boolean fillContents, @ConstantParameter Register threadRegister, @ConstantParameter boolean maybeUnroll, @ConstantParameter String typeContext,
 316                     @ConstantParameter Counters counters) {
 317         // Primitive array types are eagerly pre-resolved. We can use a floating load.
 318         KlassPointer picHub = LoadConstantIndirectlyNode.loadKlass(hub);
 319         return allocateArrayImpl(picHub, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents, threadRegister, maybeUnroll, typeContext, false, counters);
 320     }
 321 
 322     @Snippet
 323     public static Object allocateArrayPIC(KlassPointer hub, int length, Word prototypeMarkWord, @ConstantParameter int headerSize, @ConstantParameter int log2ElementSize,
 324                     @ConstantParameter boolean fillContents, @ConstantParameter Register threadRegister, @ConstantParameter boolean maybeUnroll, @ConstantParameter String typeContext,
 325                     @ConstantParameter Counters counters) {
 326         // Array type would be resolved by dominating resolution.
 327         KlassPointer picHub = LoadConstantIndirectlyFixedNode.loadKlass(hub);
 328         return allocateArrayImpl(picHub, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents, threadRegister, maybeUnroll, typeContext, false, counters);
 329     }
 330 
 331     @Snippet
 332     public static Object allocateArray(KlassPointer hub,
 333                     int length,
 334                     Word prototypeMarkWord,
 335                     @ConstantParameter int headerSize,
 336                     @ConstantParameter int log2ElementSize,
 337                     @ConstantParameter boolean fillContents,
 338                     @ConstantParameter Register threadRegister,
 339                     @ConstantParameter boolean maybeUnroll,
 340                     @ConstantParameter String typeContext,
 341                     @ConstantParameter Counters counters) {
 342         Object result = allocateArrayImpl(hub,
 343                         length,
 344                         prototypeMarkWord,
 345                         headerSize,
 346                         log2ElementSize,
 347                         fillContents,
 348                         threadRegister,
 349                         maybeUnroll,
 350                         typeContext,
 351                         false,
 352 
 353                         counters);
 354         return piArrayCastToSnippetReplaceeStamp(verifyOop(result), length);
 355     }
 356 
 357     /**
 358      * When allocating on the slow path, determines whether to use a version of the runtime call
 359      * that returns {@code null} on a failed allocation instead of raising an OutOfMemoryError.
 360      */
 361     @Fold
 362     static boolean useNullAllocationStubs(@InjectedParameter GraalHotSpotVMConfig config) {
 363         return config.areNullAllocationStubsAvailable();
 364     }
 365 
 366     private static Object allocateArrayImpl(KlassPointer hub, int length, Word prototypeMarkWord, int headerSize, int log2ElementSize, boolean fillContents, Register threadRegister,
 367                     boolean maybeUnroll, String typeContext, boolean skipNegativeCheck, Counters counters) {
 368         Object result;
 369         int allocationSize = arrayAllocationSize(length, headerSize, log2ElementSize);
 370         Word thread = registerAsWord(threadRegister);
 371         Word top = readTlabTop(thread);
 372         Word end = readTlabEnd(thread);
 373         Word newTop = top.add(allocationSize);
 374         if (probability(FREQUENT_PROBABILITY, skipNegativeCheck || belowThan(length, MAX_ARRAY_FAST_PATH_ALLOCATION_LENGTH)) && useTLAB(INJECTED_VMCONFIG) &&
 375                         probability(FAST_PATH_PROBABILITY, newTop.belowOrEqual(end))) {
 376             writeTlabTop(thread, newTop);
 377             emitPrefetchAllocate(newTop, true);
 378             Counters theCounters = counters;
 379             if (theCounters != null && theCounters.arrayLoopInit != null) {
 380                 theCounters.arrayLoopInit.inc();
 381             }
 382             result = formatArray(hub, allocationSize, length, headerSize, top, prototypeMarkWord, fillContents, maybeUnroll, counters);
 383         } else {
 384             result = newArrayStub(hub, length);
 385         }
 386         profileAllocation("array", allocationSize, typeContext);
 387         return result;
 388     }
 389 
 390     public static Object newArrayStub(KlassPointer hub, int length) {
 391         if (useNullAllocationStubs(INJECTED_VMCONFIG)) {
 392             return nonNullOrDeopt(newArrayOrNull(NEW_ARRAY_OR_NULL, hub, length));
 393         } else {
 394             return newArray(NEW_ARRAY, hub, length);
 395         }
 396     }
 397 
 398     @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = true)
 399     private static native Object newArray(@ConstantNodeParameter ForeignCallDescriptor descriptor, KlassPointer hub, int length);
 400 
 401     @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = false)
 402     private static native Object newArrayOrNull(@ConstantNodeParameter ForeignCallDescriptor descriptor, KlassPointer hub, int length);
 403 
 404     /**
 405      * New dynamic array stub that throws an {@link OutOfMemoryError} on allocation failure.
 406      */
 407     public static final ForeignCallDescriptor DYNAMIC_NEW_INSTANCE = new ForeignCallDescriptor("dynamic_new_instance", Object.class, Class.class);
 408 
 409     /**
 410      * New dynamic array stub that returns null on allocation failure.
 411      */
 412     public static final ForeignCallDescriptor DYNAMIC_NEW_INSTANCE_OR_NULL = new ForeignCallDescriptor("dynamic_new_instance_or_null", Object.class, Class.class);
 413 
 414     public static Object dynamicNewInstanceStub(Class<?> elementType) {
 415         if (useNullAllocationStubs(INJECTED_VMCONFIG)) {
 416             return nonNullOrDeopt(dynamicNewInstanceOrNull(DYNAMIC_NEW_INSTANCE_OR_NULL, elementType));
 417         } else {
 418             return dynamicNewInstance(DYNAMIC_NEW_INSTANCE, elementType);
 419         }
 420     }
 421 
 422     /**
 423      * Deoptimizes if {@code obj == null} otherwise returns {@code obj}.
 424      */
 425     private static Object nonNullOrDeopt(Object obj) {
 426         if (obj == null) {
 427             DeoptimizeNode.deopt(None, RuntimeConstraint);
 428         }
 429         return obj;
 430     }
 431 
 432     @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = true)
 433     public static native Object dynamicNewInstance(@ConstantNodeParameter ForeignCallDescriptor descriptor, Class<?> elementType);
 434 
 435     @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = false)
 436     public static native Object dynamicNewInstanceOrNull(@ConstantNodeParameter ForeignCallDescriptor descriptor, Class<?> elementType);
 437 
 438     @Snippet
 439     public static Object allocateArrayDynamic(Class<?> elementType, Class<?> voidClass, int length, @ConstantParameter boolean fillContents, @ConstantParameter Register threadRegister,
 440                     @ConstantParameter JavaKind knownElementKind, @ConstantParameter int knownLayoutHelper, Word prototypeMarkWord,
 441                     @ConstantParameter Counters counters) {
 442         Object result = allocateArrayDynamicImpl(elementType, voidClass, length, fillContents, threadRegister, knownElementKind, knownLayoutHelper, prototypeMarkWord, counters);
 443         return result;
 444     }
 445 
 446     private static Object allocateArrayDynamicImpl(Class<?> elementType, Class<?> voidClass, int length, boolean fillContents, Register threadRegister, JavaKind knownElementKind,
 447                     int knownLayoutHelper, Word prototypeMarkWord, Counters counters) {
 448         /*
 449          * We only need the dynamic check for void when we have no static information from
 450          * knownElementKind.
 451          */
 452         staticAssert(knownElementKind != JavaKind.Void, "unsupported knownElementKind");
 453         if (knownElementKind == JavaKind.Illegal && probability(SLOW_PATH_PROBABILITY, elementType == null || DynamicNewArrayNode.throwsIllegalArgumentException(elementType, voidClass))) {
 454             DeoptimizeNode.deopt(None, RuntimeConstraint);
 455         }
 456 
 457         KlassPointer klass = loadKlassFromObject(elementType, arrayKlassOffset(INJECTED_VMCONFIG), CLASS_ARRAY_KLASS_LOCATION);
 458         if (klass.isNull()) {
 459             DeoptimizeNode.deopt(None, RuntimeConstraint);
 460         }
 461         KlassPointer nonNullKlass = ClassGetHubNode.piCastNonNull(klass, SnippetAnchorNode.anchor());
 462 
 463         if (length < 0) {
 464             DeoptimizeNode.deopt(None, RuntimeConstraint);
 465         }
 466         int layoutHelper;
 467         if (knownElementKind == JavaKind.Illegal) {
 468             layoutHelper = readLayoutHelper(nonNullKlass);
 469         } else {
 470             runtimeAssert(knownLayoutHelper == readLayoutHelper(nonNullKlass), "layout mismatch");
 471             layoutHelper = knownLayoutHelper;
 472         }
 473         //@formatter:off
 474         // from src/share/vm/oops/klass.hpp:
 475         //
 476         // For arrays, layout helper is a negative number, containing four
 477         // distinct bytes, as follows:
 478         //    MSB:[tag, hsz, ebt, log2(esz)]:LSB
 479         // where:
 480         //    tag is 0x80 if the elements are oops, 0xC0 if non-oops
 481         //    hsz is array header size in bytes (i.e., offset of first element)
 482         //    ebt is the BasicType of the elements
 483         //    esz is the element size in bytes
 484         //@formatter:on
 485 
 486         int headerSize = (layoutHelper >> layoutHelperHeaderSizeShift(INJECTED_VMCONFIG)) & layoutHelperHeaderSizeMask(INJECTED_VMCONFIG);
 487         int log2ElementSize = (layoutHelper >> layoutHelperLog2ElementSizeShift(INJECTED_VMCONFIG)) & layoutHelperLog2ElementSizeMask(INJECTED_VMCONFIG);
 488 
 489         Object result = allocateArrayImpl(nonNullKlass, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents, threadRegister, false, "dynamic type", true, counters);
 490         return piArrayCastToSnippetReplaceeStamp(verifyOop(result), length);
 491     }
 492 
 493     /**
 494      * Calls the runtime stub for implementing MULTIANEWARRAY.
 495      */
 496     @Snippet
 497     private static Object newmultiarray(KlassPointer hub, @ConstantParameter int rank, @VarargsParameter int[] dimensions) {
 498         Word dims = DimensionsNode.allocaDimsArray(rank);
 499         ExplodeLoopNode.explodeLoop();
 500         for (int i = 0; i < rank; i++) {
 501             dims.writeInt(i * 4, dimensions[i], LocationIdentity.init());
 502         }
 503         return newMultiArrayStub(hub, rank, dims);
 504     }
 505 
 506     private static Object newMultiArrayStub(KlassPointer hub, int rank, Word dims) {
 507         if (useNullAllocationStubs(INJECTED_VMCONFIG)) {
 508             return nonNullOrDeopt(newMultiArrayOrNull(NEW_MULTI_ARRAY_OR_NULL, hub, rank, dims));
 509         } else {
 510             return newMultiArray(NEW_MULTI_ARRAY, hub, rank, dims);
 511         }
 512     }
 513 
 514     @Snippet
 515     private static Object newmultiarrayPIC(KlassPointer hub, @ConstantParameter int rank, @VarargsParameter int[] dimensions) {
 516         // Array type would be resolved by dominating resolution.
 517         KlassPointer picHub = LoadConstantIndirectlyFixedNode.loadKlass(hub);
 518         return newmultiarray(picHub, rank, dimensions);
 519     }
 520 
 521     @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = true)
 522     private static native Object newMultiArray(@ConstantNodeParameter ForeignCallDescriptor descriptor, KlassPointer hub, int rank, Word dims);
 523 
 524     @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = false)
 525     private static native Object newMultiArrayOrNull(@ConstantNodeParameter ForeignCallDescriptor descriptor, KlassPointer hub, int rank, Word dims);
 526 
 527     /**
 528      * Maximum number of long stores to emit when zeroing an object with a constant size. Larger
 529      * objects have their bodies initialized in a loop.
 530      */
 531     private static final int MAX_UNROLLED_OBJECT_ZEROING_STORES = 8;
 532 
 533     /**
 534      * Zero uninitialized memory in a newly allocated object, unrolling as necessary and ensuring
 535      * that stores are aligned.
 536      *
 537      * @param size number of bytes to zero
 538      * @param memory beginning of object which is being zeroed
 539      * @param constantSize is {@code size} known to be constant in the snippet
 540      * @param startOffset offset to begin zeroing. May not be word aligned.
 541      * @param manualUnroll maximally unroll zeroing
 542      */
 543     private static void zeroMemory(int size, Word memory, boolean constantSize, int startOffset, boolean manualUnroll, Counters counters) {
 544         fillMemory(0, size, memory, constantSize, startOffset, manualUnroll, counters);
 545     }
 546 
 547     private static void fillMemory(long value, int size, Word memory, boolean constantSize, int startOffset, boolean manualUnroll, Counters counters) {
 548         ReplacementsUtil.runtimeAssert((size & 0x7) == 0, "unaligned object size");
 549         int offset = startOffset;
 550         if ((offset & 0x7) != 0) {
 551             memory.writeInt(offset, (int) value, LocationIdentity.init());
 552             offset += 4;
 553         }
 554         ReplacementsUtil.runtimeAssert((offset & 0x7) == 0, "unaligned offset");
 555         Counters theCounters = counters;
 556         if (manualUnroll && ((size - offset) / 8) <= MAX_UNROLLED_OBJECT_ZEROING_STORES) {
 557             ReplacementsUtil.staticAssert(!constantSize, "size shouldn't be constant at instantiation time");
 558             // This case handles arrays of constant length. Instead of having a snippet variant for
 559             // each length, generate a chain of stores of maximum length. Once it's inlined the
 560             // break statement will trim excess stores.
 561             if (theCounters != null && theCounters.instanceSeqInit != null) {
 562                 theCounters.instanceSeqInit.inc();
 563             }
 564 
 565             explodeLoop();
 566             for (int i = 0; i < MAX_UNROLLED_OBJECT_ZEROING_STORES; i++, offset += 8) {
 567                 if (offset == size) {
 568                     break;
 569                 }
 570                 memory.initializeLong(offset, value, LocationIdentity.init());
 571             }
 572         } else {
 573             // Use Word instead of int to avoid extension to long in generated code
 574             Word off = WordFactory.signed(offset);
 575             if (constantSize && ((size - offset) / 8) <= MAX_UNROLLED_OBJECT_ZEROING_STORES) {
 576                 if (theCounters != null && theCounters.instanceSeqInit != null) {
 577                     theCounters.instanceSeqInit.inc();
 578                 }
 579                 explodeLoop();
 580             } else {
 581                 if (theCounters != null && theCounters.instanceLoopInit != null) {
 582                     theCounters.instanceLoopInit.inc();
 583                 }
 584             }
 585             for (; off.rawValue() < size; off = off.add(8)) {
 586                 memory.initializeLong(off, value, LocationIdentity.init());
 587             }
 588         }
 589     }
 590 
 591     /**
 592      * Fill uninitialized memory with garbage value in a newly allocated object, unrolling as
 593      * necessary and ensuring that stores are aligned.
 594      *
 595      * @param size number of bytes to zero
 596      * @param memory beginning of object which is being zeroed
 597      * @param constantSize is {@code  size} known to be constant in the snippet
 598      * @param startOffset offset to begin zeroing. May not be word aligned.
 599      * @param manualUnroll maximally unroll zeroing
 600      */
 601     private static void fillWithGarbage(int size, Word memory, boolean constantSize, int startOffset, boolean manualUnroll, Counters counters) {
 602         fillMemory(0xfefefefefefefefeL, size, memory, constantSize, startOffset, manualUnroll, counters);
 603     }
 604 
 605     /**
 606      * Formats some allocated memory with an object header and zeroes out the rest.
 607      */
 608     private static Object formatObject(KlassPointer hub, int size, Word memory, Word compileTimePrototypeMarkWord, boolean fillContents, boolean constantSize, Counters counters) {
 609         Word prototypeMarkWord = useBiasedLocking(INJECTED_VMCONFIG) ? hub.readWord(prototypeMarkWordOffset(INJECTED_VMCONFIG), PROTOTYPE_MARK_WORD_LOCATION) : compileTimePrototypeMarkWord;
 610         initializeObjectHeader(memory, prototypeMarkWord, hub);
 611         if (fillContents) {
 612             zeroMemory(size, memory, constantSize, instanceHeaderSize(INJECTED_VMCONFIG), false, counters);
 613         } else if (REPLACEMENTS_ASSERTIONS_ENABLED) {
 614             fillWithGarbage(size, memory, constantSize, instanceHeaderSize(INJECTED_VMCONFIG), false, counters);
 615         }
 616         MembarNode.memoryBarrier(MemoryBarriers.STORE_STORE, LocationIdentity.init());
 617         return memory.toObjectNonNull();
 618     }
 619 
 620     @Snippet
 621     private static void verifyHeap(@ConstantParameter Register threadRegister) {
 622         Word thread = registerAsWord(threadRegister);
 623         Word topValue = readTlabTop(thread);
 624         if (!topValue.equal(WordFactory.zero())) {
 625             Word topValueContents = topValue.readWord(0, MARK_WORD_LOCATION);
 626             if (topValueContents.equal(WordFactory.zero())) {
 627                 AssertionSnippets.vmMessageC(AssertionSnippets.ASSERTION_VM_MESSAGE_C, true, cstring("overzeroing of TLAB detected"), 0L, 0L, 0L);
 628             }
 629         }
 630     }
 631 
 632     /**
 633      * Formats some allocated memory with an object header and zeroes out the rest.
 634      */
 635     private static Object formatArray(KlassPointer hub, int allocationSize, int length, int headerSize, Word memory, Word prototypeMarkWord, boolean fillContents, boolean maybeUnroll,
 636                     Counters counters) {
 637         memory.writeInt(arrayLengthOffset(INJECTED_VMCONFIG), length, LocationIdentity.init());
 638         /*
 639          * store hub last as the concurrent garbage collectors assume length is valid if hub field
 640          * is not null
 641          */
 642         initializeObjectHeader(memory, prototypeMarkWord, hub);
 643         if (fillContents) {
 644             zeroMemory(allocationSize, memory, false, headerSize, maybeUnroll, counters);
 645         } else if (REPLACEMENTS_ASSERTIONS_ENABLED) {
 646             fillWithGarbage(allocationSize, memory, false, headerSize, maybeUnroll, counters);
 647         }
 648         MembarNode.memoryBarrier(MemoryBarriers.STORE_STORE, LocationIdentity.init());
 649         return memory.toObjectNonNull();
 650     }
 651 
 652     static class Counters {
 653         Counters(SnippetCounter.Group.Factory factory) {
 654             Group newInstance = factory.createSnippetCounterGroup("NewInstance");
 655             Group newArray = factory.createSnippetCounterGroup("NewArray");
 656             instanceSeqInit = new SnippetCounter(newInstance, "tlabSeqInit", "TLAB alloc with unrolled zeroing");
 657             instanceLoopInit = new SnippetCounter(newInstance, "tlabLoopInit", "TLAB alloc with zeroing in a loop");
 658             arrayLoopInit = new SnippetCounter(newArray, "tlabLoopInit", "TLAB alloc with zeroing in a loop");
 659             stub = new SnippetCounter(newInstance, "stub", "alloc and zeroing via stub");
 660         }
 661 
 662         final SnippetCounter instanceSeqInit;
 663         final SnippetCounter instanceLoopInit;
 664         final SnippetCounter arrayLoopInit;
 665         final SnippetCounter stub;
 666     }
 667 
 668     public static class Templates extends AbstractTemplates {
 669 
 670         private final SnippetInfo allocateInstance = snippet(NewObjectSnippets.class, "allocateInstance", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION, TLAB_END_LOCATION);
 671         private final SnippetInfo allocateInstancePIC = snippet(NewObjectSnippets.class, "allocateInstancePIC", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION,
 672                         TLAB_END_LOCATION);
 673         private final SnippetInfo allocateArray = snippet(NewObjectSnippets.class, "allocateArray", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION, TLAB_END_LOCATION);
 674         private final SnippetInfo allocateArrayPIC = snippet(NewObjectSnippets.class, "allocateArrayPIC", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION, TLAB_END_LOCATION);
 675         private final SnippetInfo allocatePrimitiveArrayPIC = snippet(NewObjectSnippets.class, "allocatePrimitiveArrayPIC", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION,
 676                         TLAB_END_LOCATION);
 677         private final SnippetInfo allocateArrayDynamic = snippet(NewObjectSnippets.class, "allocateArrayDynamic", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION,
 678                         TLAB_END_LOCATION);
 679         private final SnippetInfo allocateInstanceDynamic = snippet(NewObjectSnippets.class, "allocateInstanceDynamic", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION,
 680                         TLAB_END_LOCATION);
 681         private final SnippetInfo newmultiarray = snippet(NewObjectSnippets.class, "newmultiarray", TLAB_TOP_LOCATION, TLAB_END_LOCATION);
 682         private final SnippetInfo newmultiarrayPIC = snippet(NewObjectSnippets.class, "newmultiarrayPIC", TLAB_TOP_LOCATION, TLAB_END_LOCATION);
 683         private final SnippetInfo verifyHeap = snippet(NewObjectSnippets.class, "verifyHeap");
 684         private final GraalHotSpotVMConfig config;
 685         private final Counters counters;
 686 
 687         public Templates(OptionValues options, Iterable<DebugHandlersFactory> factories, SnippetCounter.Group.Factory factory, HotSpotProviders providers, TargetDescription target,
 688                         GraalHotSpotVMConfig config) {
 689             super(options, factories, providers, providers.getSnippetReflection(), target);
 690             this.config = config;
 691             counters = new Counters(factory);
 692         }
 693 
 694         /**
 695          * Lowers a {@link NewInstanceNode}.
 696          */
 697         public void lower(NewInstanceNode newInstanceNode, HotSpotRegistersProvider registers, LoweringTool tool) {
 698             StructuredGraph graph = newInstanceNode.graph();
 699             HotSpotResolvedObjectType type = (HotSpotResolvedObjectType) newInstanceNode.instanceClass();
 700             assert !type.isArray();
 701             ConstantNode hub = ConstantNode.forConstant(KlassPointerStamp.klassNonNull(), type.klass(), providers.getMetaAccess(), graph);
 702             int size = instanceSize(type);
 703 
 704             OptionValues localOptions = graph.getOptions();
 705             SnippetInfo snippet = GeneratePIC.getValue(localOptions) ? allocateInstancePIC : allocateInstance;
 706             Arguments args = new Arguments(snippet, graph.getGuardsStage(), tool.getLoweringStage());
 707             args.addConst("size", size);
 708             args.add("hub", hub);
 709             args.add("prototypeMarkWord", type.prototypeMarkWord());
 710             args.addConst("fillContents", newInstanceNode.fillContents());
 711             args.addConst("threadRegister", registers.getThreadRegister());
 712             args.addConst("constantSize", true);
 713             args.addConst("typeContext", ProfileAllocations.getValue(localOptions) ? type.toJavaName(false) : "");
 714             args.addConst("counters", counters);
 715 
 716             SnippetTemplate template = template(newInstanceNode, args);
 717             graph.getDebug().log("Lowering allocateInstance in %s: node=%s, template=%s, arguments=%s", graph, newInstanceNode, template, args);
 718             template.instantiate(providers.getMetaAccess(), newInstanceNode, DEFAULT_REPLACER, args);
 719         }
 720 
 721         /**
 722          * Lowers a {@link NewArrayNode}.
 723          */
 724         public void lower(NewArrayNode newArrayNode, HotSpotRegistersProvider registers, LoweringTool tool) {
 725             StructuredGraph graph = newArrayNode.graph();
 726             ResolvedJavaType elementType = newArrayNode.elementType();
 727             HotSpotResolvedObjectType arrayType = (HotSpotResolvedObjectType) elementType.getArrayClass();
 728             JavaKind elementKind = elementType.getJavaKind();
 729             ConstantNode hub = ConstantNode.forConstant(KlassPointerStamp.klassNonNull(), arrayType.klass(), providers.getMetaAccess(), graph);
 730             final int headerSize = tool.getMetaAccess().getArrayBaseOffset(elementKind);
 731             int log2ElementSize = CodeUtil.log2(tool.getMetaAccess().getArrayIndexScale(elementKind));
 732 
 733             OptionValues localOptions = graph.getOptions();
 734             SnippetInfo snippet;
 735             if (GeneratePIC.getValue(localOptions)) {
 736                 if (elementType.isPrimitive()) {
 737                     snippet = allocatePrimitiveArrayPIC;
 738                 } else {
 739                     snippet = allocateArrayPIC;
 740                 }
 741             } else {
 742                 snippet = allocateArray;
 743             }
 744 
 745             Arguments args = new Arguments(snippet, graph.getGuardsStage(), tool.getLoweringStage());
 746             args.add("hub", hub);
 747             ValueNode length = newArrayNode.length();
 748             args.add("length", length.isAlive() ? length : graph.addOrUniqueWithInputs(length));
 749             assert arrayType.prototypeMarkWord() == lookupArrayClass(tool, JavaKind.Object).prototypeMarkWord() : "all array types are assumed to have the same prototypeMarkWord";
 750             args.add("prototypeMarkWord", arrayType.prototypeMarkWord());
 751             args.addConst("headerSize", headerSize);
 752             args.addConst("log2ElementSize", log2ElementSize);
 753             args.addConst("fillContents", newArrayNode.fillContents());
 754             args.addConst("threadRegister", registers.getThreadRegister());
 755             args.addConst("maybeUnroll", length.isConstant());
 756             args.addConst("typeContext", ProfileAllocations.getValue(localOptions) ? arrayType.toJavaName(false) : "");
 757             args.addConst("counters", counters);
 758             SnippetTemplate template = template(newArrayNode, args);
 759             graph.getDebug().log("Lowering allocateArray in %s: node=%s, template=%s, arguments=%s", graph, newArrayNode, template, args);
 760             template.instantiate(providers.getMetaAccess(), newArrayNode, DEFAULT_REPLACER, args);
 761         }
 762 
 763         public void lower(DynamicNewInstanceNode newInstanceNode, HotSpotRegistersProvider registers, LoweringTool tool) {
 764             Arguments args = new Arguments(allocateInstanceDynamic, newInstanceNode.graph().getGuardsStage(), tool.getLoweringStage());
 765             args.add("type", newInstanceNode.getInstanceType());
 766             ValueNode classClass = newInstanceNode.getClassClass();
 767             assert classClass != null;
 768             args.add("classClass", classClass);
 769             args.addConst("fillContents", newInstanceNode.fillContents());
 770             args.addConst("threadRegister", registers.getThreadRegister());
 771             args.addConst("counters", counters);
 772 
 773             SnippetTemplate template = template(newInstanceNode, args);
 774             template.instantiate(providers.getMetaAccess(), newInstanceNode, DEFAULT_REPLACER, args);
 775         }
 776 
 777         public void lower(DynamicNewArrayNode newArrayNode, HotSpotRegistersProvider registers, LoweringTool tool) {
 778             StructuredGraph graph = newArrayNode.graph();
 779             Arguments args = new Arguments(allocateArrayDynamic, newArrayNode.graph().getGuardsStage(), tool.getLoweringStage());
 780             args.add("elementType", newArrayNode.getElementType());
 781             ValueNode voidClass = newArrayNode.getVoidClass();
 782             assert voidClass != null;
 783             args.add("voidClass", voidClass);
 784             ValueNode length = newArrayNode.length();
 785             args.add("length", length.isAlive() ? length : graph.addOrUniqueWithInputs(length));
 786             args.addConst("fillContents", newArrayNode.fillContents());
 787             args.addConst("threadRegister", registers.getThreadRegister());
 788             /*
 789              * We use Kind.Illegal as a marker value instead of null because constant snippet
 790              * parameters cannot be null.
 791              */
 792             args.addConst("knownElementKind", newArrayNode.getKnownElementKind() == null ? JavaKind.Illegal : newArrayNode.getKnownElementKind());
 793             if (newArrayNode.getKnownElementKind() != null) {
 794                 args.addConst("knownLayoutHelper", lookupArrayClass(tool, newArrayNode.getKnownElementKind()).layoutHelper());
 795             } else {
 796                 args.addConst("knownLayoutHelper", 0);
 797             }
 798             args.add("prototypeMarkWord", lookupArrayClass(tool, JavaKind.Object).prototypeMarkWord());
 799             args.addConst("counters", counters);
 800             SnippetTemplate template = template(newArrayNode, args);
 801             template.instantiate(providers.getMetaAccess(), newArrayNode, DEFAULT_REPLACER, args);
 802         }
 803 
 804         private static HotSpotResolvedObjectType lookupArrayClass(LoweringTool tool, JavaKind kind) {
 805             return (HotSpotResolvedObjectType) tool.getMetaAccess().lookupJavaType(kind == JavaKind.Object ? Object.class : kind.toJavaClass()).getArrayClass();
 806         }
 807 
 808         public void lower(NewMultiArrayNode newmultiarrayNode, LoweringTool tool) {
 809             StructuredGraph graph = newmultiarrayNode.graph();
 810             OptionValues localOptions = graph.getOptions();
 811             int rank = newmultiarrayNode.dimensionCount();
 812             ValueNode[] dims = new ValueNode[rank];
 813             for (int i = 0; i < newmultiarrayNode.dimensionCount(); i++) {
 814                 dims[i] = newmultiarrayNode.dimension(i);
 815             }
 816             HotSpotResolvedObjectType type = (HotSpotResolvedObjectType) newmultiarrayNode.type();
 817             ConstantNode hub = ConstantNode.forConstant(KlassPointerStamp.klassNonNull(), type.klass(), providers.getMetaAccess(), graph);
 818 
 819             SnippetInfo snippet = GeneratePIC.getValue(localOptions) ? newmultiarrayPIC : newmultiarray;
 820             Arguments args = new Arguments(snippet, graph.getGuardsStage(), tool.getLoweringStage());
 821             args.add("hub", hub);
 822             args.addConst("rank", rank);
 823             args.addVarargs("dimensions", int.class, StampFactory.forKind(JavaKind.Int), dims);
 824             template(newmultiarrayNode, args).instantiate(providers.getMetaAccess(), newmultiarrayNode, DEFAULT_REPLACER, args);
 825         }
 826 
 827         private static int instanceSize(HotSpotResolvedObjectType type) {
 828             int size = type.instanceSize();
 829             assert size >= 0;
 830             return size;
 831         }
 832 
 833         public void lower(VerifyHeapNode verifyHeapNode, HotSpotRegistersProvider registers, LoweringTool tool) {
 834             if (config.cAssertions) {
 835                 Arguments args = new Arguments(verifyHeap, verifyHeapNode.graph().getGuardsStage(), tool.getLoweringStage());
 836                 args.addConst("threadRegister", registers.getThreadRegister());
 837 
 838                 SnippetTemplate template = template(verifyHeapNode, args);
 839                 template.instantiate(providers.getMetaAccess(), verifyHeapNode, DEFAULT_REPLACER, args);
 840             } else {
 841                 GraphUtil.removeFixedWithUnusedInputs(verifyHeapNode);
 842             }
 843         }
 844     }
 845 }
--- EOF ---