< prev index next >

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/NewObjectSnippets.java

Print this page




   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  */
  23 
  24 
  25 package org.graalvm.compiler.hotspot.replacements;
  26 


  27 import static org.graalvm.compiler.core.common.GraalOptions.GeneratePIC;
  28 import static org.graalvm.compiler.core.common.calc.UnsignedMath.belowThan;
  29 import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfig.INJECTED_VMCONFIG;






  30 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.CLASS_ARRAY_KLASS_LOCATION;
  31 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.HUB_WRITE_LOCATION;
  32 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.MARK_WORD_LOCATION;
  33 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.PROTOTYPE_MARK_WORD_LOCATION;
  34 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_END_LOCATION;
  35 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_TOP_LOCATION;
  36 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayAllocationSize;
  37 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayKlassOffset;
  38 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayLengthOffset;
  39 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.config;
  40 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.initializeObjectHeader;
  41 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.instanceHeaderSize;
  42 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.isInstanceKlassFullyInitialized;
  43 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperHeaderSizeMask;
  44 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperHeaderSizeShift;
  45 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperLog2ElementSizeMask;
  46 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperLog2ElementSizeShift;
  47 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.loadKlassFromObject;
  48 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.prototypeMarkWordOffset;
  49 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readLayoutHelper;


  53 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.useBiasedLocking;
  54 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.useTLAB;
  55 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.verifyOop;
  56 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.writeTlabTop;
  57 import static org.graalvm.compiler.hotspot.replacements.HotspotSnippetsOptions.ProfileAllocations;
  58 import static org.graalvm.compiler.hotspot.replacements.HotspotSnippetsOptions.ProfileAllocationsContext;
  59 import static org.graalvm.compiler.nodes.PiArrayNode.piArrayCastToSnippetReplaceeStamp;
  60 import static org.graalvm.compiler.nodes.PiNode.piCastToSnippetReplaceeStamp;
  61 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.FAST_PATH_PROBABILITY;
  62 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.FREQUENT_PROBABILITY;
  63 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.SLOW_PATH_PROBABILITY;
  64 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.probability;
  65 import static org.graalvm.compiler.replacements.ReplacementsUtil.REPLACEMENTS_ASSERTIONS_ENABLED;
  66 import static org.graalvm.compiler.replacements.ReplacementsUtil.runtimeAssert;
  67 import static org.graalvm.compiler.replacements.ReplacementsUtil.staticAssert;
  68 import static org.graalvm.compiler.replacements.SnippetTemplate.DEFAULT_REPLACER;
  69 import static org.graalvm.compiler.replacements.nodes.CStringConstant.cstring;
  70 import static org.graalvm.compiler.replacements.nodes.ExplodeLoopNode.explodeLoop;
  71 
  72 import org.graalvm.compiler.api.replacements.Fold;

  73 import org.graalvm.compiler.api.replacements.Snippet;
  74 import org.graalvm.compiler.api.replacements.Snippet.ConstantParameter;
  75 import org.graalvm.compiler.api.replacements.Snippet.VarargsParameter;
  76 import org.graalvm.compiler.core.common.spi.ForeignCallDescriptor;
  77 import org.graalvm.compiler.core.common.type.StampFactory;
  78 import org.graalvm.compiler.debug.DebugHandlersFactory;
  79 import org.graalvm.compiler.debug.GraalError;
  80 import org.graalvm.compiler.graph.Node.ConstantNodeParameter;
  81 import org.graalvm.compiler.graph.Node.NodeIntrinsic;
  82 import org.graalvm.compiler.hotspot.GraalHotSpotVMConfig;
  83 import org.graalvm.compiler.hotspot.HotSpotBackend;
  84 import org.graalvm.compiler.hotspot.meta.HotSpotProviders;
  85 import org.graalvm.compiler.hotspot.meta.HotSpotRegistersProvider;
  86 import org.graalvm.compiler.hotspot.nodes.DimensionsNode;
  87 import org.graalvm.compiler.hotspot.nodes.aot.LoadConstantIndirectlyFixedNode;
  88 import org.graalvm.compiler.hotspot.nodes.aot.LoadConstantIndirectlyNode;
  89 import org.graalvm.compiler.hotspot.nodes.type.KlassPointerStamp;
  90 import org.graalvm.compiler.hotspot.word.KlassPointer;
  91 import org.graalvm.compiler.nodes.ConstantNode;
  92 import org.graalvm.compiler.nodes.DeoptimizeNode;
  93 import org.graalvm.compiler.nodes.PiNode;
  94 import org.graalvm.compiler.nodes.PrefetchAllocateNode;
  95 import org.graalvm.compiler.nodes.SnippetAnchorNode;
  96 import org.graalvm.compiler.nodes.StructuredGraph;
  97 import org.graalvm.compiler.nodes.ValueNode;
  98 import org.graalvm.compiler.nodes.debug.DynamicCounterNode;
  99 import org.graalvm.compiler.nodes.debug.VerifyHeapNode;
 100 import org.graalvm.compiler.nodes.extended.ForeignCallNode;
 101 import org.graalvm.compiler.nodes.extended.MembarNode;
 102 import org.graalvm.compiler.nodes.java.DynamicNewArrayNode;
 103 import org.graalvm.compiler.nodes.java.DynamicNewInstanceNode;


 109 import org.graalvm.compiler.nodes.util.GraphUtil;
 110 import org.graalvm.compiler.options.OptionValues;
 111 import org.graalvm.compiler.replacements.ReplacementsUtil;
 112 import org.graalvm.compiler.replacements.SnippetCounter;
 113 import org.graalvm.compiler.replacements.SnippetCounter.Group;
 114 import org.graalvm.compiler.replacements.SnippetTemplate;
 115 import org.graalvm.compiler.replacements.SnippetTemplate.AbstractTemplates;
 116 import org.graalvm.compiler.replacements.SnippetTemplate.Arguments;
 117 import org.graalvm.compiler.replacements.SnippetTemplate.SnippetInfo;
 118 import org.graalvm.compiler.replacements.Snippets;
 119 import org.graalvm.compiler.replacements.nodes.ExplodeLoopNode;
 120 import org.graalvm.compiler.word.Word;
 121 import jdk.internal.vm.compiler.word.LocationIdentity;
 122 import jdk.internal.vm.compiler.word.WordFactory;
 123 
 124 import jdk.vm.ci.code.CodeUtil;
 125 import jdk.vm.ci.code.MemoryBarriers;
 126 import jdk.vm.ci.code.Register;
 127 import jdk.vm.ci.code.TargetDescription;
 128 import jdk.vm.ci.hotspot.HotSpotResolvedObjectType;
 129 import jdk.vm.ci.meta.DeoptimizationAction;
 130 import jdk.vm.ci.meta.DeoptimizationReason;
 131 import jdk.vm.ci.meta.JavaKind;
 132 import jdk.vm.ci.meta.ResolvedJavaType;
 133 
 134 /**
 135  * Snippets used for implementing NEW, ANEWARRAY and NEWARRAY.
 136  */
 137 public class NewObjectSnippets implements Snippets {
 138 
 139     enum ProfileContext {
 140         AllocatingMethod,
 141         InstanceOrArray,
 142         AllocatedType,
 143         AllocatedTypesInMethod,
 144         Total
 145     }
 146 
 147     @Fold
 148     static String createName(String path, String typeContext, OptionValues options) {
 149         switch (ProfileAllocationsContext.getValue(options)) {
 150             case AllocatingMethod:


 203                     @ConstantParameter Register threadRegister, @ConstantParameter boolean constantSize, @ConstantParameter String typeContext, @ConstantParameter OptionValues options,
 204                     @ConstantParameter Counters counters) {
 205         return piCastToSnippetReplaceeStamp(allocateInstanceHelper(size, hub, prototypeMarkWord, fillContents, threadRegister, constantSize, typeContext, options, counters));
 206     }
 207 
 208     public static Object allocateInstanceHelper(int size, KlassPointer hub, Word prototypeMarkWord, boolean fillContents,
 209                     Register threadRegister, boolean constantSize, String typeContext, OptionValues options, Counters counters) {
 210         Object result;
 211         Word thread = registerAsWord(threadRegister);
 212         Word top = readTlabTop(thread);
 213         Word end = readTlabEnd(thread);
 214         Word newTop = top.add(size);
 215         if (useTLAB(INJECTED_VMCONFIG) && probability(FAST_PATH_PROBABILITY, newTop.belowOrEqual(end))) {
 216             writeTlabTop(thread, newTop);
 217             emitPrefetchAllocate(newTop, false);
 218             result = formatObject(hub, size, top, prototypeMarkWord, fillContents, constantSize, counters);
 219         } else {
 220             if (counters != null && counters.stub != null) {
 221                 counters.stub.inc();
 222             }
 223             result = newInstance(HotSpotBackend.NEW_INSTANCE, hub);
 224         }
 225         profileAllocation("instance", size, typeContext, options);
 226         return verifyOop(result);
 227     }
 228 








 229     @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = true)
 230     public static native Object newInstance(@ConstantNodeParameter ForeignCallDescriptor descriptor, KlassPointer hub);



 231 
 232     @Snippet
 233     public static Object allocateInstancePIC(@ConstantParameter int size, KlassPointer hub, Word prototypeMarkWord, @ConstantParameter boolean fillContents,
 234                     @ConstantParameter Register threadRegister, @ConstantParameter boolean constantSize, @ConstantParameter String typeContext, @ConstantParameter OptionValues options,
 235                     @ConstantParameter Counters counters) {
 236         // Klass must be initialized by the time the first instance is allocated, therefore we can
 237         // just load it from the corresponding cell and avoid the resolution check. We have to use a
 238         // fixed load though, to prevent it from floating above the initialization.
 239         KlassPointer picHub = LoadConstantIndirectlyFixedNode.loadKlass(hub);
 240         return piCastToSnippetReplaceeStamp(allocateInstanceHelper(size, picHub, prototypeMarkWord, fillContents, threadRegister, constantSize, typeContext, options, counters));
 241     }
 242 
 243     @Snippet
 244     public static Object allocateInstanceDynamic(Class<?> type, Class<?> classClass, @ConstantParameter boolean fillContents, @ConstantParameter Register threadRegister,
 245                     @ConstantParameter OptionValues options, @ConstantParameter Counters counters) {
 246         if (probability(SLOW_PATH_PROBABILITY, type == null)) {
 247             DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
 248         }
 249         Class<?> nonNullType = PiNode.piCastNonNullClass(type, SnippetAnchorNode.anchor());
 250 
 251         if (probability(SLOW_PATH_PROBABILITY, DynamicNewInstanceNode.throwsInstantiationException(type, classClass))) {
 252             DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
 253         }
 254 
 255         return PiNode.piCastToSnippetReplaceeStamp(allocateInstanceDynamicHelper(type, fillContents, threadRegister, options, counters, nonNullType));
 256     }
 257 
 258     private static Object allocateInstanceDynamicHelper(Class<?> type, boolean fillContents, Register threadRegister, OptionValues options, Counters counters, Class<?> nonNullType) {
 259         KlassPointer hub = ClassGetHubNode.readClass(nonNullType);
 260         if (probability(FAST_PATH_PROBABILITY, !hub.isNull())) {
 261             KlassPointer nonNullHub = ClassGetHubNode.piCastNonNull(hub, SnippetAnchorNode.anchor());
 262 
 263             if (probability(FAST_PATH_PROBABILITY, isInstanceKlassFullyInitialized(nonNullHub))) {
 264                 int layoutHelper = readLayoutHelper(nonNullHub);
 265                 /*
 266                  * src/share/vm/oops/klass.hpp: For instances, layout helper is a positive number,
 267                  * the instance size. This size is already passed through align_object_size and
 268                  * scaled to bytes. The low order bit is set if instances of this class cannot be
 269                  * allocated using the fastpath.
 270                  */
 271                 if (probability(FAST_PATH_PROBABILITY, (layoutHelper & 1) == 0)) {
 272                     Word prototypeMarkWord = nonNullHub.readWord(prototypeMarkWordOffset(INJECTED_VMCONFIG), PROTOTYPE_MARK_WORD_LOCATION);
 273                     /*
 274                      * FIXME(je,ds): we should actually pass typeContext instead of "" but late
 275                      * binding of parameters is not yet supported by the GraphBuilderPlugin system.
 276                      */
 277                     return allocateInstanceHelper(layoutHelper, nonNullHub, prototypeMarkWord, fillContents, threadRegister, false, "", options, counters);
 278                 }
 279             } else {
 280                 DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
 281             }
 282         }
 283         return dynamicNewInstanceStub(type);

 284     }
 285 
 286     /**
 287      * Maximum array length for which fast path allocation is used.
 288      */
 289     public static final int MAX_ARRAY_FAST_PATH_ALLOCATION_LENGTH = 0x00FFFFFF;
 290 
 291     @Snippet
 292     public static Object allocatePrimitiveArrayPIC(KlassPointer hub, int length, Word prototypeMarkWord, @ConstantParameter int headerSize, @ConstantParameter int log2ElementSize,
 293                     @ConstantParameter boolean fillContents, @ConstantParameter Register threadRegister, @ConstantParameter boolean maybeUnroll, @ConstantParameter String typeContext,
 294                     @ConstantParameter OptionValues options, @ConstantParameter Counters counters) {
 295         // Primitive array types are eagerly pre-resolved. We can use a floating load.
 296         KlassPointer picHub = LoadConstantIndirectlyNode.loadKlass(hub);
 297         return allocateArrayImpl(picHub, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents, threadRegister, maybeUnroll, typeContext, false, options, counters);
 298     }
 299 
 300     @Snippet
 301     public static Object allocateArrayPIC(KlassPointer hub, int length, Word prototypeMarkWord, @ConstantParameter int headerSize, @ConstantParameter int log2ElementSize,
 302                     @ConstantParameter boolean fillContents, @ConstantParameter Register threadRegister, @ConstantParameter boolean maybeUnroll, @ConstantParameter String typeContext,
 303                     @ConstantParameter OptionValues options, @ConstantParameter Counters counters) {
 304         // Array type would be resolved by dominating resolution.
 305         KlassPointer picHub = LoadConstantIndirectlyFixedNode.loadKlass(hub);
 306         return allocateArrayImpl(picHub, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents, threadRegister, maybeUnroll, typeContext, false, options, counters);
 307     }
 308 
 309     @Snippet
 310     public static Object allocateArray(KlassPointer hub, int length, Word prototypeMarkWord, @ConstantParameter int headerSize, @ConstantParameter int log2ElementSize,
 311                     @ConstantParameter boolean fillContents, @ConstantParameter Register threadRegister, @ConstantParameter boolean maybeUnroll, @ConstantParameter String typeContext,
 312                     @ConstantParameter OptionValues options, @ConstantParameter Counters counters) {








 313         Object result = allocateArrayImpl(hub, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents, threadRegister, maybeUnroll, typeContext, false, options, counters);
 314         return piArrayCastToSnippetReplaceeStamp(verifyOop(result), length);
 315     }
 316 









 317     private static Object allocateArrayImpl(KlassPointer hub, int length, Word prototypeMarkWord, int headerSize, int log2ElementSize, boolean fillContents, Register threadRegister,
 318                     boolean maybeUnroll, String typeContext, boolean skipNegativeCheck, OptionValues options, Counters counters) {
 319         Object result;
 320         int allocationSize = arrayAllocationSize(length, headerSize, log2ElementSize);
 321         Word thread = registerAsWord(threadRegister);
 322         Word top = readTlabTop(thread);
 323         Word end = readTlabEnd(thread);
 324         Word newTop = top.add(allocationSize);
 325         if (probability(FREQUENT_PROBABILITY, skipNegativeCheck || belowThan(length, MAX_ARRAY_FAST_PATH_ALLOCATION_LENGTH)) && useTLAB(INJECTED_VMCONFIG) &&
 326                         probability(FAST_PATH_PROBABILITY, newTop.belowOrEqual(end))) {
 327             writeTlabTop(thread, newTop);
 328             emitPrefetchAllocate(newTop, true);
 329             if (counters != null && counters.arrayLoopInit != null) {
 330                 counters.arrayLoopInit.inc();
 331             }
 332             result = formatArray(hub, allocationSize, length, headerSize, top, prototypeMarkWord, fillContents, maybeUnroll, counters);
 333         } else {
 334             result = newArray(HotSpotBackend.NEW_ARRAY, hub, length);
 335         }
 336         profileAllocation("array", allocationSize, typeContext, options);
 337         return result;
 338     }
 339 
 340     @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = true)
 341     public static native Object newArray(@ConstantNodeParameter ForeignCallDescriptor descriptor, KlassPointer hub, int length);
 342 
 343     public static final ForeignCallDescriptor DYNAMIC_NEW_ARRAY = new ForeignCallDescriptor("dynamic_new_array", Object.class, Class.class, int.class);
 344     public static final ForeignCallDescriptor DYNAMIC_NEW_INSTANCE = new ForeignCallDescriptor("dynamic_new_instance", Object.class, Class.class);


 345 
 346     @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = true)
 347     public static native Object dynamicNewArrayStub(@ConstantNodeParameter ForeignCallDescriptor descriptor, Class<?> elementType, int length);
 348 
 349     public static Object dynamicNewInstanceStub(Class<?> elementType) {
 350         return dynamicNewInstanceStubCall(DYNAMIC_NEW_INSTANCE, elementType);









 351     }
 352 
 353     @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = true)
 354     public static native Object dynamicNewInstanceStubCall(@ConstantNodeParameter ForeignCallDescriptor descriptor, Class<?> elementType);



 355 
 356     @Snippet
 357     public static Object allocateArrayDynamic(Class<?> elementType, Class<?> voidClass, int length, @ConstantParameter boolean fillContents, @ConstantParameter Register threadRegister,
 358                     @ConstantParameter JavaKind knownElementKind, @ConstantParameter int knownLayoutHelper, Word prototypeMarkWord, @ConstantParameter OptionValues options,
 359                     @ConstantParameter Counters counters) {
 360         Object result = allocateArrayDynamicImpl(elementType, voidClass, length, fillContents, threadRegister, knownElementKind, knownLayoutHelper, prototypeMarkWord, options, counters);
 361         return result;
 362     }
 363 
 364     private static Object allocateArrayDynamicImpl(Class<?> elementType, Class<?> voidClass, int length, boolean fillContents, Register threadRegister, JavaKind knownElementKind,
 365                     int knownLayoutHelper, Word prototypeMarkWord, OptionValues options, Counters counters) {
 366         /*
 367          * We only need the dynamic check for void when we have no static information from
 368          * knownElementKind.
 369          */
 370         staticAssert(knownElementKind != JavaKind.Void, "unsupported knownElementKind");
 371         if (knownElementKind == JavaKind.Illegal && probability(SLOW_PATH_PROBABILITY, elementType == null || DynamicNewArrayNode.throwsIllegalArgumentException(elementType, voidClass))) {
 372             DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
 373         }
 374 
 375         KlassPointer klass = loadKlassFromObject(elementType, arrayKlassOffset(INJECTED_VMCONFIG), CLASS_ARRAY_KLASS_LOCATION);
 376         if (klass.isNull()) {
 377             DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
 378         }
 379         KlassPointer nonNullKlass = ClassGetHubNode.piCastNonNull(klass, SnippetAnchorNode.anchor());
 380 
 381         if (length < 0) {
 382             DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
 383         }
 384         int layoutHelper;
 385         if (knownElementKind == JavaKind.Illegal) {
 386             layoutHelper = readLayoutHelper(nonNullKlass);
 387         } else {
 388             runtimeAssert(knownLayoutHelper == readLayoutHelper(nonNullKlass), "layout mismatch");
 389             layoutHelper = knownLayoutHelper;
 390         }
 391         //@formatter:off
 392         // from src/share/vm/oops/klass.hpp:
 393         //
 394         // For arrays, layout helper is a negative number, containing four
 395         // distinct bytes, as follows:
 396         //    MSB:[tag, hsz, ebt, log2(esz)]:LSB
 397         // where:
 398         //    tag is 0x80 if the elements are oops, 0xC0 if non-oops
 399         //    hsz is array header size in bytes (i.e., offset of first element)
 400         //    ebt is the BasicType of the elements
 401         //    esz is the element size in bytes
 402         //@formatter:on
 403 
 404         int headerSize = (layoutHelper >> layoutHelperHeaderSizeShift(INJECTED_VMCONFIG)) & layoutHelperHeaderSizeMask(INJECTED_VMCONFIG);
 405         int log2ElementSize = (layoutHelper >> layoutHelperLog2ElementSizeShift(INJECTED_VMCONFIG)) & layoutHelperLog2ElementSizeMask(INJECTED_VMCONFIG);
 406 
 407         Object result = allocateArrayImpl(nonNullKlass, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents, threadRegister, false, "dynamic type", true, options, counters);
 408         return piArrayCastToSnippetReplaceeStamp(verifyOop(result), length);
 409     }
 410 
 411     /**
 412      * Calls the runtime stub for implementing MULTIANEWARRAY.
 413      */
 414     @Snippet
 415     public static Object newmultiarray(KlassPointer hub, @ConstantParameter int rank, @VarargsParameter int[] dimensions) {
 416         Word dims = DimensionsNode.allocaDimsArray(rank);
 417         ExplodeLoopNode.explodeLoop();
 418         for (int i = 0; i < rank; i++) {
 419             dims.writeInt(i * 4, dimensions[i], LocationIdentity.init());
 420         }
 421         return newArrayCall(HotSpotBackend.NEW_MULTI_ARRAY, hub, rank, dims);








 422     }
 423 
 424     @Snippet
 425     public static Object newmultiarrayPIC(KlassPointer hub, @ConstantParameter int rank, @VarargsParameter int[] dimensions) {
 426         // Array type would be resolved by dominating resolution.
 427         KlassPointer picHub = LoadConstantIndirectlyFixedNode.loadKlass(hub);
 428         return newmultiarray(picHub, rank, dimensions);
 429     }
 430 
 431     @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = true)
 432     public static native Object newArrayCall(@ConstantNodeParameter ForeignCallDescriptor descriptor, KlassPointer hub, int rank, Word dims);



 433 
 434     /**
 435      * Maximum number of long stores to emit when zeroing an object with a constant size. Larger
 436      * objects have their bodies initialized in a loop.
 437      */
 438     private static final int MAX_UNROLLED_OBJECT_ZEROING_STORES = 8;
 439 
 440     /**
 441      * Zero uninitialized memory in a newly allocated object, unrolling as necessary and ensuring
 442      * that stores are aligned.
 443      *
 444      * @param size number of bytes to zero
 445      * @param memory beginning of object which is being zeroed
 446      * @param constantSize is {@code size} known to be constant in the snippet
 447      * @param startOffset offset to begin zeroing. May not be word aligned.
 448      * @param manualUnroll maximally unroll zeroing
 449      */
 450     private static void zeroMemory(int size, Word memory, boolean constantSize, int startOffset, boolean manualUnroll, Counters counters) {
 451         fillMemory(0, size, memory, constantSize, startOffset, manualUnroll, counters);
 452     }


 492                 memory.initializeLong(off, value, LocationIdentity.init());
 493             }
 494         }
 495     }
 496 
 497     /**
 498      * Fill uninitialized memory with garbage value in a newly allocated object, unrolling as
 499      * necessary and ensuring that stores are aligned.
 500      *
 501      * @param size number of bytes to zero
 502      * @param memory beginning of object which is being zeroed
 503      * @param constantSize is {@code  size} known to be constant in the snippet
 504      * @param startOffset offset to begin zeroing. May not be word aligned.
 505      * @param manualUnroll maximally unroll zeroing
 506      */
 507     private static void fillWithGarbage(int size, Word memory, boolean constantSize, int startOffset, boolean manualUnroll, Counters counters) {
 508         fillMemory(0xfefefefefefefefeL, size, memory, constantSize, startOffset, manualUnroll, counters);
 509     }
 510 
 511     /**
 512      * Formats some allocated memory with an object header and zeroes out the rest. Disables asserts
 513      * since they can't be compiled in stubs.
 514      */
 515     public static Object formatObjectForStub(KlassPointer hub, int size, Word memory, Word compileTimePrototypeMarkWord) {
 516         return formatObject(hub, size, memory, compileTimePrototypeMarkWord, true, false, null);
 517     }
 518 
 519     /**
 520      * Formats some allocated memory with an object header and zeroes out the rest.
 521      */
 522     protected static Object formatObject(KlassPointer hub, int size, Word memory, Word compileTimePrototypeMarkWord, boolean fillContents, boolean constantSize, Counters counters) {
 523         Word prototypeMarkWord = useBiasedLocking(INJECTED_VMCONFIG) ? hub.readWord(prototypeMarkWordOffset(INJECTED_VMCONFIG), PROTOTYPE_MARK_WORD_LOCATION) : compileTimePrototypeMarkWord;
 524         initializeObjectHeader(memory, prototypeMarkWord, hub);
 525         if (fillContents) {
 526             zeroMemory(size, memory, constantSize, instanceHeaderSize(INJECTED_VMCONFIG), false, counters);
 527         } else if (REPLACEMENTS_ASSERTIONS_ENABLED) {
 528             fillWithGarbage(size, memory, constantSize, instanceHeaderSize(INJECTED_VMCONFIG), false, counters);
 529         }
 530         MembarNode.memoryBarrier(MemoryBarriers.STORE_STORE, LocationIdentity.init());
 531         return memory.toObjectNonNull();
 532     }
 533 
 534     @Snippet
 535     protected static void verifyHeap(@ConstantParameter Register threadRegister) {
 536         Word thread = registerAsWord(threadRegister);
 537         Word topValue = readTlabTop(thread);
 538         if (!topValue.equal(WordFactory.zero())) {
 539             Word topValueContents = topValue.readWord(0, MARK_WORD_LOCATION);
 540             if (topValueContents.equal(WordFactory.zero())) {
 541                 AssertionSnippets.vmMessageC(AssertionSnippets.ASSERTION_VM_MESSAGE_C, true, cstring("overzeroing of TLAB detected"), 0L, 0L, 0L);
 542             }
 543         }
 544     }
 545 
 546     /**
 547      * Formats some allocated memory with an object header and zeroes out the rest.
 548      */
 549     public static Object formatArray(KlassPointer hub, int allocationSize, int length, int headerSize, Word memory, Word prototypeMarkWord, boolean fillContents, boolean maybeUnroll,
 550                     Counters counters) {
 551         memory.writeInt(arrayLengthOffset(INJECTED_VMCONFIG), length, LocationIdentity.init());
 552         /*
 553          * store hub last as the concurrent garbage collectors assume length is valid if hub field
 554          * is not null
 555          */
 556         initializeObjectHeader(memory, prototypeMarkWord, hub);
 557         if (fillContents) {
 558             zeroMemory(allocationSize, memory, false, headerSize, maybeUnroll, counters);
 559         } else if (REPLACEMENTS_ASSERTIONS_ENABLED) {
 560             fillWithGarbage(allocationSize, memory, false, headerSize, maybeUnroll, counters);
 561         }
 562         MembarNode.memoryBarrier(MemoryBarriers.STORE_STORE, LocationIdentity.init());
 563         return memory.toObjectNonNull();
 564     }
 565 
 566     static class Counters {
 567         Counters(SnippetCounter.Group.Factory factory) {
 568             Group newInstance = factory.createSnippetCounterGroup("NewInstance");
 569             Group newArray = factory.createSnippetCounterGroup("NewArray");




   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  */
  23 
  24 
  25 package org.graalvm.compiler.hotspot.replacements;
  26 
  27 import static jdk.vm.ci.meta.DeoptimizationAction.None;
  28 import static jdk.vm.ci.meta.DeoptimizationReason.RuntimeConstraint;
  29 import static org.graalvm.compiler.core.common.GraalOptions.GeneratePIC;
  30 import static org.graalvm.compiler.core.common.calc.UnsignedMath.belowThan;
  31 import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfigBase.INJECTED_VMCONFIG;
  32 import static org.graalvm.compiler.hotspot.HotSpotBackend.NEW_ARRAY;
  33 import static org.graalvm.compiler.hotspot.HotSpotBackend.NEW_ARRAY_OR_NULL;
  34 import static org.graalvm.compiler.hotspot.HotSpotBackend.NEW_INSTANCE;
  35 import static org.graalvm.compiler.hotspot.HotSpotBackend.NEW_INSTANCE_OR_NULL;
  36 import static org.graalvm.compiler.hotspot.HotSpotBackend.NEW_MULTI_ARRAY;
  37 import static org.graalvm.compiler.hotspot.HotSpotBackend.NEW_MULTI_ARRAY_OR_NULL;
  38 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.CLASS_ARRAY_KLASS_LOCATION;
  39 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.HUB_WRITE_LOCATION;
  40 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.MARK_WORD_LOCATION;
  41 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.PROTOTYPE_MARK_WORD_LOCATION;
  42 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_END_LOCATION;
  43 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_TOP_LOCATION;
  44 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayAllocationSize;
  45 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayKlassOffset;
  46 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayLengthOffset;
  47 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.config;
  48 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.initializeObjectHeader;
  49 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.instanceHeaderSize;
  50 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.isInstanceKlassFullyInitialized;
  51 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperHeaderSizeMask;
  52 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperHeaderSizeShift;
  53 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperLog2ElementSizeMask;
  54 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperLog2ElementSizeShift;
  55 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.loadKlassFromObject;
  56 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.prototypeMarkWordOffset;
  57 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readLayoutHelper;


  61 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.useBiasedLocking;
  62 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.useTLAB;
  63 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.verifyOop;
  64 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.writeTlabTop;
  65 import static org.graalvm.compiler.hotspot.replacements.HotspotSnippetsOptions.ProfileAllocations;
  66 import static org.graalvm.compiler.hotspot.replacements.HotspotSnippetsOptions.ProfileAllocationsContext;
  67 import static org.graalvm.compiler.nodes.PiArrayNode.piArrayCastToSnippetReplaceeStamp;
  68 import static org.graalvm.compiler.nodes.PiNode.piCastToSnippetReplaceeStamp;
  69 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.FAST_PATH_PROBABILITY;
  70 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.FREQUENT_PROBABILITY;
  71 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.SLOW_PATH_PROBABILITY;
  72 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.probability;
  73 import static org.graalvm.compiler.replacements.ReplacementsUtil.REPLACEMENTS_ASSERTIONS_ENABLED;
  74 import static org.graalvm.compiler.replacements.ReplacementsUtil.runtimeAssert;
  75 import static org.graalvm.compiler.replacements.ReplacementsUtil.staticAssert;
  76 import static org.graalvm.compiler.replacements.SnippetTemplate.DEFAULT_REPLACER;
  77 import static org.graalvm.compiler.replacements.nodes.CStringConstant.cstring;
  78 import static org.graalvm.compiler.replacements.nodes.ExplodeLoopNode.explodeLoop;
  79 
  80 import org.graalvm.compiler.api.replacements.Fold;
  81 import org.graalvm.compiler.api.replacements.Fold.InjectedParameter;
  82 import org.graalvm.compiler.api.replacements.Snippet;
  83 import org.graalvm.compiler.api.replacements.Snippet.ConstantParameter;
  84 import org.graalvm.compiler.api.replacements.Snippet.VarargsParameter;
  85 import org.graalvm.compiler.core.common.spi.ForeignCallDescriptor;
  86 import org.graalvm.compiler.core.common.type.StampFactory;
  87 import org.graalvm.compiler.debug.DebugHandlersFactory;
  88 import org.graalvm.compiler.debug.GraalError;
  89 import org.graalvm.compiler.graph.Node.ConstantNodeParameter;
  90 import org.graalvm.compiler.graph.Node.NodeIntrinsic;
  91 import org.graalvm.compiler.hotspot.GraalHotSpotVMConfig;

  92 import org.graalvm.compiler.hotspot.meta.HotSpotProviders;
  93 import org.graalvm.compiler.hotspot.meta.HotSpotRegistersProvider;
  94 import org.graalvm.compiler.hotspot.nodes.DimensionsNode;
  95 import org.graalvm.compiler.hotspot.nodes.aot.LoadConstantIndirectlyFixedNode;
  96 import org.graalvm.compiler.hotspot.nodes.aot.LoadConstantIndirectlyNode;
  97 import org.graalvm.compiler.hotspot.nodes.type.KlassPointerStamp;
  98 import org.graalvm.compiler.hotspot.word.KlassPointer;
  99 import org.graalvm.compiler.nodes.ConstantNode;
 100 import org.graalvm.compiler.nodes.DeoptimizeNode;
 101 import org.graalvm.compiler.nodes.PiNode;
 102 import org.graalvm.compiler.nodes.PrefetchAllocateNode;
 103 import org.graalvm.compiler.nodes.SnippetAnchorNode;
 104 import org.graalvm.compiler.nodes.StructuredGraph;
 105 import org.graalvm.compiler.nodes.ValueNode;
 106 import org.graalvm.compiler.nodes.debug.DynamicCounterNode;
 107 import org.graalvm.compiler.nodes.debug.VerifyHeapNode;
 108 import org.graalvm.compiler.nodes.extended.ForeignCallNode;
 109 import org.graalvm.compiler.nodes.extended.MembarNode;
 110 import org.graalvm.compiler.nodes.java.DynamicNewArrayNode;
 111 import org.graalvm.compiler.nodes.java.DynamicNewInstanceNode;


 117 import org.graalvm.compiler.nodes.util.GraphUtil;
 118 import org.graalvm.compiler.options.OptionValues;
 119 import org.graalvm.compiler.replacements.ReplacementsUtil;
 120 import org.graalvm.compiler.replacements.SnippetCounter;
 121 import org.graalvm.compiler.replacements.SnippetCounter.Group;
 122 import org.graalvm.compiler.replacements.SnippetTemplate;
 123 import org.graalvm.compiler.replacements.SnippetTemplate.AbstractTemplates;
 124 import org.graalvm.compiler.replacements.SnippetTemplate.Arguments;
 125 import org.graalvm.compiler.replacements.SnippetTemplate.SnippetInfo;
 126 import org.graalvm.compiler.replacements.Snippets;
 127 import org.graalvm.compiler.replacements.nodes.ExplodeLoopNode;
 128 import org.graalvm.compiler.word.Word;
 129 import jdk.internal.vm.compiler.word.LocationIdentity;
 130 import jdk.internal.vm.compiler.word.WordFactory;
 131 
 132 import jdk.vm.ci.code.CodeUtil;
 133 import jdk.vm.ci.code.MemoryBarriers;
 134 import jdk.vm.ci.code.Register;
 135 import jdk.vm.ci.code.TargetDescription;
 136 import jdk.vm.ci.hotspot.HotSpotResolvedObjectType;


 137 import jdk.vm.ci.meta.JavaKind;
 138 import jdk.vm.ci.meta.ResolvedJavaType;
 139 
 140 /**
 141  * Snippets used for implementing NEW, ANEWARRAY and NEWARRAY.
 142  */
 143 public class NewObjectSnippets implements Snippets {
 144 
 145     enum ProfileContext {
 146         AllocatingMethod,
 147         InstanceOrArray,
 148         AllocatedType,
 149         AllocatedTypesInMethod,
 150         Total
 151     }
 152 
 153     @Fold
 154     static String createName(String path, String typeContext, OptionValues options) {
 155         switch (ProfileAllocationsContext.getValue(options)) {
 156             case AllocatingMethod:


 209                     @ConstantParameter Register threadRegister, @ConstantParameter boolean constantSize, @ConstantParameter String typeContext, @ConstantParameter OptionValues options,
 210                     @ConstantParameter Counters counters) {
 211         return piCastToSnippetReplaceeStamp(allocateInstanceHelper(size, hub, prototypeMarkWord, fillContents, threadRegister, constantSize, typeContext, options, counters));
 212     }
 213 
 214     public static Object allocateInstanceHelper(int size, KlassPointer hub, Word prototypeMarkWord, boolean fillContents,
 215                     Register threadRegister, boolean constantSize, String typeContext, OptionValues options, Counters counters) {
 216         Object result;
 217         Word thread = registerAsWord(threadRegister);
 218         Word top = readTlabTop(thread);
 219         Word end = readTlabEnd(thread);
 220         Word newTop = top.add(size);
 221         if (useTLAB(INJECTED_VMCONFIG) && probability(FAST_PATH_PROBABILITY, newTop.belowOrEqual(end))) {
 222             writeTlabTop(thread, newTop);
 223             emitPrefetchAllocate(newTop, false);
 224             result = formatObject(hub, size, top, prototypeMarkWord, fillContents, constantSize, counters);
 225         } else {
 226             if (counters != null && counters.stub != null) {
 227                 counters.stub.inc();
 228             }
 229             result = newInstanceStub(hub);
 230         }
 231         profileAllocation("instance", size, typeContext, options);
 232         return verifyOop(result);
 233     }
 234 
 235     public static Object newInstanceStub(KlassPointer hub) {
 236         if (useNullAllocationStubs(INJECTED_VMCONFIG)) {
 237             return nonNullOrDeopt(newInstanceOrNull(NEW_INSTANCE_OR_NULL, hub));
 238         } else {
 239             return newInstance(NEW_INSTANCE, hub);
 240         }
 241     }
 242 
 243     @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = true)
 244     private static native Object newInstance(@ConstantNodeParameter ForeignCallDescriptor descriptor, KlassPointer hub);
 245 
 246     @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = false)
 247     private static native Object newInstanceOrNull(@ConstantNodeParameter ForeignCallDescriptor descriptor, KlassPointer hub);
 248 
 249     @Snippet
 250     public static Object allocateInstancePIC(@ConstantParameter int size, KlassPointer hub, Word prototypeMarkWord, @ConstantParameter boolean fillContents,
 251                     @ConstantParameter Register threadRegister, @ConstantParameter boolean constantSize, @ConstantParameter String typeContext, @ConstantParameter OptionValues options,
 252                     @ConstantParameter Counters counters) {
 253         // Klass must be initialized by the time the first instance is allocated, therefore we can
 254         // just load it from the corresponding cell and avoid the resolution check. We have to use a
 255         // fixed load though, to prevent it from floating above the initialization.
 256         KlassPointer picHub = LoadConstantIndirectlyFixedNode.loadKlass(hub);
 257         return piCastToSnippetReplaceeStamp(allocateInstanceHelper(size, picHub, prototypeMarkWord, fillContents, threadRegister, constantSize, typeContext, options, counters));
 258     }
 259 
 260     @Snippet
 261     public static Object allocateInstanceDynamic(Class<?> type, Class<?> classClass, @ConstantParameter boolean fillContents, @ConstantParameter Register threadRegister,
 262                     @ConstantParameter OptionValues options, @ConstantParameter Counters counters) {
 263         if (probability(SLOW_PATH_PROBABILITY, type == null)) {
 264             DeoptimizeNode.deopt(None, RuntimeConstraint);
 265         }
 266         Class<?> nonNullType = PiNode.piCastNonNullClass(type, SnippetAnchorNode.anchor());
 267 
 268         if (probability(SLOW_PATH_PROBABILITY, DynamicNewInstanceNode.throwsInstantiationException(type, classClass))) {
 269             DeoptimizeNode.deopt(None, RuntimeConstraint);
 270         }
 271 
 272         return PiNode.piCastToSnippetReplaceeStamp(allocateInstanceDynamicHelper(fillContents, threadRegister, options, counters, nonNullType));
 273     }
 274 
 275     private static Object allocateInstanceDynamicHelper(boolean fillContents, Register threadRegister, OptionValues options, Counters counters, Class<?> nonNullType) {
 276         KlassPointer hub = ClassGetHubNode.readClass(nonNullType);
 277         if (probability(FAST_PATH_PROBABILITY, !hub.isNull())) {
 278             KlassPointer nonNullHub = ClassGetHubNode.piCastNonNull(hub, SnippetAnchorNode.anchor());
 279 
 280             if (probability(FAST_PATH_PROBABILITY, isInstanceKlassFullyInitialized(nonNullHub))) {
 281                 int layoutHelper = readLayoutHelper(nonNullHub);
 282                 /*
 283                  * src/share/vm/oops/klass.hpp: For instances, layout helper is a positive number,
 284                  * the instance size. This size is already passed through align_object_size and
 285                  * scaled to bytes. The low order bit is set if instances of this class cannot be
 286                  * allocated using the fastpath.
 287                  */
 288                 if (probability(FAST_PATH_PROBABILITY, (layoutHelper & 1) == 0)) {
 289                     Word prototypeMarkWord = nonNullHub.readWord(prototypeMarkWordOffset(INJECTED_VMCONFIG), PROTOTYPE_MARK_WORD_LOCATION);
 290                     /*
 291                      * FIXME(je,ds): we should actually pass typeContext instead of "" but late
 292                      * binding of parameters is not yet supported by the GraphBuilderPlugin system.
 293                      */
 294                     return allocateInstanceHelper(layoutHelper, nonNullHub, prototypeMarkWord, fillContents, threadRegister, false, "", options, counters);
 295                 }
 296             } else {
 297                 DeoptimizeNode.deopt(None, RuntimeConstraint);
 298             }
 299         }
 300         DeoptimizeNode.deopt(None, RuntimeConstraint);
 301         return null;
 302     }
 303 
 304     /**
 305      * Maximum array length for which fast path allocation is used.
 306      */
 307     public static final int MAX_ARRAY_FAST_PATH_ALLOCATION_LENGTH = 0x00FFFFFF;
 308 
 309     @Snippet
 310     public static Object allocatePrimitiveArrayPIC(KlassPointer hub, int length, Word prototypeMarkWord, @ConstantParameter int headerSize, @ConstantParameter int log2ElementSize,
 311                     @ConstantParameter boolean fillContents, @ConstantParameter Register threadRegister, @ConstantParameter boolean maybeUnroll, @ConstantParameter String typeContext,
 312                     @ConstantParameter OptionValues options, @ConstantParameter Counters counters) {
 313         // Primitive array types are eagerly pre-resolved. We can use a floating load.
 314         KlassPointer picHub = LoadConstantIndirectlyNode.loadKlass(hub);
 315         return allocateArrayImpl(picHub, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents, threadRegister, maybeUnroll, typeContext, false, options, counters);
 316     }
 317 
 318     @Snippet
 319     public static Object allocateArrayPIC(KlassPointer hub, int length, Word prototypeMarkWord, @ConstantParameter int headerSize, @ConstantParameter int log2ElementSize,
 320                     @ConstantParameter boolean fillContents, @ConstantParameter Register threadRegister, @ConstantParameter boolean maybeUnroll, @ConstantParameter String typeContext,
 321                     @ConstantParameter OptionValues options, @ConstantParameter Counters counters) {
 322         // Array type would be resolved by dominating resolution.
 323         KlassPointer picHub = LoadConstantIndirectlyFixedNode.loadKlass(hub);
 324         return allocateArrayImpl(picHub, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents, threadRegister, maybeUnroll, typeContext, false, options, counters);
 325     }
 326 
 327     @Snippet
 328     public static Object allocateArray(KlassPointer hub,
 329                     int length,
 330                     Word prototypeMarkWord,
 331                     @ConstantParameter int headerSize,
 332                     @ConstantParameter int log2ElementSize,
 333                     @ConstantParameter boolean fillContents,
 334                     @ConstantParameter Register threadRegister,
 335                     @ConstantParameter boolean maybeUnroll,
 336                     @ConstantParameter String typeContext,
 337                     @ConstantParameter OptionValues options,
 338                     @ConstantParameter Counters counters) {
 339         Object result = allocateArrayImpl(hub, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents, threadRegister, maybeUnroll, typeContext, false, options, counters);
 340         return piArrayCastToSnippetReplaceeStamp(verifyOop(result), length);
 341     }
 342 
 343     /**
 344      * When allocating on the slow path, determines whether to use a version of the runtime call
 345      * that returns {@code null} on a failed allocation instead of raising an OutOfMemoryError.
 346      */
 347     @Fold
 348     static boolean useNullAllocationStubs(@InjectedParameter GraalHotSpotVMConfig config) {
 349         return config.areNullAllocationStubsAvailable();
 350     }
 351 
 352     private static Object allocateArrayImpl(KlassPointer hub, int length, Word prototypeMarkWord, int headerSize, int log2ElementSize, boolean fillContents, Register threadRegister,
 353                     boolean maybeUnroll, String typeContext, boolean skipNegativeCheck, OptionValues options, Counters counters) {
 354         Object result;
 355         int allocationSize = arrayAllocationSize(length, headerSize, log2ElementSize);
 356         Word thread = registerAsWord(threadRegister);
 357         Word top = readTlabTop(thread);
 358         Word end = readTlabEnd(thread);
 359         Word newTop = top.add(allocationSize);
 360         if (probability(FREQUENT_PROBABILITY, skipNegativeCheck || belowThan(length, MAX_ARRAY_FAST_PATH_ALLOCATION_LENGTH)) && useTLAB(INJECTED_VMCONFIG) &&
 361                         probability(FAST_PATH_PROBABILITY, newTop.belowOrEqual(end))) {
 362             writeTlabTop(thread, newTop);
 363             emitPrefetchAllocate(newTop, true);
 364             if (counters != null && counters.arrayLoopInit != null) {
 365                 counters.arrayLoopInit.inc();
 366             }
 367             result = formatArray(hub, allocationSize, length, headerSize, top, prototypeMarkWord, fillContents, maybeUnroll, counters);
 368         } else {
 369             result = newArrayStub(hub, length);
 370         }
 371         profileAllocation("array", allocationSize, typeContext, options);
 372         return result;
 373     }
 374 
 375     public static Object newArrayStub(KlassPointer hub, int length) {
 376         if (useNullAllocationStubs(INJECTED_VMCONFIG)) {
 377             return nonNullOrDeopt(newArrayOrNull(NEW_ARRAY_OR_NULL, hub, length));
 378         } else {
 379             return newArray(NEW_ARRAY, hub, length);
 380         }
 381     }
 382 
 383     @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = true)
 384     private static native Object newArray(@ConstantNodeParameter ForeignCallDescriptor descriptor, KlassPointer hub, int length);
 385 
 386     @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = false)
 387     private static native Object newArrayOrNull(@ConstantNodeParameter ForeignCallDescriptor descriptor, KlassPointer hub, int length);
 388 
 389     /**
 390      * Deoptimizes if {@code obj == null} otherwise returns {@code obj}.
 391      */
 392     private static Object nonNullOrDeopt(Object obj) {
 393         if (obj == null) {
 394             DeoptimizeNode.deopt(None, RuntimeConstraint);
 395         }
 396         return obj;
 397     }
 398 
 399     @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = true)
 400     public static native Object dynamicNewInstance(@ConstantNodeParameter ForeignCallDescriptor descriptor, Class<?> elementType);
 401 
 402     @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = false)
 403     public static native Object dynamicNewInstanceOrNull(@ConstantNodeParameter ForeignCallDescriptor descriptor, Class<?> elementType);
 404 
 405     @Snippet
 406     public static Object allocateArrayDynamic(Class<?> elementType, Class<?> voidClass, int length, @ConstantParameter boolean fillContents, @ConstantParameter Register threadRegister,
 407                     @ConstantParameter JavaKind knownElementKind, @ConstantParameter int knownLayoutHelper, Word prototypeMarkWord, @ConstantParameter OptionValues options,
 408                     @ConstantParameter Counters counters) {
 409         Object result = allocateArrayDynamicImpl(elementType, voidClass, length, fillContents, threadRegister, knownElementKind, knownLayoutHelper, prototypeMarkWord, options, counters);
 410         return result;
 411     }
 412 
 413     private static Object allocateArrayDynamicImpl(Class<?> elementType, Class<?> voidClass, int length, boolean fillContents, Register threadRegister, JavaKind knownElementKind,
 414                     int knownLayoutHelper, Word prototypeMarkWord, OptionValues options, Counters counters) {
 415         /*
 416          * We only need the dynamic check for void when we have no static information from
 417          * knownElementKind.
 418          */
 419         staticAssert(knownElementKind != JavaKind.Void, "unsupported knownElementKind");
 420         if (knownElementKind == JavaKind.Illegal && probability(SLOW_PATH_PROBABILITY, elementType == null || DynamicNewArrayNode.throwsIllegalArgumentException(elementType, voidClass))) {
 421             DeoptimizeNode.deopt(None, RuntimeConstraint);
 422         }
 423 
 424         KlassPointer klass = loadKlassFromObject(elementType, arrayKlassOffset(INJECTED_VMCONFIG), CLASS_ARRAY_KLASS_LOCATION);
 425         if (klass.isNull()) {
 426             DeoptimizeNode.deopt(None, RuntimeConstraint);
 427         }
 428         KlassPointer nonNullKlass = ClassGetHubNode.piCastNonNull(klass, SnippetAnchorNode.anchor());
 429 
 430         if (length < 0) {
 431             DeoptimizeNode.deopt(None, RuntimeConstraint);
 432         }
 433         int layoutHelper;
 434         if (knownElementKind == JavaKind.Illegal) {
 435             layoutHelper = readLayoutHelper(nonNullKlass);
 436         } else {
 437             runtimeAssert(knownLayoutHelper == readLayoutHelper(nonNullKlass), "layout mismatch");
 438             layoutHelper = knownLayoutHelper;
 439         }
 440         //@formatter:off
 441         // from src/share/vm/oops/klass.hpp:
 442         //
 443         // For arrays, layout helper is a negative number, containing four
 444         // distinct bytes, as follows:
 445         //    MSB:[tag, hsz, ebt, log2(esz)]:LSB
 446         // where:
 447         //    tag is 0x80 if the elements are oops, 0xC0 if non-oops
 448         //    hsz is array header size in bytes (i.e., offset of first element)
 449         //    ebt is the BasicType of the elements
 450         //    esz is the element size in bytes
 451         //@formatter:on
 452 
 453         int headerSize = (layoutHelper >> layoutHelperHeaderSizeShift(INJECTED_VMCONFIG)) & layoutHelperHeaderSizeMask(INJECTED_VMCONFIG);
 454         int log2ElementSize = (layoutHelper >> layoutHelperLog2ElementSizeShift(INJECTED_VMCONFIG)) & layoutHelperLog2ElementSizeMask(INJECTED_VMCONFIG);
 455 
 456         Object result = allocateArrayImpl(nonNullKlass, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents, threadRegister, false, "dynamic type", true, options, counters);
 457         return piArrayCastToSnippetReplaceeStamp(verifyOop(result), length);
 458     }
 459 
 460     /**
 461      * Calls the runtime stub for implementing MULTIANEWARRAY.
 462      */
 463     @Snippet
 464     private static Object newmultiarray(KlassPointer hub, @ConstantParameter int rank, @VarargsParameter int[] dimensions) {
 465         Word dims = DimensionsNode.allocaDimsArray(rank);
 466         ExplodeLoopNode.explodeLoop();
 467         for (int i = 0; i < rank; i++) {
 468             dims.writeInt(i * 4, dimensions[i], LocationIdentity.init());
 469         }
 470         return newMultiArrayStub(hub, rank, dims);
 471     }
 472 
 473     private static Object newMultiArrayStub(KlassPointer hub, int rank, Word dims) {
 474         if (useNullAllocationStubs(INJECTED_VMCONFIG)) {
 475             return nonNullOrDeopt(newMultiArrayOrNull(NEW_MULTI_ARRAY_OR_NULL, hub, rank, dims));
 476         } else {
 477             return newMultiArray(NEW_MULTI_ARRAY, hub, rank, dims);
 478         }
 479     }
 480 
 481     @Snippet
 482     private static Object newmultiarrayPIC(KlassPointer hub, @ConstantParameter int rank, @VarargsParameter int[] dimensions) {
 483         // Array type would be resolved by dominating resolution.
 484         KlassPointer picHub = LoadConstantIndirectlyFixedNode.loadKlass(hub);
 485         return newmultiarray(picHub, rank, dimensions);
 486     }
 487 
 488     @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = true)
 489     private static native Object newMultiArray(@ConstantNodeParameter ForeignCallDescriptor descriptor, KlassPointer hub, int rank, Word dims);
 490 
 491     @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = false)
 492     private static native Object newMultiArrayOrNull(@ConstantNodeParameter ForeignCallDescriptor descriptor, KlassPointer hub, int rank, Word dims);
 493 
 494     /**
 495      * Maximum number of long stores to emit when zeroing an object with a constant size. Larger
 496      * objects have their bodies initialized in a loop.
 497      */
 498     private static final int MAX_UNROLLED_OBJECT_ZEROING_STORES = 8;
 499 
 500     /**
 501      * Zero uninitialized memory in a newly allocated object, unrolling as necessary and ensuring
 502      * that stores are aligned.
 503      *
 504      * @param size number of bytes to zero
 505      * @param memory beginning of object which is being zeroed
 506      * @param constantSize is {@code size} known to be constant in the snippet
 507      * @param startOffset offset to begin zeroing. May not be word aligned.
 508      * @param manualUnroll maximally unroll zeroing
 509      */
 510     private static void zeroMemory(int size, Word memory, boolean constantSize, int startOffset, boolean manualUnroll, Counters counters) {
 511         fillMemory(0, size, memory, constantSize, startOffset, manualUnroll, counters);
 512     }


 552                 memory.initializeLong(off, value, LocationIdentity.init());
 553             }
 554         }
 555     }
 556 
 557     /**
 558      * Fill uninitialized memory with garbage value in a newly allocated object, unrolling as
 559      * necessary and ensuring that stores are aligned.
 560      *
 561      * @param size number of bytes to zero
 562      * @param memory beginning of object which is being zeroed
 563      * @param constantSize is {@code  size} known to be constant in the snippet
 564      * @param startOffset offset to begin zeroing. May not be word aligned.
 565      * @param manualUnroll maximally unroll zeroing
 566      */
 567     private static void fillWithGarbage(int size, Word memory, boolean constantSize, int startOffset, boolean manualUnroll, Counters counters) {
 568         fillMemory(0xfefefefefefefefeL, size, memory, constantSize, startOffset, manualUnroll, counters);
 569     }
 570 
 571     /**








 572      * Formats some allocated memory with an object header and zeroes out the rest.
 573      */
 574     private static Object formatObject(KlassPointer hub, int size, Word memory, Word compileTimePrototypeMarkWord, boolean fillContents, boolean constantSize, Counters counters) {
 575         Word prototypeMarkWord = useBiasedLocking(INJECTED_VMCONFIG) ? hub.readWord(prototypeMarkWordOffset(INJECTED_VMCONFIG), PROTOTYPE_MARK_WORD_LOCATION) : compileTimePrototypeMarkWord;
 576         initializeObjectHeader(memory, prototypeMarkWord, hub);
 577         if (fillContents) {
 578             zeroMemory(size, memory, constantSize, instanceHeaderSize(INJECTED_VMCONFIG), false, counters);
 579         } else if (REPLACEMENTS_ASSERTIONS_ENABLED) {
 580             fillWithGarbage(size, memory, constantSize, instanceHeaderSize(INJECTED_VMCONFIG), false, counters);
 581         }
 582         MembarNode.memoryBarrier(MemoryBarriers.STORE_STORE, LocationIdentity.init());
 583         return memory.toObjectNonNull();
 584     }
 585 
 586     @Snippet
 587     private static void verifyHeap(@ConstantParameter Register threadRegister) {
 588         Word thread = registerAsWord(threadRegister);
 589         Word topValue = readTlabTop(thread);
 590         if (!topValue.equal(WordFactory.zero())) {
 591             Word topValueContents = topValue.readWord(0, MARK_WORD_LOCATION);
 592             if (topValueContents.equal(WordFactory.zero())) {
 593                 AssertionSnippets.vmMessageC(AssertionSnippets.ASSERTION_VM_MESSAGE_C, true, cstring("overzeroing of TLAB detected"), 0L, 0L, 0L);
 594             }
 595         }
 596     }
 597 
 598     /**
 599      * Formats some allocated memory with an object header and zeroes out the rest.
 600      */
 601     private static Object formatArray(KlassPointer hub, int allocationSize, int length, int headerSize, Word memory, Word prototypeMarkWord, boolean fillContents, boolean maybeUnroll,
 602                     Counters counters) {
 603         memory.writeInt(arrayLengthOffset(INJECTED_VMCONFIG), length, LocationIdentity.init());
 604         /*
 605          * store hub last as the concurrent garbage collectors assume length is valid if hub field
 606          * is not null
 607          */
 608         initializeObjectHeader(memory, prototypeMarkWord, hub);
 609         if (fillContents) {
 610             zeroMemory(allocationSize, memory, false, headerSize, maybeUnroll, counters);
 611         } else if (REPLACEMENTS_ASSERTIONS_ENABLED) {
 612             fillWithGarbage(allocationSize, memory, false, headerSize, maybeUnroll, counters);
 613         }
 614         MembarNode.memoryBarrier(MemoryBarriers.STORE_STORE, LocationIdentity.init());
 615         return memory.toObjectNonNull();
 616     }
 617 
 618     static class Counters {
 619         Counters(SnippetCounter.Group.Factory factory) {
 620             Group newInstance = factory.createSnippetCounterGroup("NewInstance");
 621             Group newArray = factory.createSnippetCounterGroup("NewArray");


< prev index next >