< prev index next >

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/NewObjectSnippets.java

Print this page




 192             DynamicCounterNode.counter("number of allocations", name, 1, context);
 193         }
 194     }
 195 
 196     public static void emitPrefetchAllocate(Word address, boolean isArray) {
 197         if (allocatePrefetchStyle(INJECTED_VMCONFIG) > 0) {
 198             // Insert a prefetch for each allocation only on the fast-path
 199             // Generate several prefetch instructions.
 200             int lines = isArray ? allocatePrefetchLines(INJECTED_VMCONFIG) : allocateInstancePrefetchLines(INJECTED_VMCONFIG);
 201             int stepSize = allocatePrefetchStepSize(INJECTED_VMCONFIG);
 202             int distance = allocatePrefetchDistance(INJECTED_VMCONFIG);
 203             ExplodeLoopNode.explodeLoop();
 204             for (int i = 0; i < lines; i++) {
 205                 PrefetchAllocateNode.prefetch(OffsetAddressNode.address(address, distance));
 206                 distance += stepSize;
 207             }
 208         }
 209     }
 210 
 211     @Snippet
 212     public static Object allocateInstance(@ConstantParameter int size, KlassPointer hub, Word prototypeMarkWord, @ConstantParameter boolean fillContents,
 213                     @ConstantParameter Register threadRegister, @ConstantParameter boolean constantSize, @ConstantParameter String typeContext,
 214                     @ConstantParameter Counters counters) {
 215         return piCastToSnippetReplaceeStamp(allocateInstanceHelper(size, hub, prototypeMarkWord, fillContents, threadRegister, constantSize, typeContext, counters));
 216     }
 217 
 218     public static Object allocateInstanceHelper(int size, KlassPointer hub, Word prototypeMarkWord, boolean fillContents,
 219                     Register threadRegister, boolean constantSize, String typeContext, Counters counters) {
 220         Object result;
 221         Word thread = registerAsWord(threadRegister);
 222         Word top = readTlabTop(thread);
 223         Word end = readTlabEnd(thread);
 224         Word newTop = top.add(size);
 225         if (useTLAB(INJECTED_VMCONFIG) && probability(FAST_PATH_PROBABILITY, newTop.belowOrEqual(end))) {
 226             writeTlabTop(thread, newTop);
 227             emitPrefetchAllocate(newTop, false);
 228             result = formatObject(hub, size, top, prototypeMarkWord, fillContents, constantSize, counters);
 229         } else {
 230             Counters theCounters = counters;
 231             if (theCounters != null && theCounters.stub != null) {
 232                 theCounters.stub.inc();
 233             }
 234             result = newInstanceStub(hub);
 235         }
 236         profileAllocation("instance", size, typeContext);
 237         return verifyOop(result);
 238     }
 239 
 240     public static Object newInstanceStub(KlassPointer hub) {
 241         if (useNullAllocationStubs(INJECTED_VMCONFIG)) {
 242             return nonNullOrDeopt(newInstanceOrNull(NEW_INSTANCE_OR_NULL, hub));
 243         } else {
 244             return newInstance(NEW_INSTANCE, hub);
 245         }
 246     }
 247 
 248     @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = true)
 249     private static native Object newInstance(@ConstantNodeParameter ForeignCallDescriptor descriptor, KlassPointer hub);
 250 
 251     @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = false)
 252     private static native Object newInstanceOrNull(@ConstantNodeParameter ForeignCallDescriptor descriptor, KlassPointer hub);
 253 
 254     @Snippet
 255     public static Object allocateInstancePIC(@ConstantParameter int size, KlassPointer hub, Word prototypeMarkWord, @ConstantParameter boolean fillContents,
 256                     @ConstantParameter Register threadRegister, @ConstantParameter boolean constantSize, @ConstantParameter String typeContext,
 257                     @ConstantParameter Counters counters) {
 258         // Klass must be initialized by the time the first instance is allocated, therefore we can
 259         // just load it from the corresponding cell and avoid the resolution check. We have to use a
 260         // fixed load though, to prevent it from floating above the initialization.
 261         KlassPointer picHub = LoadConstantIndirectlyFixedNode.loadKlass(hub);
 262         return piCastToSnippetReplaceeStamp(allocateInstanceHelper(size, picHub, prototypeMarkWord, fillContents, threadRegister, constantSize, typeContext, counters));
 263     }
 264 
 265     @Snippet
 266     public static Object allocateInstanceDynamic(Class<?> type, Class<?> classClass, @ConstantParameter boolean fillContents, @ConstantParameter Register threadRegister,
 267                     @ConstantParameter Counters counters) {
 268         if (probability(SLOW_PATH_PROBABILITY, type == null)) {
 269             DeoptimizeNode.deopt(None, RuntimeConstraint);
 270         }
 271         Class<?> nonNullType = PiNode.piCastNonNullClass(type, SnippetAnchorNode.anchor());
 272 
 273         if (probability(SLOW_PATH_PROBABILITY, DynamicNewInstanceNode.throwsInstantiationException(type, classClass))) {
 274             DeoptimizeNode.deopt(None, RuntimeConstraint);
 275         }


 299                     return allocateInstanceHelper(layoutHelper, nonNullHub, prototypeMarkWord, fillContents, threadRegister, false, "", counters);
 300                 }
 301             } else {
 302                 DeoptimizeNode.deopt(None, RuntimeConstraint);
 303             }
 304         }
 305         return dynamicNewInstanceStub(type);
 306     }
 307 
 308     /**
 309      * Maximum array length for which fast path allocation is used.
 310      */
 311     public static final int MAX_ARRAY_FAST_PATH_ALLOCATION_LENGTH = 0x00FFFFFF;
 312 
 313     @Snippet
 314     public static Object allocatePrimitiveArrayPIC(KlassPointer hub, int length, Word prototypeMarkWord, @ConstantParameter int headerSize, @ConstantParameter int log2ElementSize,
 315                     @ConstantParameter boolean fillContents, @ConstantParameter Register threadRegister, @ConstantParameter boolean maybeUnroll, @ConstantParameter String typeContext,
 316                     @ConstantParameter Counters counters) {
 317         // Primitive array types are eagerly pre-resolved. We can use a floating load.
 318         KlassPointer picHub = LoadConstantIndirectlyNode.loadKlass(hub);
 319         return allocateArrayImpl(picHub, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents, threadRegister, maybeUnroll, typeContext, false, counters);
 320     }
 321 
 322     @Snippet
 323     public static Object allocateArrayPIC(KlassPointer hub, int length, Word prototypeMarkWord, @ConstantParameter int headerSize, @ConstantParameter int log2ElementSize,
 324                     @ConstantParameter boolean fillContents, @ConstantParameter Register threadRegister, @ConstantParameter boolean maybeUnroll, @ConstantParameter String typeContext,
 325                     @ConstantParameter Counters counters) {
 326         // Array type would be resolved by dominating resolution.
 327         KlassPointer picHub = LoadConstantIndirectlyFixedNode.loadKlass(hub);
 328         return allocateArrayImpl(picHub, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents, threadRegister, maybeUnroll, typeContext, false, counters);
 329     }
 330 
 331     @Snippet
 332     public static Object allocateArray(KlassPointer hub,
 333                     int length,
 334                     Word prototypeMarkWord,
 335                     @ConstantParameter int headerSize,
 336                     @ConstantParameter int log2ElementSize,
 337                     @ConstantParameter boolean fillContents,
 338                     @ConstantParameter Register threadRegister,
 339                     @ConstantParameter boolean maybeUnroll,
 340                     @ConstantParameter String typeContext,
 341                     @ConstantParameter Counters counters) {
 342         Object result = allocateArrayImpl(hub,
 343                         length,
 344                         prototypeMarkWord,
 345                         headerSize,
 346                         log2ElementSize,
 347                         fillContents,
 348                         threadRegister,
 349                         maybeUnroll,
 350                         typeContext,
 351                         false,
 352 
 353                         counters);
 354         return piArrayCastToSnippetReplaceeStamp(verifyOop(result), length);
 355     }
 356 
 357     /**
 358      * When allocating on the slow path, determines whether to use a version of the runtime call
 359      * that returns {@code null} on a failed allocation instead of raising an OutOfMemoryError.
 360      */
 361     @Fold
 362     static boolean useNullAllocationStubs(@InjectedParameter GraalHotSpotVMConfig config) {
 363         return config.areNullAllocationStubsAvailable();
 364     }
 365 
 366     private static Object allocateArrayImpl(KlassPointer hub, int length, Word prototypeMarkWord, int headerSize, int log2ElementSize, boolean fillContents, Register threadRegister,
 367                     boolean maybeUnroll, String typeContext, boolean skipNegativeCheck, Counters counters) {
 368         Object result;
 369         int allocationSize = arrayAllocationSize(length, headerSize, log2ElementSize);
 370         Word thread = registerAsWord(threadRegister);
 371         Word top = readTlabTop(thread);
 372         Word end = readTlabEnd(thread);
 373         Word newTop = top.add(allocationSize);
 374         if (probability(FREQUENT_PROBABILITY, skipNegativeCheck || belowThan(length, MAX_ARRAY_FAST_PATH_ALLOCATION_LENGTH)) && useTLAB(INJECTED_VMCONFIG) &&
 375                         probability(FAST_PATH_PROBABILITY, newTop.belowOrEqual(end))) {
 376             writeTlabTop(thread, newTop);
 377             emitPrefetchAllocate(newTop, true);
 378             Counters theCounters = counters;
 379             if (theCounters != null && theCounters.arrayLoopInit != null) {
 380                 theCounters.arrayLoopInit.inc();
 381             }
 382             result = formatArray(hub, allocationSize, length, headerSize, top, prototypeMarkWord, fillContents, maybeUnroll, counters);
 383         } else {
 384             result = newArrayStub(hub, length);
 385         }
 386         profileAllocation("array", allocationSize, typeContext);
 387         return result;
 388     }
 389 
 390     public static Object newArrayStub(KlassPointer hub, int length) {
 391         if (useNullAllocationStubs(INJECTED_VMCONFIG)) {
 392             return nonNullOrDeopt(newArrayOrNull(NEW_ARRAY_OR_NULL, hub, length));
 393         } else {
 394             return newArray(NEW_ARRAY, hub, length);


 469         } else {
 470             runtimeAssert(knownLayoutHelper == readLayoutHelper(nonNullKlass), "layout mismatch");
 471             layoutHelper = knownLayoutHelper;
 472         }
 473         //@formatter:off
 474         // from src/share/vm/oops/klass.hpp:
 475         //
 476         // For arrays, layout helper is a negative number, containing four
 477         // distinct bytes, as follows:
 478         //    MSB:[tag, hsz, ebt, log2(esz)]:LSB
 479         // where:
 480         //    tag is 0x80 if the elements are oops, 0xC0 if non-oops
 481         //    hsz is array header size in bytes (i.e., offset of first element)
 482         //    ebt is the BasicType of the elements
 483         //    esz is the element size in bytes
 484         //@formatter:on
 485 
 486         int headerSize = (layoutHelper >> layoutHelperHeaderSizeShift(INJECTED_VMCONFIG)) & layoutHelperHeaderSizeMask(INJECTED_VMCONFIG);
 487         int log2ElementSize = (layoutHelper >> layoutHelperLog2ElementSizeShift(INJECTED_VMCONFIG)) & layoutHelperLog2ElementSizeMask(INJECTED_VMCONFIG);
 488 
 489         Object result = allocateArrayImpl(nonNullKlass, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents, threadRegister, false, "dynamic type", true, counters);
 490         return piArrayCastToSnippetReplaceeStamp(verifyOop(result), length);
 491     }
 492 
 493     /**
 494      * Calls the runtime stub for implementing MULTIANEWARRAY.
 495      */
 496     @Snippet
 497     private static Object newmultiarray(KlassPointer hub, @ConstantParameter int rank, @VarargsParameter int[] dimensions) {
 498         Word dims = DimensionsNode.allocaDimsArray(rank);
 499         ExplodeLoopNode.explodeLoop();
 500         for (int i = 0; i < rank; i++) {
 501             dims.writeInt(i * 4, dimensions[i], LocationIdentity.init());
 502         }
 503         return newMultiArrayStub(hub, rank, dims);
 504     }
 505 
 506     private static Object newMultiArrayStub(KlassPointer hub, int rank, Word dims) {
 507         if (useNullAllocationStubs(INJECTED_VMCONFIG)) {
 508             return nonNullOrDeopt(newMultiArrayOrNull(NEW_MULTI_ARRAY_OR_NULL, hub, rank, dims));
 509         } else {


 523 
 524     @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = false)
 525     private static native Object newMultiArrayOrNull(@ConstantNodeParameter ForeignCallDescriptor descriptor, KlassPointer hub, int rank, Word dims);
 526 
 527     /**
 528      * Maximum number of long stores to emit when zeroing an object with a constant size. Larger
 529      * objects have their bodies initialized in a loop.
 530      */
 531     private static final int MAX_UNROLLED_OBJECT_ZEROING_STORES = 8;
 532 
 533     /**
 534      * Zero uninitialized memory in a newly allocated object, unrolling as necessary and ensuring
 535      * that stores are aligned.
 536      *
 537      * @param size number of bytes to zero
 538      * @param memory beginning of object which is being zeroed
 539      * @param constantSize is {@code size} known to be constant in the snippet
 540      * @param startOffset offset to begin zeroing. May not be word aligned.
 541      * @param manualUnroll maximally unroll zeroing
 542      */
 543     private static void zeroMemory(int size, Word memory, boolean constantSize, int startOffset, boolean manualUnroll, Counters counters) {
 544         fillMemory(0, size, memory, constantSize, startOffset, manualUnroll, counters);
 545     }
 546 
 547     private static void fillMemory(long value, int size, Word memory, boolean constantSize, int startOffset, boolean manualUnroll, Counters counters) {
 548         ReplacementsUtil.runtimeAssert((size & 0x7) == 0, "unaligned object size");
 549         int offset = startOffset;
 550         if ((offset & 0x7) != 0) {
 551             memory.writeInt(offset, (int) value, LocationIdentity.init());
 552             offset += 4;
 553         }
 554         ReplacementsUtil.runtimeAssert((offset & 0x7) == 0, "unaligned offset");
 555         Counters theCounters = counters;
 556         if (manualUnroll && ((size - offset) / 8) <= MAX_UNROLLED_OBJECT_ZEROING_STORES) {
 557             ReplacementsUtil.staticAssert(!constantSize, "size shouldn't be constant at instantiation time");
 558             // This case handles arrays of constant length. Instead of having a snippet variant for
 559             // each length, generate a chain of stores of maximum length. Once it's inlined the
 560             // break statement will trim excess stores.
 561             if (theCounters != null && theCounters.instanceSeqInit != null) {
 562                 theCounters.instanceSeqInit.inc();
 563             }
 564 
 565             explodeLoop();
 566             for (int i = 0; i < MAX_UNROLLED_OBJECT_ZEROING_STORES; i++, offset += 8) {
 567                 if (offset == size) {


 581                 if (theCounters != null && theCounters.instanceLoopInit != null) {
 582                     theCounters.instanceLoopInit.inc();
 583                 }
 584             }
 585             for (; off.rawValue() < size; off = off.add(8)) {
 586                 memory.initializeLong(off, value, LocationIdentity.init());
 587             }
 588         }
 589     }
 590 
 591     /**
 592      * Fill uninitialized memory with garbage value in a newly allocated object, unrolling as
 593      * necessary and ensuring that stores are aligned.
 594      *
 595      * @param size number of bytes to zero
 596      * @param memory beginning of object which is being zeroed
 597      * @param constantSize is {@code  size} known to be constant in the snippet
 598      * @param startOffset offset to begin zeroing. May not be word aligned.
 599      * @param manualUnroll maximally unroll zeroing
 600      */
 601     private static void fillWithGarbage(int size, Word memory, boolean constantSize, int startOffset, boolean manualUnroll, Counters counters) {
 602         fillMemory(0xfefefefefefefefeL, size, memory, constantSize, startOffset, manualUnroll, counters);
 603     }
 604 
 605     /**
 606      * Formats some allocated memory with an object header and zeroes out the rest.
 607      */
 608     private static Object formatObject(KlassPointer hub, int size, Word memory, Word compileTimePrototypeMarkWord, boolean fillContents, boolean constantSize, Counters counters) {
 609         Word prototypeMarkWord = useBiasedLocking(INJECTED_VMCONFIG) ? hub.readWord(prototypeMarkWordOffset(INJECTED_VMCONFIG), PROTOTYPE_MARK_WORD_LOCATION) : compileTimePrototypeMarkWord;
 610         initializeObjectHeader(memory, prototypeMarkWord, hub);
 611         if (fillContents) {
 612             zeroMemory(size, memory, constantSize, instanceHeaderSize(INJECTED_VMCONFIG), false, counters);
 613         } else if (REPLACEMENTS_ASSERTIONS_ENABLED) {
 614             fillWithGarbage(size, memory, constantSize, instanceHeaderSize(INJECTED_VMCONFIG), false, counters);
 615         }
 616         MembarNode.memoryBarrier(MemoryBarriers.STORE_STORE, LocationIdentity.init());
 617         return memory.toObjectNonNull();
 618     }
 619 
 620     @Snippet
 621     private static void verifyHeap(@ConstantParameter Register threadRegister) {
 622         Word thread = registerAsWord(threadRegister);
 623         Word topValue = readTlabTop(thread);
 624         if (!topValue.equal(WordFactory.zero())) {
 625             Word topValueContents = topValue.readWord(0, MARK_WORD_LOCATION);
 626             if (topValueContents.equal(WordFactory.zero())) {
 627                 AssertionSnippets.vmMessageC(AssertionSnippets.ASSERTION_VM_MESSAGE_C, true, cstring("overzeroing of TLAB detected"), 0L, 0L, 0L);
 628             }
 629         }
 630     }
 631 
 632     /**
 633      * Formats some allocated memory with an object header and zeroes out the rest.
 634      */
 635     private static Object formatArray(KlassPointer hub, int allocationSize, int length, int headerSize, Word memory, Word prototypeMarkWord, boolean fillContents, boolean maybeUnroll,
 636                     Counters counters) {
 637         memory.writeInt(arrayLengthOffset(INJECTED_VMCONFIG), length, LocationIdentity.init());
 638         /*
 639          * store hub last as the concurrent garbage collectors assume length is valid if hub field
 640          * is not null
 641          */
 642         initializeObjectHeader(memory, prototypeMarkWord, hub);
 643         if (fillContents) {
 644             zeroMemory(allocationSize, memory, false, headerSize, maybeUnroll, counters);
 645         } else if (REPLACEMENTS_ASSERTIONS_ENABLED) {
 646             fillWithGarbage(allocationSize, memory, false, headerSize, maybeUnroll, counters);
 647         }
 648         MembarNode.memoryBarrier(MemoryBarriers.STORE_STORE, LocationIdentity.init());
 649         return memory.toObjectNonNull();
 650     }
 651 
 652     static class Counters {
 653         Counters(SnippetCounter.Group.Factory factory) {
 654             Group newInstance = factory.createSnippetCounterGroup("NewInstance");
 655             Group newArray = factory.createSnippetCounterGroup("NewArray");


 682         private final SnippetInfo newmultiarrayPIC = snippet(NewObjectSnippets.class, "newmultiarrayPIC", TLAB_TOP_LOCATION, TLAB_END_LOCATION);
 683         private final SnippetInfo verifyHeap = snippet(NewObjectSnippets.class, "verifyHeap");
 684         private final GraalHotSpotVMConfig config;
 685         private final Counters counters;
 686 
 687         public Templates(OptionValues options, Iterable<DebugHandlersFactory> factories, SnippetCounter.Group.Factory factory, HotSpotProviders providers, TargetDescription target,
 688                         GraalHotSpotVMConfig config) {
 689             super(options, factories, providers, providers.getSnippetReflection(), target);
 690             this.config = config;
 691             counters = new Counters(factory);
 692         }
 693 
 694         /**
 695          * Lowers a {@link NewInstanceNode}.
 696          */
 697         public void lower(NewInstanceNode newInstanceNode, HotSpotRegistersProvider registers, LoweringTool tool) {
 698             StructuredGraph graph = newInstanceNode.graph();
 699             HotSpotResolvedObjectType type = (HotSpotResolvedObjectType) newInstanceNode.instanceClass();
 700             assert !type.isArray();
 701             ConstantNode hub = ConstantNode.forConstant(KlassPointerStamp.klassNonNull(), type.klass(), providers.getMetaAccess(), graph);
 702             int size = instanceSize(type);
 703 
 704             OptionValues localOptions = graph.getOptions();
 705             SnippetInfo snippet = GeneratePIC.getValue(localOptions) ? allocateInstancePIC : allocateInstance;
 706             Arguments args = new Arguments(snippet, graph.getGuardsStage(), tool.getLoweringStage());
 707             args.addConst("size", size);
 708             args.add("hub", hub);
 709             args.add("prototypeMarkWord", type.prototypeMarkWord());
 710             args.addConst("fillContents", newInstanceNode.fillContents());
 711             args.addConst("threadRegister", registers.getThreadRegister());
 712             args.addConst("constantSize", true);
 713             args.addConst("typeContext", ProfileAllocations.getValue(localOptions) ? type.toJavaName(false) : "");
 714             args.addConst("counters", counters);
 715 
 716             SnippetTemplate template = template(newInstanceNode, args);
 717             graph.getDebug().log("Lowering allocateInstance in %s: node=%s, template=%s, arguments=%s", graph, newInstanceNode, template, args);
 718             template.instantiate(providers.getMetaAccess(), newInstanceNode, DEFAULT_REPLACER, args);
 719         }
 720 
 721         /**
 722          * Lowers a {@link NewArrayNode}.


 807 
 808         public void lower(NewMultiArrayNode newmultiarrayNode, LoweringTool tool) {
 809             StructuredGraph graph = newmultiarrayNode.graph();
 810             OptionValues localOptions = graph.getOptions();
 811             int rank = newmultiarrayNode.dimensionCount();
 812             ValueNode[] dims = new ValueNode[rank];
 813             for (int i = 0; i < newmultiarrayNode.dimensionCount(); i++) {
 814                 dims[i] = newmultiarrayNode.dimension(i);
 815             }
 816             HotSpotResolvedObjectType type = (HotSpotResolvedObjectType) newmultiarrayNode.type();
 817             ConstantNode hub = ConstantNode.forConstant(KlassPointerStamp.klassNonNull(), type.klass(), providers.getMetaAccess(), graph);
 818 
 819             SnippetInfo snippet = GeneratePIC.getValue(localOptions) ? newmultiarrayPIC : newmultiarray;
 820             Arguments args = new Arguments(snippet, graph.getGuardsStage(), tool.getLoweringStage());
 821             args.add("hub", hub);
 822             args.addConst("rank", rank);
 823             args.addVarargs("dimensions", int.class, StampFactory.forKind(JavaKind.Int), dims);
 824             template(newmultiarrayNode, args).instantiate(providers.getMetaAccess(), newmultiarrayNode, DEFAULT_REPLACER, args);
 825         }
 826 
 827         private static int instanceSize(HotSpotResolvedObjectType type) {
 828             int size = type.instanceSize();
 829             assert size >= 0;
 830             return size;
 831         }
 832 
 833         public void lower(VerifyHeapNode verifyHeapNode, HotSpotRegistersProvider registers, LoweringTool tool) {
 834             if (config.cAssertions) {
 835                 Arguments args = new Arguments(verifyHeap, verifyHeapNode.graph().getGuardsStage(), tool.getLoweringStage());
 836                 args.addConst("threadRegister", registers.getThreadRegister());
 837 
 838                 SnippetTemplate template = template(verifyHeapNode, args);
 839                 template.instantiate(providers.getMetaAccess(), verifyHeapNode, DEFAULT_REPLACER, args);
 840             } else {
 841                 GraphUtil.removeFixedWithUnusedInputs(verifyHeapNode);
 842             }
 843         }
 844     }
 845 }


 192             DynamicCounterNode.counter("number of allocations", name, 1, context);
 193         }
 194     }
 195 
 196     public static void emitPrefetchAllocate(Word address, boolean isArray) {
 197         if (allocatePrefetchStyle(INJECTED_VMCONFIG) > 0) {
 198             // Insert a prefetch for each allocation only on the fast-path
 199             // Generate several prefetch instructions.
 200             int lines = isArray ? allocatePrefetchLines(INJECTED_VMCONFIG) : allocateInstancePrefetchLines(INJECTED_VMCONFIG);
 201             int stepSize = allocatePrefetchStepSize(INJECTED_VMCONFIG);
 202             int distance = allocatePrefetchDistance(INJECTED_VMCONFIG);
 203             ExplodeLoopNode.explodeLoop();
 204             for (int i = 0; i < lines; i++) {
 205                 PrefetchAllocateNode.prefetch(OffsetAddressNode.address(address, distance));
 206                 distance += stepSize;
 207             }
 208         }
 209     }
 210 
 211     @Snippet
 212     public static Object allocateInstance(@ConstantParameter long size, KlassPointer hub, Word prototypeMarkWord, @ConstantParameter boolean fillContents,
 213                     @ConstantParameter Register threadRegister, @ConstantParameter boolean constantSize, @ConstantParameter String typeContext,
 214                     @ConstantParameter Counters counters) {
 215         return piCastToSnippetReplaceeStamp(allocateInstanceHelper(size, hub, prototypeMarkWord, fillContents, threadRegister, constantSize, typeContext, counters));
 216     }
 217 
 218     public static Object allocateInstanceHelper(long size, KlassPointer hub, Word prototypeMarkWord, boolean fillContents,
 219                     Register threadRegister, boolean constantSize, String typeContext, Counters counters) {
 220         Object result;
 221         Word thread = registerAsWord(threadRegister);
 222         Word top = readTlabTop(thread);
 223         Word end = readTlabEnd(thread);
 224         Word newTop = top.add(WordFactory.unsigned(size));
 225         if (useTLAB(INJECTED_VMCONFIG) && probability(FAST_PATH_PROBABILITY, newTop.belowOrEqual(end))) {
 226             writeTlabTop(thread, newTop);
 227             emitPrefetchAllocate(newTop, false);
 228             result = formatObject(hub, size, top, prototypeMarkWord, fillContents, constantSize, counters);
 229         } else {
 230             Counters theCounters = counters;
 231             if (theCounters != null && theCounters.stub != null) {
 232                 theCounters.stub.inc();
 233             }
 234             result = newInstanceStub(hub);
 235         }
 236         profileAllocation("instance", size, typeContext);
 237         return verifyOop(result);
 238     }
 239 
 240     public static Object newInstanceStub(KlassPointer hub) {
 241         if (useNullAllocationStubs(INJECTED_VMCONFIG)) {
 242             return nonNullOrDeopt(newInstanceOrNull(NEW_INSTANCE_OR_NULL, hub));
 243         } else {
 244             return newInstance(NEW_INSTANCE, hub);
 245         }
 246     }
 247 
 248     @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = true)
 249     private static native Object newInstance(@ConstantNodeParameter ForeignCallDescriptor descriptor, KlassPointer hub);
 250 
 251     @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = false)
 252     private static native Object newInstanceOrNull(@ConstantNodeParameter ForeignCallDescriptor descriptor, KlassPointer hub);
 253 
 254     @Snippet
 255     public static Object allocateInstancePIC(@ConstantParameter long size, KlassPointer hub, Word prototypeMarkWord, @ConstantParameter boolean fillContents,
 256                     @ConstantParameter Register threadRegister, @ConstantParameter boolean constantSize, @ConstantParameter String typeContext,
 257                     @ConstantParameter Counters counters) {
 258         // Klass must be initialized by the time the first instance is allocated, therefore we can
 259         // just load it from the corresponding cell and avoid the resolution check. We have to use a
 260         // fixed load though, to prevent it from floating above the initialization.
 261         KlassPointer picHub = LoadConstantIndirectlyFixedNode.loadKlass(hub);
 262         return piCastToSnippetReplaceeStamp(allocateInstanceHelper(size, picHub, prototypeMarkWord, fillContents, threadRegister, constantSize, typeContext, counters));
 263     }
 264 
 265     @Snippet
 266     public static Object allocateInstanceDynamic(Class<?> type, Class<?> classClass, @ConstantParameter boolean fillContents, @ConstantParameter Register threadRegister,
 267                     @ConstantParameter Counters counters) {
 268         if (probability(SLOW_PATH_PROBABILITY, type == null)) {
 269             DeoptimizeNode.deopt(None, RuntimeConstraint);
 270         }
 271         Class<?> nonNullType = PiNode.piCastNonNullClass(type, SnippetAnchorNode.anchor());
 272 
 273         if (probability(SLOW_PATH_PROBABILITY, DynamicNewInstanceNode.throwsInstantiationException(type, classClass))) {
 274             DeoptimizeNode.deopt(None, RuntimeConstraint);
 275         }


 299                     return allocateInstanceHelper(layoutHelper, nonNullHub, prototypeMarkWord, fillContents, threadRegister, false, "", counters);
 300                 }
 301             } else {
 302                 DeoptimizeNode.deopt(None, RuntimeConstraint);
 303             }
 304         }
 305         return dynamicNewInstanceStub(type);
 306     }
 307 
 308     /**
 309      * Maximum array length for which fast path allocation is used.
 310      */
 311     public static final int MAX_ARRAY_FAST_PATH_ALLOCATION_LENGTH = 0x00FFFFFF;
 312 
 313     @Snippet
 314     public static Object allocatePrimitiveArrayPIC(KlassPointer hub, int length, Word prototypeMarkWord, @ConstantParameter int headerSize, @ConstantParameter int log2ElementSize,
 315                     @ConstantParameter boolean fillContents, @ConstantParameter Register threadRegister, @ConstantParameter boolean maybeUnroll, @ConstantParameter String typeContext,
 316                     @ConstantParameter Counters counters) {
 317         // Primitive array types are eagerly pre-resolved. We can use a floating load.
 318         KlassPointer picHub = LoadConstantIndirectlyNode.loadKlass(hub);
 319         return allocateArrayImpl(picHub, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents, threadRegister, maybeUnroll, typeContext, counters);
 320     }
 321 
 322     @Snippet
 323     public static Object allocateArrayPIC(KlassPointer hub, int length, Word prototypeMarkWord, @ConstantParameter int headerSize, @ConstantParameter int log2ElementSize,
 324                     @ConstantParameter boolean fillContents, @ConstantParameter Register threadRegister, @ConstantParameter boolean maybeUnroll, @ConstantParameter String typeContext,
 325                     @ConstantParameter Counters counters) {
 326         // Array type would be resolved by dominating resolution.
 327         KlassPointer picHub = LoadConstantIndirectlyFixedNode.loadKlass(hub);
 328         return allocateArrayImpl(picHub, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents, threadRegister, maybeUnroll, typeContext, counters);
 329     }
 330 
 331     @Snippet
 332     public static Object allocateArray(KlassPointer hub,
 333                     int length,
 334                     Word prototypeMarkWord,
 335                     @ConstantParameter int headerSize,
 336                     @ConstantParameter int log2ElementSize,
 337                     @ConstantParameter boolean fillContents,
 338                     @ConstantParameter Register threadRegister,
 339                     @ConstantParameter boolean maybeUnroll,
 340                     @ConstantParameter String typeContext,
 341                     @ConstantParameter Counters counters) {
 342         Object result = allocateArrayImpl(hub,
 343                         length,
 344                         prototypeMarkWord,
 345                         headerSize,
 346                         log2ElementSize,
 347                         fillContents,
 348                         threadRegister,
 349                         maybeUnroll,
 350                         typeContext,

 351 
 352                         counters);
 353         return piArrayCastToSnippetReplaceeStamp(verifyOop(result), length);
 354     }
 355 
 356     /**
 357      * When allocating on the slow path, determines whether to use a version of the runtime call
 358      * that returns {@code null} on a failed allocation instead of raising an OutOfMemoryError.
 359      */
 360     @Fold
 361     static boolean useNullAllocationStubs(@InjectedParameter GraalHotSpotVMConfig config) {
 362         return config.areNullAllocationStubsAvailable();
 363     }
 364 
 365     private static Object allocateArrayImpl(KlassPointer hub, int length, Word prototypeMarkWord, int headerSize, int log2ElementSize, boolean fillContents, Register threadRegister,
 366                     boolean maybeUnroll, String typeContext, Counters counters) {
 367         Object result;
 368         long allocationSize = arrayAllocationSize(length, headerSize, log2ElementSize);
 369         Word thread = registerAsWord(threadRegister);
 370         Word top = readTlabTop(thread);
 371         Word end = readTlabEnd(thread);
 372         Word newTop = top.add(WordFactory.unsigned(allocationSize));
 373         if (probability(FREQUENT_PROBABILITY, belowThan(length, MAX_ARRAY_FAST_PATH_ALLOCATION_LENGTH)) && useTLAB(INJECTED_VMCONFIG) &&
 374                         probability(FAST_PATH_PROBABILITY, newTop.belowOrEqual(end))) {
 375             writeTlabTop(thread, newTop);
 376             emitPrefetchAllocate(newTop, true);
 377             Counters theCounters = counters;
 378             if (theCounters != null && theCounters.arrayLoopInit != null) {
 379                 theCounters.arrayLoopInit.inc();
 380             }
 381             result = formatArray(hub, allocationSize, length, headerSize, top, prototypeMarkWord, fillContents, maybeUnroll, counters);
 382         } else {
 383             result = newArrayStub(hub, length);
 384         }
 385         profileAllocation("array", allocationSize, typeContext);
 386         return result;
 387     }
 388 
 389     public static Object newArrayStub(KlassPointer hub, int length) {
 390         if (useNullAllocationStubs(INJECTED_VMCONFIG)) {
 391             return nonNullOrDeopt(newArrayOrNull(NEW_ARRAY_OR_NULL, hub, length));
 392         } else {
 393             return newArray(NEW_ARRAY, hub, length);


 468         } else {
 469             runtimeAssert(knownLayoutHelper == readLayoutHelper(nonNullKlass), "layout mismatch");
 470             layoutHelper = knownLayoutHelper;
 471         }
 472         //@formatter:off
 473         // from src/share/vm/oops/klass.hpp:
 474         //
 475         // For arrays, layout helper is a negative number, containing four
 476         // distinct bytes, as follows:
 477         //    MSB:[tag, hsz, ebt, log2(esz)]:LSB
 478         // where:
 479         //    tag is 0x80 if the elements are oops, 0xC0 if non-oops
 480         //    hsz is array header size in bytes (i.e., offset of first element)
 481         //    ebt is the BasicType of the elements
 482         //    esz is the element size in bytes
 483         //@formatter:on
 484 
 485         int headerSize = (layoutHelper >> layoutHelperHeaderSizeShift(INJECTED_VMCONFIG)) & layoutHelperHeaderSizeMask(INJECTED_VMCONFIG);
 486         int log2ElementSize = (layoutHelper >> layoutHelperLog2ElementSizeShift(INJECTED_VMCONFIG)) & layoutHelperLog2ElementSizeMask(INJECTED_VMCONFIG);
 487 
 488         Object result = allocateArrayImpl(nonNullKlass, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents, threadRegister, false, "dynamic type", counters);
 489         return piArrayCastToSnippetReplaceeStamp(verifyOop(result), length);
 490     }
 491 
 492     /**
 493      * Calls the runtime stub for implementing MULTIANEWARRAY.
 494      */
 495     @Snippet
 496     private static Object newmultiarray(KlassPointer hub, @ConstantParameter int rank, @VarargsParameter int[] dimensions) {
 497         Word dims = DimensionsNode.allocaDimsArray(rank);
 498         ExplodeLoopNode.explodeLoop();
 499         for (int i = 0; i < rank; i++) {
 500             dims.writeInt(i * 4, dimensions[i], LocationIdentity.init());
 501         }
 502         return newMultiArrayStub(hub, rank, dims);
 503     }
 504 
 505     private static Object newMultiArrayStub(KlassPointer hub, int rank, Word dims) {
 506         if (useNullAllocationStubs(INJECTED_VMCONFIG)) {
 507             return nonNullOrDeopt(newMultiArrayOrNull(NEW_MULTI_ARRAY_OR_NULL, hub, rank, dims));
 508         } else {


 522 
 523     @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = false)
 524     private static native Object newMultiArrayOrNull(@ConstantNodeParameter ForeignCallDescriptor descriptor, KlassPointer hub, int rank, Word dims);
 525 
 526     /**
 527      * Maximum number of long stores to emit when zeroing an object with a constant size. Larger
 528      * objects have their bodies initialized in a loop.
 529      */
 530     private static final int MAX_UNROLLED_OBJECT_ZEROING_STORES = 8;
 531 
 532     /**
 533      * Zero uninitialized memory in a newly allocated object, unrolling as necessary and ensuring
 534      * that stores are aligned.
 535      *
 536      * @param size number of bytes to zero
 537      * @param memory beginning of object which is being zeroed
 538      * @param constantSize is {@code size} known to be constant in the snippet
 539      * @param startOffset offset to begin zeroing. May not be word aligned.
 540      * @param manualUnroll maximally unroll zeroing
 541      */
 542     private static void zeroMemory(long size, Word memory, boolean constantSize, int startOffset, boolean manualUnroll, Counters counters) {
 543         fillMemory(0, size, memory, constantSize, startOffset, manualUnroll, counters);
 544     }
 545 
 546     private static void fillMemory(long value, long size, Word memory, boolean constantSize, int startOffset, boolean manualUnroll, Counters counters) {
 547         ReplacementsUtil.runtimeAssert((size & 0x7) == 0, "unaligned object size");
 548         int offset = startOffset;
 549         if ((offset & 0x7) != 0) {
 550             memory.writeInt(offset, (int) value, LocationIdentity.init());
 551             offset += 4;
 552         }
 553         ReplacementsUtil.runtimeAssert((offset & 0x7) == 0, "unaligned offset");
 554         Counters theCounters = counters;
 555         if (manualUnroll && ((size - offset) / 8) <= MAX_UNROLLED_OBJECT_ZEROING_STORES) {
 556             ReplacementsUtil.staticAssert(!constantSize, "size shouldn't be constant at instantiation time");
 557             // This case handles arrays of constant length. Instead of having a snippet variant for
 558             // each length, generate a chain of stores of maximum length. Once it's inlined the
 559             // break statement will trim excess stores.
 560             if (theCounters != null && theCounters.instanceSeqInit != null) {
 561                 theCounters.instanceSeqInit.inc();
 562             }
 563 
 564             explodeLoop();
 565             for (int i = 0; i < MAX_UNROLLED_OBJECT_ZEROING_STORES; i++, offset += 8) {
 566                 if (offset == size) {


 580                 if (theCounters != null && theCounters.instanceLoopInit != null) {
 581                     theCounters.instanceLoopInit.inc();
 582                 }
 583             }
 584             for (; off.rawValue() < size; off = off.add(8)) {
 585                 memory.initializeLong(off, value, LocationIdentity.init());
 586             }
 587         }
 588     }
 589 
 590     /**
 591      * Fill uninitialized memory with garbage value in a newly allocated object, unrolling as
 592      * necessary and ensuring that stores are aligned.
 593      *
 594      * @param size number of bytes to zero
 595      * @param memory beginning of object which is being zeroed
 596      * @param constantSize is {@code  size} known to be constant in the snippet
 597      * @param startOffset offset to begin zeroing. May not be word aligned.
 598      * @param manualUnroll maximally unroll zeroing
 599      */
 600     private static void fillWithGarbage(long size, Word memory, boolean constantSize, int startOffset, boolean manualUnroll, Counters counters) {
 601         fillMemory(0xfefefefefefefefeL, size, memory, constantSize, startOffset, manualUnroll, counters);
 602     }
 603 
 604     /**
 605      * Formats some allocated memory with an object header and zeroes out the rest.
 606      */
 607     private static Object formatObject(KlassPointer hub, long size, Word memory, Word compileTimePrototypeMarkWord, boolean fillContents, boolean constantSize, Counters counters) {
 608         Word prototypeMarkWord = useBiasedLocking(INJECTED_VMCONFIG) ? hub.readWord(prototypeMarkWordOffset(INJECTED_VMCONFIG), PROTOTYPE_MARK_WORD_LOCATION) : compileTimePrototypeMarkWord;
 609         initializeObjectHeader(memory, prototypeMarkWord, hub);
 610         if (fillContents) {
 611             zeroMemory(size, memory, constantSize, instanceHeaderSize(INJECTED_VMCONFIG), false, counters);
 612         } else if (REPLACEMENTS_ASSERTIONS_ENABLED) {
 613             fillWithGarbage(size, memory, constantSize, instanceHeaderSize(INJECTED_VMCONFIG), false, counters);
 614         }
 615         MembarNode.memoryBarrier(MemoryBarriers.STORE_STORE, LocationIdentity.init());
 616         return memory.toObjectNonNull();
 617     }
 618 
 619     @Snippet
 620     private static void verifyHeap(@ConstantParameter Register threadRegister) {
 621         Word thread = registerAsWord(threadRegister);
 622         Word topValue = readTlabTop(thread);
 623         if (!topValue.equal(WordFactory.zero())) {
 624             Word topValueContents = topValue.readWord(0, MARK_WORD_LOCATION);
 625             if (topValueContents.equal(WordFactory.zero())) {
 626                 AssertionSnippets.vmMessageC(AssertionSnippets.ASSERTION_VM_MESSAGE_C, true, cstring("overzeroing of TLAB detected"), 0L, 0L, 0L);
 627             }
 628         }
 629     }
 630 
 631     /**
 632      * Formats some allocated memory with an object header and zeroes out the rest.
 633      */
 634     private static Object formatArray(KlassPointer hub, long allocationSize, int length, int headerSize, Word memory, Word prototypeMarkWord, boolean fillContents, boolean maybeUnroll,
 635                     Counters counters) {
 636         memory.writeInt(arrayLengthOffset(INJECTED_VMCONFIG), length, LocationIdentity.init());
 637         /*
 638          * store hub last as the concurrent garbage collectors assume length is valid if hub field
 639          * is not null
 640          */
 641         initializeObjectHeader(memory, prototypeMarkWord, hub);
 642         if (fillContents) {
 643             zeroMemory(allocationSize, memory, false, headerSize, maybeUnroll, counters);
 644         } else if (REPLACEMENTS_ASSERTIONS_ENABLED) {
 645             fillWithGarbage(allocationSize, memory, false, headerSize, maybeUnroll, counters);
 646         }
 647         MembarNode.memoryBarrier(MemoryBarriers.STORE_STORE, LocationIdentity.init());
 648         return memory.toObjectNonNull();
 649     }
 650 
 651     static class Counters {
 652         Counters(SnippetCounter.Group.Factory factory) {
 653             Group newInstance = factory.createSnippetCounterGroup("NewInstance");
 654             Group newArray = factory.createSnippetCounterGroup("NewArray");


 681         private final SnippetInfo newmultiarrayPIC = snippet(NewObjectSnippets.class, "newmultiarrayPIC", TLAB_TOP_LOCATION, TLAB_END_LOCATION);
 682         private final SnippetInfo verifyHeap = snippet(NewObjectSnippets.class, "verifyHeap");
 683         private final GraalHotSpotVMConfig config;
 684         private final Counters counters;
 685 
 686         public Templates(OptionValues options, Iterable<DebugHandlersFactory> factories, SnippetCounter.Group.Factory factory, HotSpotProviders providers, TargetDescription target,
 687                         GraalHotSpotVMConfig config) {
 688             super(options, factories, providers, providers.getSnippetReflection(), target);
 689             this.config = config;
 690             counters = new Counters(factory);
 691         }
 692 
 693         /**
 694          * Lowers a {@link NewInstanceNode}.
 695          */
 696         public void lower(NewInstanceNode newInstanceNode, HotSpotRegistersProvider registers, LoweringTool tool) {
 697             StructuredGraph graph = newInstanceNode.graph();
 698             HotSpotResolvedObjectType type = (HotSpotResolvedObjectType) newInstanceNode.instanceClass();
 699             assert !type.isArray();
 700             ConstantNode hub = ConstantNode.forConstant(KlassPointerStamp.klassNonNull(), type.klass(), providers.getMetaAccess(), graph);
 701             long size = instanceSize(type);
 702 
 703             OptionValues localOptions = graph.getOptions();
 704             SnippetInfo snippet = GeneratePIC.getValue(localOptions) ? allocateInstancePIC : allocateInstance;
 705             Arguments args = new Arguments(snippet, graph.getGuardsStage(), tool.getLoweringStage());
 706             args.addConst("size", size);
 707             args.add("hub", hub);
 708             args.add("prototypeMarkWord", type.prototypeMarkWord());
 709             args.addConst("fillContents", newInstanceNode.fillContents());
 710             args.addConst("threadRegister", registers.getThreadRegister());
 711             args.addConst("constantSize", true);
 712             args.addConst("typeContext", ProfileAllocations.getValue(localOptions) ? type.toJavaName(false) : "");
 713             args.addConst("counters", counters);
 714 
 715             SnippetTemplate template = template(newInstanceNode, args);
 716             graph.getDebug().log("Lowering allocateInstance in %s: node=%s, template=%s, arguments=%s", graph, newInstanceNode, template, args);
 717             template.instantiate(providers.getMetaAccess(), newInstanceNode, DEFAULT_REPLACER, args);
 718         }
 719 
 720         /**
 721          * Lowers a {@link NewArrayNode}.


 806 
 807         public void lower(NewMultiArrayNode newmultiarrayNode, LoweringTool tool) {
 808             StructuredGraph graph = newmultiarrayNode.graph();
 809             OptionValues localOptions = graph.getOptions();
 810             int rank = newmultiarrayNode.dimensionCount();
 811             ValueNode[] dims = new ValueNode[rank];
 812             for (int i = 0; i < newmultiarrayNode.dimensionCount(); i++) {
 813                 dims[i] = newmultiarrayNode.dimension(i);
 814             }
 815             HotSpotResolvedObjectType type = (HotSpotResolvedObjectType) newmultiarrayNode.type();
 816             ConstantNode hub = ConstantNode.forConstant(KlassPointerStamp.klassNonNull(), type.klass(), providers.getMetaAccess(), graph);
 817 
 818             SnippetInfo snippet = GeneratePIC.getValue(localOptions) ? newmultiarrayPIC : newmultiarray;
 819             Arguments args = new Arguments(snippet, graph.getGuardsStage(), tool.getLoweringStage());
 820             args.add("hub", hub);
 821             args.addConst("rank", rank);
 822             args.addVarargs("dimensions", int.class, StampFactory.forKind(JavaKind.Int), dims);
 823             template(newmultiarrayNode, args).instantiate(providers.getMetaAccess(), newmultiarrayNode, DEFAULT_REPLACER, args);
 824         }
 825 
 826         private static long instanceSize(HotSpotResolvedObjectType type) {
 827             long size = type.instanceSize();
 828             assert size >= 0;
 829             return size;
 830         }
 831 
 832         public void lower(VerifyHeapNode verifyHeapNode, HotSpotRegistersProvider registers, LoweringTool tool) {
 833             if (config.cAssertions) {
 834                 Arguments args = new Arguments(verifyHeap, verifyHeapNode.graph().getGuardsStage(), tool.getLoweringStage());
 835                 args.addConst("threadRegister", registers.getThreadRegister());
 836 
 837                 SnippetTemplate template = template(verifyHeapNode, args);
 838                 template.instantiate(providers.getMetaAccess(), verifyHeapNode, DEFAULT_REPLACER, args);
 839             } else {
 840                 GraphUtil.removeFixedWithUnusedInputs(verifyHeapNode);
 841             }
 842         }
 843     }
 844 }
< prev index next >