13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23 package org.graalvm.compiler.hotspot.replacements;
24
25 import static jdk.vm.ci.hotspot.HotSpotJVMCIRuntimeProvider.getArrayBaseOffset;
26 import static org.graalvm.compiler.core.common.GraalOptions.GeneratePIC;
27 import static org.graalvm.compiler.core.common.calc.UnsignedMath.belowThan;
28 import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfig.INJECTED_VMCONFIG;
29 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.CLASS_ARRAY_KLASS_LOCATION;
30 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.HUB_WRITE_LOCATION;
31 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.MARK_WORD_LOCATION;
32 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.PROTOTYPE_MARK_WORD_LOCATION;
33 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_END_LOCATION;
34 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_TOP_LOCATION;
35 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayAllocationSize;
36 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayKlassOffset;
37 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayLengthOffset;
38 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.config;
39 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.initializeObjectHeader;
40 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.instanceHeaderSize;
41 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.isInstanceKlassFullyInitialized;
42 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperHeaderSizeMask;
43 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperHeaderSizeShift;
44 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperLog2ElementSizeMask;
45 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperLog2ElementSizeShift;
46 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.loadKlassFromObject;
47 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.prototypeMarkWordOffset;
48 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readLayoutHelper;
49 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readTlabEnd;
50 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readTlabTop;
51 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.registerAsWord;
52 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.useBiasedLocking;
53 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.useTLAB;
54 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.verifyOop;
55 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.writeTlabTop;
56 import static org.graalvm.compiler.hotspot.replacements.HotspotSnippetsOptions.ProfileAllocations;
57 import static org.graalvm.compiler.hotspot.replacements.HotspotSnippetsOptions.ProfileAllocationsContext;
58 import static org.graalvm.compiler.nodes.PiArrayNode.piArrayCastToSnippetReplaceeStamp;
59 import static org.graalvm.compiler.nodes.PiNode.piCastToSnippetReplaceeStamp;
60 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.FAST_PATH_PROBABILITY;
61 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.FREQUENT_PROBABILITY;
62 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.SLOW_PATH_PROBABILITY;
63 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.probability;
64 import static org.graalvm.compiler.replacements.ReplacementsUtil.REPLACEMENTS_ASSERTIONS_ENABLED;
65 import static org.graalvm.compiler.replacements.ReplacementsUtil.runtimeAssert;
66 import static org.graalvm.compiler.replacements.ReplacementsUtil.staticAssert;
67 import static org.graalvm.compiler.replacements.SnippetTemplate.DEFAULT_REPLACER;
68 import static org.graalvm.compiler.replacements.nodes.CStringConstant.cstring;
69 import static org.graalvm.compiler.replacements.nodes.ExplodeLoopNode.explodeLoop;
193 ExplodeLoopNode.explodeLoop();
194 for (int i = 0; i < lines; i++) {
195 PrefetchAllocateNode.prefetch(OffsetAddressNode.address(address, distance));
196 distance += stepSize;
197 }
198 }
199 }
200
201 @Snippet
202 public static Object allocateInstance(@ConstantParameter int size, KlassPointer hub, Word prototypeMarkWord, @ConstantParameter boolean fillContents,
203 @ConstantParameter Register threadRegister, @ConstantParameter boolean constantSize, @ConstantParameter String typeContext, @ConstantParameter OptionValues options,
204 @ConstantParameter Counters counters) {
205 return piCastToSnippetReplaceeStamp(allocateInstanceHelper(size, hub, prototypeMarkWord, fillContents, threadRegister, constantSize, typeContext, options, counters));
206 }
207
208 public static Object allocateInstanceHelper(int size, KlassPointer hub, Word prototypeMarkWord, boolean fillContents,
209 Register threadRegister, boolean constantSize, String typeContext, OptionValues options, Counters counters) {
210 Object result;
211 Word thread = registerAsWord(threadRegister);
212 Word top = readTlabTop(thread);
213 Word end = readTlabEnd(thread);
214 Word newTop = top.add(size);
215 if (useTLAB(INJECTED_VMCONFIG) && probability(FAST_PATH_PROBABILITY, newTop.belowOrEqual(end))) {
216 writeTlabTop(thread, newTop);
217 emitPrefetchAllocate(newTop, false);
218 result = formatObject(hub, size, top, prototypeMarkWord, fillContents, constantSize, counters);
219 } else {
220 if (counters != null && counters.stub != null) {
221 counters.stub.inc();
222 }
223 result = newInstance(HotSpotBackend.NEW_INSTANCE, hub);
224 }
225 profileAllocation("instance", size, typeContext, options);
226 return verifyOop(result);
227 }
228
229 @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = true)
230 public static native Object newInstance(@ConstantNodeParameter ForeignCallDescriptor descriptor, KlassPointer hub);
231
232 @Snippet
233 public static Object allocateInstancePIC(@ConstantParameter int size, KlassPointer hub, Word prototypeMarkWord, @ConstantParameter boolean fillContents,
301 @ConstantParameter OptionValues options, @ConstantParameter Counters counters) {
302 // Array type would be resolved by dominating resolution.
303 KlassPointer picHub = LoadConstantIndirectlyFixedNode.loadKlass(hub);
304 return allocateArrayImpl(picHub, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents, threadRegister, maybeUnroll, typeContext, false, options, counters);
305 }
306
307 @Snippet
308 public static Object allocateArray(KlassPointer hub, int length, Word prototypeMarkWord, @ConstantParameter int headerSize, @ConstantParameter int log2ElementSize,
309 @ConstantParameter boolean fillContents, @ConstantParameter Register threadRegister, @ConstantParameter boolean maybeUnroll, @ConstantParameter String typeContext,
310 @ConstantParameter OptionValues options, @ConstantParameter Counters counters) {
311 Object result = allocateArrayImpl(hub, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents, threadRegister, maybeUnroll, typeContext, false, options, counters);
312 return piArrayCastToSnippetReplaceeStamp(verifyOop(result), length);
313 }
314
315 private static Object allocateArrayImpl(KlassPointer hub, int length, Word prototypeMarkWord, int headerSize, int log2ElementSize, boolean fillContents, Register threadRegister,
316 boolean maybeUnroll, String typeContext, boolean skipNegativeCheck, OptionValues options, Counters counters) {
317 Object result;
318 int allocationSize = arrayAllocationSize(length, headerSize, log2ElementSize);
319 Word thread = registerAsWord(threadRegister);
320 Word top = readTlabTop(thread);
321 Word end = readTlabEnd(thread);
322 Word newTop = top.add(allocationSize);
323 if (probability(FREQUENT_PROBABILITY, skipNegativeCheck || belowThan(length, MAX_ARRAY_FAST_PATH_ALLOCATION_LENGTH)) && useTLAB(INJECTED_VMCONFIG) &&
324 probability(FAST_PATH_PROBABILITY, newTop.belowOrEqual(end))) {
325 writeTlabTop(thread, newTop);
326 emitPrefetchAllocate(newTop, true);
327 if (counters != null && counters.arrayLoopInit != null) {
328 counters.arrayLoopInit.inc();
329 }
330 result = formatArray(hub, allocationSize, length, headerSize, top, prototypeMarkWord, fillContents, maybeUnroll, counters);
331 } else {
332 result = newArray(HotSpotBackend.NEW_ARRAY, hub, length, fillContents);
333 }
334 profileAllocation("array", allocationSize, typeContext, options);
335 return result;
336 }
337
338 @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = true)
339 public static native Object newArray(@ConstantNodeParameter ForeignCallDescriptor descriptor, KlassPointer hub, int length, boolean fillContents);
340
341 public static final ForeignCallDescriptor DYNAMIC_NEW_ARRAY = new ForeignCallDescriptor("dynamic_new_array", Object.class, Class.class, int.class);
562 }
563
564 static class Counters {
565 Counters(SnippetCounter.Group.Factory factory) {
566 Group newInstance = factory.createSnippetCounterGroup("NewInstance");
567 Group newArray = factory.createSnippetCounterGroup("NewArray");
568 instanceSeqInit = new SnippetCounter(newInstance, "tlabSeqInit", "TLAB alloc with unrolled zeroing");
569 instanceLoopInit = new SnippetCounter(newInstance, "tlabLoopInit", "TLAB alloc with zeroing in a loop");
570 arrayLoopInit = new SnippetCounter(newArray, "tlabLoopInit", "TLAB alloc with zeroing in a loop");
571 stub = new SnippetCounter(newInstance, "stub", "alloc and zeroing via stub");
572 }
573
574 final SnippetCounter instanceSeqInit;
575 final SnippetCounter instanceLoopInit;
576 final SnippetCounter arrayLoopInit;
577 final SnippetCounter stub;
578 }
579
580 public static class Templates extends AbstractTemplates {
581
582 private final SnippetInfo allocateInstance = snippet(NewObjectSnippets.class, "allocateInstance", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION, TLAB_END_LOCATION);
583 private final SnippetInfo allocateInstancePIC = snippet(NewObjectSnippets.class, "allocateInstancePIC", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION,
584 TLAB_END_LOCATION);
585 private final SnippetInfo allocateArray = snippet(NewObjectSnippets.class, "allocateArray", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION, TLAB_END_LOCATION);
586 private final SnippetInfo allocateArrayPIC = snippet(NewObjectSnippets.class, "allocateArrayPIC", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION, TLAB_END_LOCATION);
587 private final SnippetInfo allocatePrimitiveArrayPIC = snippet(NewObjectSnippets.class, "allocatePrimitiveArrayPIC", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION,
588 TLAB_END_LOCATION);
589 private final SnippetInfo allocateArrayDynamic = snippet(NewObjectSnippets.class, "allocateArrayDynamic", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION,
590 TLAB_END_LOCATION);
591 private final SnippetInfo allocateInstanceDynamic = snippet(NewObjectSnippets.class, "allocateInstanceDynamic", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION,
592 TLAB_END_LOCATION);
593 private final SnippetInfo newmultiarray = snippet(NewObjectSnippets.class, "newmultiarray", TLAB_TOP_LOCATION, TLAB_END_LOCATION);
594 private final SnippetInfo newmultiarrayPIC = snippet(NewObjectSnippets.class, "newmultiarrayPIC", TLAB_TOP_LOCATION, TLAB_END_LOCATION);
595 private final SnippetInfo verifyHeap = snippet(NewObjectSnippets.class, "verifyHeap");
596 private final GraalHotSpotVMConfig config;
597 private final Counters counters;
598
599 public Templates(OptionValues options, Iterable<DebugHandlersFactory> factories, SnippetCounter.Group.Factory factory, HotSpotProviders providers, TargetDescription target,
600 GraalHotSpotVMConfig config) {
601 super(options, factories, providers, providers.getSnippetReflection(), target);
602 this.config = config;
603 counters = new Counters(factory);
604 }
605
606 /**
607 * Lowers a {@link NewInstanceNode}.
608 */
609 public void lower(NewInstanceNode newInstanceNode, HotSpotRegistersProvider registers, LoweringTool tool) {
610 StructuredGraph graph = newInstanceNode.graph();
611 HotSpotResolvedObjectType type = (HotSpotResolvedObjectType) newInstanceNode.instanceClass();
612 assert !type.isArray();
613 ConstantNode hub = ConstantNode.forConstant(KlassPointerStamp.klassNonNull(), type.klass(), providers.getMetaAccess(), graph);
614 int size = instanceSize(type);
|
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23 package org.graalvm.compiler.hotspot.replacements;
24
25 import static jdk.vm.ci.hotspot.HotSpotJVMCIRuntimeProvider.getArrayBaseOffset;
26 import static org.graalvm.compiler.core.common.GraalOptions.GeneratePIC;
27 import static org.graalvm.compiler.core.common.calc.UnsignedMath.belowThan;
28 import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfig.INJECTED_VMCONFIG;
29 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.CLASS_ARRAY_KLASS_LOCATION;
30 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.HUB_WRITE_LOCATION;
31 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.MARK_WORD_LOCATION;
32 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.PROTOTYPE_MARK_WORD_LOCATION;
33 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_FAST_PATH_END_LOCATION;
34 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_TOP_LOCATION;
35 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayAllocationSize;
36 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayKlassOffset;
37 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayLengthOffset;
38 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.config;
39 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.initializeObjectHeader;
40 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.instanceHeaderSize;
41 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.isInstanceKlassFullyInitialized;
42 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperHeaderSizeMask;
43 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperHeaderSizeShift;
44 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperLog2ElementSizeMask;
45 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperLog2ElementSizeShift;
46 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.loadKlassFromObject;
47 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.prototypeMarkWordOffset;
48 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readLayoutHelper;
49 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readTlabFastPathEnd;
50 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readTlabTop;
51 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.registerAsWord;
52 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.useBiasedLocking;
53 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.useTLAB;
54 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.verifyOop;
55 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.writeTlabTop;
56 import static org.graalvm.compiler.hotspot.replacements.HotspotSnippetsOptions.ProfileAllocations;
57 import static org.graalvm.compiler.hotspot.replacements.HotspotSnippetsOptions.ProfileAllocationsContext;
58 import static org.graalvm.compiler.nodes.PiArrayNode.piArrayCastToSnippetReplaceeStamp;
59 import static org.graalvm.compiler.nodes.PiNode.piCastToSnippetReplaceeStamp;
60 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.FAST_PATH_PROBABILITY;
61 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.FREQUENT_PROBABILITY;
62 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.SLOW_PATH_PROBABILITY;
63 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.probability;
64 import static org.graalvm.compiler.replacements.ReplacementsUtil.REPLACEMENTS_ASSERTIONS_ENABLED;
65 import static org.graalvm.compiler.replacements.ReplacementsUtil.runtimeAssert;
66 import static org.graalvm.compiler.replacements.ReplacementsUtil.staticAssert;
67 import static org.graalvm.compiler.replacements.SnippetTemplate.DEFAULT_REPLACER;
68 import static org.graalvm.compiler.replacements.nodes.CStringConstant.cstring;
69 import static org.graalvm.compiler.replacements.nodes.ExplodeLoopNode.explodeLoop;
193 ExplodeLoopNode.explodeLoop();
194 for (int i = 0; i < lines; i++) {
195 PrefetchAllocateNode.prefetch(OffsetAddressNode.address(address, distance));
196 distance += stepSize;
197 }
198 }
199 }
200
201 @Snippet
202 public static Object allocateInstance(@ConstantParameter int size, KlassPointer hub, Word prototypeMarkWord, @ConstantParameter boolean fillContents,
203 @ConstantParameter Register threadRegister, @ConstantParameter boolean constantSize, @ConstantParameter String typeContext, @ConstantParameter OptionValues options,
204 @ConstantParameter Counters counters) {
205 return piCastToSnippetReplaceeStamp(allocateInstanceHelper(size, hub, prototypeMarkWord, fillContents, threadRegister, constantSize, typeContext, options, counters));
206 }
207
208 public static Object allocateInstanceHelper(int size, KlassPointer hub, Word prototypeMarkWord, boolean fillContents,
209 Register threadRegister, boolean constantSize, String typeContext, OptionValues options, Counters counters) {
210 Object result;
211 Word thread = registerAsWord(threadRegister);
212 Word top = readTlabTop(thread);
213 Word end = readTlabFastPathEnd(thread);
214 Word newTop = top.add(size);
215 if (useTLAB(INJECTED_VMCONFIG) && probability(FAST_PATH_PROBABILITY, newTop.belowOrEqual(end))) {
216 writeTlabTop(thread, newTop);
217 emitPrefetchAllocate(newTop, false);
218 result = formatObject(hub, size, top, prototypeMarkWord, fillContents, constantSize, counters);
219 } else {
220 if (counters != null && counters.stub != null) {
221 counters.stub.inc();
222 }
223 result = newInstance(HotSpotBackend.NEW_INSTANCE, hub);
224 }
225 profileAllocation("instance", size, typeContext, options);
226 return verifyOop(result);
227 }
228
229 @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = true)
230 public static native Object newInstance(@ConstantNodeParameter ForeignCallDescriptor descriptor, KlassPointer hub);
231
232 @Snippet
233 public static Object allocateInstancePIC(@ConstantParameter int size, KlassPointer hub, Word prototypeMarkWord, @ConstantParameter boolean fillContents,
301 @ConstantParameter OptionValues options, @ConstantParameter Counters counters) {
302 // Array type would be resolved by dominating resolution.
303 KlassPointer picHub = LoadConstantIndirectlyFixedNode.loadKlass(hub);
304 return allocateArrayImpl(picHub, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents, threadRegister, maybeUnroll, typeContext, false, options, counters);
305 }
306
307 @Snippet
308 public static Object allocateArray(KlassPointer hub, int length, Word prototypeMarkWord, @ConstantParameter int headerSize, @ConstantParameter int log2ElementSize,
309 @ConstantParameter boolean fillContents, @ConstantParameter Register threadRegister, @ConstantParameter boolean maybeUnroll, @ConstantParameter String typeContext,
310 @ConstantParameter OptionValues options, @ConstantParameter Counters counters) {
311 Object result = allocateArrayImpl(hub, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents, threadRegister, maybeUnroll, typeContext, false, options, counters);
312 return piArrayCastToSnippetReplaceeStamp(verifyOop(result), length);
313 }
314
315 private static Object allocateArrayImpl(KlassPointer hub, int length, Word prototypeMarkWord, int headerSize, int log2ElementSize, boolean fillContents, Register threadRegister,
316 boolean maybeUnroll, String typeContext, boolean skipNegativeCheck, OptionValues options, Counters counters) {
317 Object result;
318 int allocationSize = arrayAllocationSize(length, headerSize, log2ElementSize);
319 Word thread = registerAsWord(threadRegister);
320 Word top = readTlabTop(thread);
321 Word end = readTlabFastPathEnd(thread);
322 Word newTop = top.add(allocationSize);
323 if (probability(FREQUENT_PROBABILITY, skipNegativeCheck || belowThan(length, MAX_ARRAY_FAST_PATH_ALLOCATION_LENGTH)) && useTLAB(INJECTED_VMCONFIG) &&
324 probability(FAST_PATH_PROBABILITY, newTop.belowOrEqual(end))) {
325 writeTlabTop(thread, newTop);
326 emitPrefetchAllocate(newTop, true);
327 if (counters != null && counters.arrayLoopInit != null) {
328 counters.arrayLoopInit.inc();
329 }
330 result = formatArray(hub, allocationSize, length, headerSize, top, prototypeMarkWord, fillContents, maybeUnroll, counters);
331 } else {
332 result = newArray(HotSpotBackend.NEW_ARRAY, hub, length, fillContents);
333 }
334 profileAllocation("array", allocationSize, typeContext, options);
335 return result;
336 }
337
338 @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = true)
339 public static native Object newArray(@ConstantNodeParameter ForeignCallDescriptor descriptor, KlassPointer hub, int length, boolean fillContents);
340
341 public static final ForeignCallDescriptor DYNAMIC_NEW_ARRAY = new ForeignCallDescriptor("dynamic_new_array", Object.class, Class.class, int.class);
562 }
563
564 static class Counters {
565 Counters(SnippetCounter.Group.Factory factory) {
566 Group newInstance = factory.createSnippetCounterGroup("NewInstance");
567 Group newArray = factory.createSnippetCounterGroup("NewArray");
568 instanceSeqInit = new SnippetCounter(newInstance, "tlabSeqInit", "TLAB alloc with unrolled zeroing");
569 instanceLoopInit = new SnippetCounter(newInstance, "tlabLoopInit", "TLAB alloc with zeroing in a loop");
570 arrayLoopInit = new SnippetCounter(newArray, "tlabLoopInit", "TLAB alloc with zeroing in a loop");
571 stub = new SnippetCounter(newInstance, "stub", "alloc and zeroing via stub");
572 }
573
574 final SnippetCounter instanceSeqInit;
575 final SnippetCounter instanceLoopInit;
576 final SnippetCounter arrayLoopInit;
577 final SnippetCounter stub;
578 }
579
580 public static class Templates extends AbstractTemplates {
581
582 private final SnippetInfo allocateInstance = snippet(NewObjectSnippets.class, "allocateInstance", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION, TLAB_FAST_PATH_END_LOCATION);
583 private final SnippetInfo allocateInstancePIC = snippet(NewObjectSnippets.class, "allocateInstancePIC", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION,
584 TLAB_FAST_PATH_END_LOCATION);
585 private final SnippetInfo allocateArray = snippet(NewObjectSnippets.class, "allocateArray", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION, TLAB_FAST_PATH_END_LOCATION);
586 private final SnippetInfo allocateArrayPIC = snippet(NewObjectSnippets.class, "allocateArrayPIC", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION, TLAB_FAST_PATH_END_LOCATION);
587 private final SnippetInfo allocatePrimitiveArrayPIC = snippet(NewObjectSnippets.class, "allocatePrimitiveArrayPIC", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION,
588 TLAB_FAST_PATH_END_LOCATION);
589 private final SnippetInfo allocateArrayDynamic = snippet(NewObjectSnippets.class, "allocateArrayDynamic", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION,
590 TLAB_FAST_PATH_END_LOCATION);
591 private final SnippetInfo allocateInstanceDynamic = snippet(NewObjectSnippets.class, "allocateInstanceDynamic", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION,
592 TLAB_FAST_PATH_END_LOCATION);
593 private final SnippetInfo newmultiarray = snippet(NewObjectSnippets.class, "newmultiarray", TLAB_TOP_LOCATION, TLAB_FAST_PATH_END_LOCATION);
594 private final SnippetInfo newmultiarrayPIC = snippet(NewObjectSnippets.class, "newmultiarrayPIC", TLAB_TOP_LOCATION, TLAB_FAST_PATH_END_LOCATION);
595 private final SnippetInfo verifyHeap = snippet(NewObjectSnippets.class, "verifyHeap");
596 private final GraalHotSpotVMConfig config;
597 private final Counters counters;
598
599 public Templates(OptionValues options, Iterable<DebugHandlersFactory> factories, SnippetCounter.Group.Factory factory, HotSpotProviders providers, TargetDescription target,
600 GraalHotSpotVMConfig config) {
601 super(options, factories, providers, providers.getSnippetReflection(), target);
602 this.config = config;
603 counters = new Counters(factory);
604 }
605
606 /**
607 * Lowers a {@link NewInstanceNode}.
608 */
609 public void lower(NewInstanceNode newInstanceNode, HotSpotRegistersProvider registers, LoweringTool tool) {
610 StructuredGraph graph = newInstanceNode.graph();
611 HotSpotResolvedObjectType type = (HotSpotResolvedObjectType) newInstanceNode.instanceClass();
612 assert !type.isArray();
613 ConstantNode hub = ConstantNode.forConstant(KlassPointerStamp.klassNonNull(), type.klass(), providers.getMetaAccess(), graph);
614 int size = instanceSize(type);
|