7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23
24
25 package org.graalvm.compiler.hotspot.replacements;
26
27 import static jdk.vm.ci.meta.DeoptimizationAction.None;
28 import static jdk.vm.ci.meta.DeoptimizationReason.RuntimeConstraint;
29 import static org.graalvm.compiler.core.common.GraalOptions.GeneratePIC;
30 import static org.graalvm.compiler.core.common.GraalOptions.MinimalBulkZeroingSize;
31 import static org.graalvm.compiler.core.common.calc.UnsignedMath.belowThan;
32 import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfigBase.INJECTED_OPTIONVALUES;
33 import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfigBase.INJECTED_VMCONFIG;
34 import static org.graalvm.compiler.hotspot.HotSpotBackend.NEW_ARRAY;
35 import static org.graalvm.compiler.hotspot.HotSpotBackend.NEW_ARRAY_OR_NULL;
36 import static org.graalvm.compiler.hotspot.HotSpotBackend.NEW_INSTANCE;
37 import static org.graalvm.compiler.hotspot.HotSpotBackend.NEW_INSTANCE_OR_NULL;
38 import static org.graalvm.compiler.hotspot.HotSpotBackend.NEW_MULTI_ARRAY;
39 import static org.graalvm.compiler.hotspot.HotSpotBackend.NEW_MULTI_ARRAY_OR_NULL;
40 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.CLASS_ARRAY_KLASS_LOCATION;
41 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.CLASS_STATE_LOCATION;
42 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.HUB_WRITE_LOCATION;
43 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.MARK_WORD_LOCATION;
44 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.PROTOTYPE_MARK_WORD_LOCATION;
45 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_END_LOCATION;
46 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_TOP_LOCATION;
47 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.allocateInstancePrefetchLines;
48 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.allocatePrefetchDistance;
49 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.allocatePrefetchLines;
50 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.allocatePrefetchStepSize;
51 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.allocatePrefetchStyle;
52 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayAllocationSize;
53 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayKlassOffset;
54 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayLengthOffset;
55 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.initializeObjectHeader;
56 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.instanceHeaderSize;
57 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.isInstanceKlassFullyInitialized;
58 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperHeaderSizeMask;
59 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperHeaderSizeShift;
60 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperLog2ElementSizeMask;
61 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperLog2ElementSizeShift;
62 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.loadKlassFromObject;
63 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.prototypeMarkWordOffset;
64 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readLayoutHelper;
65 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readTlabEnd;
66 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readTlabTop;
67 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.registerAsWord;
68 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.useBiasedLocking;
69 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.useTLAB;
70 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.verifyOop;
71 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.writeTlabTop;
72 import static org.graalvm.compiler.hotspot.replacements.HotspotSnippetsOptions.ProfileAllocations;
73 import static org.graalvm.compiler.hotspot.replacements.HotspotSnippetsOptions.ProfileAllocationsContext;
74 import static org.graalvm.compiler.nodes.PiArrayNode.piArrayCastToSnippetReplaceeStamp;
75 import static org.graalvm.compiler.nodes.PiNode.piCastToSnippetReplaceeStamp;
76 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.FAST_PATH_PROBABILITY;
77 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.FREQUENT_PROBABILITY;
78 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.SLOW_PATH_PROBABILITY;
79 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.probability;
80 import static org.graalvm.compiler.replacements.ReplacementsUtil.REPLACEMENTS_ASSERTIONS_ENABLED;
81 import static org.graalvm.compiler.replacements.ReplacementsUtil.runtimeAssert;
82 import static org.graalvm.compiler.replacements.ReplacementsUtil.staticAssert;
83 import static org.graalvm.compiler.replacements.SnippetTemplate.DEFAULT_REPLACER;
84 import static org.graalvm.compiler.replacements.nodes.CStringConstant.cstring;
85 import static org.graalvm.compiler.replacements.nodes.ExplodeLoopNode.explodeLoop;
86
87 import org.graalvm.compiler.api.replacements.Fold;
88 import org.graalvm.compiler.api.replacements.Fold.InjectedParameter;
89 import org.graalvm.compiler.api.replacements.Snippet;
90 import org.graalvm.compiler.api.replacements.Snippet.ConstantParameter;
91 import org.graalvm.compiler.api.replacements.Snippet.VarargsParameter;
92 import org.graalvm.compiler.core.common.spi.ForeignCallDescriptor;
93 import org.graalvm.compiler.core.common.type.StampFactory;
94 import org.graalvm.compiler.debug.DebugHandlersFactory;
95 import org.graalvm.compiler.debug.GraalError;
96 import org.graalvm.compiler.graph.Node.ConstantNodeParameter;
97 import org.graalvm.compiler.graph.Node.NodeIntrinsic;
98 import org.graalvm.compiler.hotspot.GraalHotSpotVMConfig;
99 import org.graalvm.compiler.hotspot.meta.HotSpotProviders;
100 import org.graalvm.compiler.hotspot.meta.HotSpotRegistersProvider;
101 import org.graalvm.compiler.hotspot.nodes.DimensionsNode;
102 import org.graalvm.compiler.hotspot.nodes.aot.LoadConstantIndirectlyFixedNode;
103 import org.graalvm.compiler.hotspot.nodes.aot.LoadConstantIndirectlyNode;
104 import org.graalvm.compiler.hotspot.nodes.type.KlassPointerStamp;
105 import org.graalvm.compiler.hotspot.word.KlassPointer;
106 import org.graalvm.compiler.nodes.ConstantNode;
107 import org.graalvm.compiler.nodes.DeoptimizeNode;
108 import org.graalvm.compiler.nodes.PiNode;
109 import org.graalvm.compiler.nodes.PrefetchAllocateNode;
110 import org.graalvm.compiler.nodes.SnippetAnchorNode;
111 import org.graalvm.compiler.nodes.StructuredGraph;
112 import org.graalvm.compiler.nodes.ValueNode;
113 import org.graalvm.compiler.nodes.debug.DynamicCounterNode;
114 import org.graalvm.compiler.nodes.debug.VerifyHeapNode;
115 import org.graalvm.compiler.nodes.extended.ForeignCallNode;
116 import org.graalvm.compiler.nodes.extended.MembarNode;
117 import org.graalvm.compiler.nodes.java.DynamicNewArrayNode;
118 import org.graalvm.compiler.nodes.java.DynamicNewInstanceNode;
119 import org.graalvm.compiler.nodes.java.NewArrayNode;
120 import org.graalvm.compiler.nodes.java.NewInstanceNode;
121 import org.graalvm.compiler.nodes.java.NewMultiArrayNode;
122 import org.graalvm.compiler.nodes.memory.address.OffsetAddressNode;
123 import org.graalvm.compiler.nodes.spi.LoweringTool;
124 import org.graalvm.compiler.nodes.util.GraphUtil;
125 import org.graalvm.compiler.options.OptionValues;
126 import org.graalvm.compiler.replacements.ReplacementsUtil;
127 import org.graalvm.compiler.replacements.SnippetCounter;
128 import org.graalvm.compiler.replacements.SnippetCounter.Group;
129 import org.graalvm.compiler.replacements.SnippetTemplate;
130 import org.graalvm.compiler.replacements.SnippetTemplate.AbstractTemplates;
131 import org.graalvm.compiler.replacements.SnippetTemplate.Arguments;
132 import org.graalvm.compiler.replacements.SnippetTemplate.SnippetInfo;
133 import org.graalvm.compiler.replacements.Snippets;
134 import org.graalvm.compiler.replacements.nodes.ExplodeLoopNode;
273 Word prototypeMarkWord,
274 @ConstantParameter boolean fillContents,
275 @ConstantParameter boolean emitMemoryBarrier,
276 @ConstantParameter Register threadRegister,
277 @ConstantParameter boolean constantSize,
278 @ConstantParameter String typeContext,
279 @ConstantParameter Counters counters) {
280 // Klass must be initialized by the time the first instance is allocated, therefore we can
281 // just load it from the corresponding cell and avoid the resolution check. We have to use a
282 // fixed load though, to prevent it from floating above the initialization.
283 KlassPointer picHub = LoadConstantIndirectlyFixedNode.loadKlass(hub);
284 return piCastToSnippetReplaceeStamp(allocateInstanceHelper(size, picHub, prototypeMarkWord, fillContents, emitMemoryBarrier, threadRegister, constantSize, typeContext, counters));
285 }
286
287 @Snippet
288 public static Object allocateInstanceDynamic(Class<?> type, Class<?> classClass,
289 @ConstantParameter boolean fillContents,
290 @ConstantParameter boolean emitMemoryBarrier,
291 @ConstantParameter Register threadRegister,
292 @ConstantParameter Counters counters) {
293 if (probability(SLOW_PATH_PROBABILITY, type == null)) {
294 DeoptimizeNode.deopt(None, RuntimeConstraint);
295 }
296 Class<?> nonNullType = PiNode.piCastNonNullClass(type, SnippetAnchorNode.anchor());
297
298 if (probability(SLOW_PATH_PROBABILITY, DynamicNewInstanceNode.throwsInstantiationException(type, classClass))) {
299 DeoptimizeNode.deopt(None, RuntimeConstraint);
300 }
301
302 return PiNode.piCastToSnippetReplaceeStamp(allocateInstanceDynamicHelper(type, fillContents, emitMemoryBarrier, threadRegister, counters, nonNullType));
303 }
304
305 private static Object allocateInstanceDynamicHelper(Class<?> type,
306 boolean fillContents,
307 boolean emitMemoryBarrier,
308 Register threadRegister,
309 Counters counters,
310 Class<?> nonNullType) {
311 KlassPointer hub = ClassGetHubNode.readClass(nonNullType);
312 if (probability(FAST_PATH_PROBABILITY, !hub.isNull())) {
313 KlassPointer nonNullHub = ClassGetHubNode.piCastNonNull(hub, SnippetAnchorNode.anchor());
314
315 if (probability(FAST_PATH_PROBABILITY, isInstanceKlassFullyInitialized(nonNullHub))) {
316 int layoutHelper = readLayoutHelper(nonNullHub);
317 /*
318 * src/share/vm/oops/klass.hpp: For instances, layout helper is a positive number,
319 * the instance size. This size is already passed through align_object_size and
320 * scaled to bytes. The low order bit is set if instances of this class cannot be
321 * allocated using the fastpath.
322 */
323 if (probability(FAST_PATH_PROBABILITY, (layoutHelper & 1) == 0)) {
324 Word prototypeMarkWord = nonNullHub.readWord(prototypeMarkWordOffset(INJECTED_VMCONFIG), PROTOTYPE_MARK_WORD_LOCATION);
325 /*
326 * FIXME(je,ds): we should actually pass typeContext instead of "" but late
327 * binding of parameters is not yet supported by the GraphBuilderPlugin system.
328 */
329 return allocateInstanceHelper(layoutHelper, nonNullHub, prototypeMarkWord, fillContents, emitMemoryBarrier, threadRegister, false, "", counters);
330 }
331 } else {
332 DeoptimizeNode.deopt(None, RuntimeConstraint);
333 }
334 }
335 return dynamicNewInstanceStub(type);
336 }
337
338 /**
339 * Maximum array length for which fast path allocation is used.
340 */
341 public static final int MAX_ARRAY_FAST_PATH_ALLOCATION_LENGTH = 0x00FFFFFF;
342
343 @Snippet
344 public static Object allocatePrimitiveArrayPIC(KlassPointer hub,
345 int length,
346 Word prototypeMarkWord,
347 @ConstantParameter int headerSize,
348 @ConstantParameter int log2ElementSize,
349 @ConstantParameter boolean fillContents,
350 @ConstantParameter boolean emitMemoryBarrier,
351 @ConstantParameter Register threadRegister,
352 @ConstantParameter boolean maybeUnroll,
353 @ConstantParameter String typeContext,
354 @ConstantParameter boolean useBulkZeroing,
355 @ConstantParameter Counters counters) {
356 // Primitive array types are eagerly pre-resolved. We can use a floating load.
357 KlassPointer picHub = LoadConstantIndirectlyNode.loadKlass(hub);
358 return allocateArrayImpl(picHub, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents,
359 emitMemoryBarrier, threadRegister, maybeUnroll, typeContext, useBulkZeroing, counters);
360 }
361
362 @Snippet
363 public static Object allocateArrayPIC(KlassPointer hub,
364 int length,
365 Word prototypeMarkWord,
366 @ConstantParameter int headerSize,
367 @ConstantParameter int log2ElementSize,
368 @ConstantParameter boolean fillContents,
369 @ConstantParameter boolean emitMemoryBarrier,
370 @ConstantParameter Register threadRegister,
371 @ConstantParameter boolean maybeUnroll,
372 @ConstantParameter String typeContext,
373 @ConstantParameter boolean useBulkZeroing,
374 @ConstantParameter Counters counters) {
375 // Array type would be resolved by dominating resolution.
376 KlassPointer picHub = LoadConstantIndirectlyFixedNode.loadKlass(hub);
377 return allocateArrayImpl(picHub, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents,
378 emitMemoryBarrier, threadRegister, maybeUnroll, typeContext, useBulkZeroing, counters);
379 }
380
381 @Snippet
382 public static Object allocateArray(KlassPointer hub,
383 int length,
384 Word prototypeMarkWord,
385 @ConstantParameter int headerSize,
386 @ConstantParameter int log2ElementSize,
387 @ConstantParameter boolean fillContents,
388 @ConstantParameter boolean emitMemoryBarrier,
389 @ConstantParameter Register threadRegister,
390 @ConstantParameter boolean maybeUnroll,
391 @ConstantParameter String typeContext,
392 @ConstantParameter boolean useBulkZeroing,
393 @ConstantParameter Counters counters) {
394 Object result = allocateArrayImpl(hub,
395 length,
396 prototypeMarkWord,
397 headerSize,
398 log2ElementSize,
399 fillContents,
400 emitMemoryBarrier, threadRegister,
401 maybeUnroll,
402 typeContext,
403 useBulkZeroing,
404 counters);
405 return piArrayCastToSnippetReplaceeStamp(verifyOop(result), length);
406 }
407
408 /**
409 * When allocating on the slow path, determines whether to use a version of the runtime call
410 * that returns {@code null} on a failed allocation instead of raising an OutOfMemoryError.
411 */
412 @Fold
413 static boolean useNullAllocationStubs(@InjectedParameter GraalHotSpotVMConfig config) {
414 return config.areNullAllocationStubsAvailable();
415 }
416
417 private static Object allocateArrayImpl(KlassPointer hub,
418 int length,
419 Word prototypeMarkWord,
420 int headerSize,
421 int log2ElementSize,
422 boolean fillContents,
423 boolean emitMemoryBarrier,
424 Register threadRegister,
425 boolean maybeUnroll,
426 String typeContext,
427 boolean useBulkZeroing,
428 Counters counters) {
429 Object result;
430 long allocationSize = arrayAllocationSize(length, headerSize, log2ElementSize);
431 Word thread = registerAsWord(threadRegister);
432 Word top = readTlabTop(thread);
433 Word end = readTlabEnd(thread);
434 Word newTop = top.add(WordFactory.unsigned(allocationSize));
435 if (probability(FREQUENT_PROBABILITY, belowThan(length, MAX_ARRAY_FAST_PATH_ALLOCATION_LENGTH)) && useTLAB(INJECTED_VMCONFIG) &&
436 probability(FAST_PATH_PROBABILITY, newTop.belowOrEqual(end))) {
437 writeTlabTop(thread, newTop);
438 emitPrefetchAllocate(newTop, true);
439 Counters theCounters = counters;
440 if (theCounters != null && theCounters.arrayLoopInit != null) {
441 theCounters.arrayLoopInit.inc();
442 }
443 result = formatArray(hub, allocationSize, length, headerSize, top, prototypeMarkWord, fillContents, emitMemoryBarrier, maybeUnroll, useBulkZeroing, counters);
444 } else {
445 result = newArrayStub(hub, length);
446 }
447 profileAllocation("array", allocationSize, typeContext);
448 return result;
449 }
450
451 public static Object newArrayStub(KlassPointer hub, int length) {
452 if (useNullAllocationStubs(INJECTED_VMCONFIG)) {
453 return nonNullOrDeopt(newArrayOrNull(NEW_ARRAY_OR_NULL, hub, length));
454 } else {
455 return newArray(NEW_ARRAY, hub, length);
456 }
457 }
458
459 @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = true)
460 private static native Object newArray(@ConstantNodeParameter ForeignCallDescriptor descriptor, KlassPointer hub, int length);
461
462 @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = false)
463 private static native Object newArrayOrNull(@ConstantNodeParameter ForeignCallDescriptor descriptor, KlassPointer hub, int length);
467 */
468 public static final ForeignCallDescriptor DYNAMIC_NEW_INSTANCE = new ForeignCallDescriptor("dynamic_new_instance", Object.class, Class.class);
469
470 /**
471 * New dynamic array stub that returns null on allocation failure.
472 */
473 public static final ForeignCallDescriptor DYNAMIC_NEW_INSTANCE_OR_NULL = new ForeignCallDescriptor("dynamic_new_instance_or_null", Object.class, Class.class);
474
475 public static Object dynamicNewInstanceStub(Class<?> elementType) {
476 if (useNullAllocationStubs(INJECTED_VMCONFIG)) {
477 return nonNullOrDeopt(dynamicNewInstanceOrNull(DYNAMIC_NEW_INSTANCE_OR_NULL, elementType));
478 } else {
479 return dynamicNewInstance(DYNAMIC_NEW_INSTANCE, elementType);
480 }
481 }
482
483 /**
484 * Deoptimizes if {@code obj == null} otherwise returns {@code obj}.
485 */
486 private static Object nonNullOrDeopt(Object obj) {
487 if (obj == null) {
488 DeoptimizeNode.deopt(None, RuntimeConstraint);
489 }
490 return obj;
491 }
492
493 @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = true)
494 public static native Object dynamicNewInstance(@ConstantNodeParameter ForeignCallDescriptor descriptor, Class<?> elementType);
495
496 @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = false)
497 public static native Object dynamicNewInstanceOrNull(@ConstantNodeParameter ForeignCallDescriptor descriptor, Class<?> elementType);
498
499 @Snippet
500 public static Object allocateArrayDynamic(Class<?> elementType,
501 Class<?> voidClass,
502 int length,
503 @ConstantParameter boolean fillContents,
504 @ConstantParameter boolean emitMemoryBarrier,
505 @ConstantParameter Register threadRegister,
506 @ConstantParameter JavaKind knownElementKind,
507 @ConstantParameter int knownLayoutHelper,
508 @ConstantParameter boolean useBulkZeroing,
509 Word prototypeMarkWord,
510 @ConstantParameter Counters counters) {
511 Object result = allocateArrayDynamicImpl(elementType, voidClass, length, fillContents, emitMemoryBarrier, threadRegister, knownElementKind,
512 knownLayoutHelper, useBulkZeroing, prototypeMarkWord, counters);
513 return result;
514 }
515
516 private static Object allocateArrayDynamicImpl(Class<?> elementType,
517 Class<?> voidClass,
518 int length,
519 boolean fillContents,
520 boolean emitMemoryBarrier,
521 Register threadRegister,
522 JavaKind knownElementKind,
523 int knownLayoutHelper,
524 boolean useBulkZeroing,
525 Word prototypeMarkWord,
526 Counters counters) {
527 /*
528 * We only need the dynamic check for void when we have no static information from
529 * knownElementKind.
530 */
531 staticAssert(knownElementKind != JavaKind.Void, "unsupported knownElementKind");
532 if (knownElementKind == JavaKind.Illegal && probability(SLOW_PATH_PROBABILITY, elementType == null || DynamicNewArrayNode.throwsIllegalArgumentException(elementType, voidClass))) {
533 DeoptimizeNode.deopt(None, RuntimeConstraint);
534 }
535
536 KlassPointer klass = loadKlassFromObject(elementType, arrayKlassOffset(INJECTED_VMCONFIG), CLASS_ARRAY_KLASS_LOCATION);
537 if (klass.isNull()) {
538 DeoptimizeNode.deopt(None, RuntimeConstraint);
539 }
540 KlassPointer nonNullKlass = ClassGetHubNode.piCastNonNull(klass, SnippetAnchorNode.anchor());
541
542 if (length < 0) {
543 DeoptimizeNode.deopt(None, RuntimeConstraint);
544 }
545 int layoutHelper;
546 if (knownElementKind == JavaKind.Illegal) {
547 layoutHelper = readLayoutHelper(nonNullKlass);
548 } else {
549 runtimeAssert(knownLayoutHelper == readLayoutHelper(nonNullKlass), "layout mismatch");
550 layoutHelper = knownLayoutHelper;
551 }
552 //@formatter:off
553 // from src/share/vm/oops/klass.hpp:
554 //
555 // For arrays, layout helper is a negative number, containing four
556 // distinct bytes, as follows:
557 // MSB:[tag, hsz, ebt, log2(esz)]:LSB
558 // where:
559 // tag is 0x80 if the elements are oops, 0xC0 if non-oops
560 // hsz is array header size in bytes (i.e., offset of first element)
561 // ebt is the BasicType of the elements
562 // esz is the element size in bytes
563 //@formatter:on
564
565 int headerSize = (layoutHelper >> layoutHelperHeaderSizeShift(INJECTED_VMCONFIG)) & layoutHelperHeaderSizeMask(INJECTED_VMCONFIG);
566 int log2ElementSize = (layoutHelper >> layoutHelperLog2ElementSizeShift(INJECTED_VMCONFIG)) & layoutHelperLog2ElementSizeMask(INJECTED_VMCONFIG);
567
568 Object result = allocateArrayImpl(nonNullKlass, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents,
569 emitMemoryBarrier, threadRegister, false, "dynamic type", useBulkZeroing, counters);
570 return piArrayCastToSnippetReplaceeStamp(verifyOop(result), length);
571 }
572
573 /**
574 * Calls the runtime stub for implementing MULTIANEWARRAY.
575 */
576 @Snippet
577 private static Object newmultiarray(KlassPointer hub, @ConstantParameter int rank, @VarargsParameter int[] dimensions) {
578 Word dims = DimensionsNode.allocaDimsArray(rank);
579 ExplodeLoopNode.explodeLoop();
580 for (int i = 0; i < rank; i++) {
581 dims.writeInt(i * 4, dimensions[i], LocationIdentity.init());
582 }
583 return newMultiArrayStub(hub, rank, dims);
584 }
585
586 private static Object newMultiArrayStub(KlassPointer hub, int rank, Word dims) {
587 if (useNullAllocationStubs(INJECTED_VMCONFIG)) {
588 return nonNullOrDeopt(newMultiArrayOrNull(NEW_MULTI_ARRAY_OR_NULL, hub, rank, dims));
589 } else {
597 KlassPointer picHub = LoadConstantIndirectlyFixedNode.loadKlass(hub);
598 return newmultiarray(picHub, rank, dimensions);
599 }
600
601 @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = true)
602 private static native Object newMultiArray(@ConstantNodeParameter ForeignCallDescriptor descriptor, KlassPointer hub, int rank, Word dims);
603
604 @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = false)
605 private static native Object newMultiArrayOrNull(@ConstantNodeParameter ForeignCallDescriptor descriptor, KlassPointer hub, int rank, Word dims);
606
607 /**
608 * Maximum number of long stores to emit when zeroing an object with a constant size. Larger
609 * objects have their bodies initialized in a loop.
610 */
611 private static final int MAX_UNROLLED_OBJECT_ZEROING_STORES = 8;
612
613 /**
614 * Zero uninitialized memory in a newly allocated object, unrolling as necessary and ensuring
615 * that stores are aligned.
616 *
617 * @param size number of bytes to zero
618 * @param memory beginning of object which is being zeroed
619 * @param constantSize is {@code size} known to be constant in the snippet
620 * @param startOffset offset to begin zeroing. May not be word aligned.
621 * @param manualUnroll maximally unroll zeroing
622 * @param useBulkZeroing apply bulk zeroing
623 */
624 private static void zeroMemory(long size, Word memory, boolean constantSize, int startOffset, boolean manualUnroll,
625 boolean useBulkZeroing, Counters counters) {
626 fillMemory(0, size, memory, constantSize, startOffset, manualUnroll, useBulkZeroing, counters);
627 }
628
629 private static void fillMemory(long value, long size, Word memory, boolean constantSize, int startOffset, boolean manualUnroll,
630 boolean useBulkZeroing, Counters counters) {
631 ReplacementsUtil.runtimeAssert((size & 0x7) == 0, "unaligned object size");
632 int offset = startOffset;
633 if ((offset & 0x7) != 0) {
634 memory.writeInt(offset, (int) value, LocationIdentity.init());
635 offset += 4;
636 }
637 ReplacementsUtil.runtimeAssert((offset & 0x7) == 0, "unaligned offset");
638 Counters theCounters = counters;
639 if (manualUnroll && ((size - offset) / 8) <= MAX_UNROLLED_OBJECT_ZEROING_STORES) {
640 ReplacementsUtil.staticAssert(!constantSize, "size shouldn't be constant at instantiation time");
641 // This case handles arrays of constant length. Instead of having a snippet variant for
642 // each length, generate a chain of stores of maximum length. Once it's inlined the
643 // break statement will trim excess stores.
644 if (theCounters != null && theCounters.instanceSeqInit != null) {
645 theCounters.instanceSeqInit.inc();
646 }
647
648 explodeLoop();
649 for (int i = 0; i < MAX_UNROLLED_OBJECT_ZEROING_STORES; i++, offset += 8) {
650 if (offset == size) {
651 break;
652 }
653 memory.initializeLong(offset, value, LocationIdentity.init());
654 }
655 } else {
656 // Use Word instead of int to avoid extension to long in generated code
657 Word off = WordFactory.signed(offset);
658 if (useBulkZeroing && value == 0 && probability(SLOW_PATH_PROBABILITY, (size - offset) >= getMinimalBulkZeroingSize(INJECTED_OPTIONVALUES))) {
659 if (theCounters != null && theCounters.instanceBulkInit != null) {
660 theCounters.instanceBulkInit.inc();
661 }
662 ZeroMemoryNode.zero(memory.add(off), size - offset, LocationIdentity.init());
663 } else {
664 if (constantSize && ((size - offset) / 8) <= MAX_UNROLLED_OBJECT_ZEROING_STORES) {
665 if (theCounters != null && theCounters.instanceSeqInit != null) {
666 theCounters.instanceSeqInit.inc();
667 }
668 explodeLoop();
669 } else {
670 if (theCounters != null && theCounters.instanceLoopInit != null) {
671 theCounters.instanceLoopInit.inc();
672 }
673 }
674 for (; off.rawValue() < size; off = off.add(8)) {
675 memory.initializeLong(off, value, LocationIdentity.init());
676 }
677 }
678 }
679 }
680
681 @Fold
682 static int getMinimalBulkZeroingSize(@InjectedParameter OptionValues optionValues) {
683 return MinimalBulkZeroingSize.getValue(optionValues);
684 }
685
686 /**
687 * Fill uninitialized memory with garbage value in a newly allocated object, unrolling as
688 * necessary and ensuring that stores are aligned.
689 *
690 * @param size number of bytes to zero
691 * @param memory beginning of object which is being zeroed
692 * @param constantSize is {@code size} known to be constant in the snippet
693 * @param startOffset offset to begin zeroing. May not be word aligned.
694 * @param manualUnroll maximally unroll zeroing
695 */
696 private static void fillWithGarbage(long size, Word memory, boolean constantSize, int startOffset, boolean manualUnroll, Counters counters) {
697 fillMemory(0xfefefefefefefefeL, size, memory, constantSize, startOffset, manualUnroll, false, counters);
698 }
699
700 /**
701 * Formats some allocated memory with an object header and zeroes out the rest.
702 */
703 private static Object formatObject(KlassPointer hub,
704 long size,
705 Word memory,
706 Word compileTimePrototypeMarkWord,
707 boolean fillContents,
708 boolean emitMemoryBarrier,
709 boolean constantSize,
710 Counters counters) {
711 Word prototypeMarkWord = useBiasedLocking(INJECTED_VMCONFIG) ? hub.readWord(prototypeMarkWordOffset(INJECTED_VMCONFIG), PROTOTYPE_MARK_WORD_LOCATION) : compileTimePrototypeMarkWord;
712 initializeObjectHeader(memory, prototypeMarkWord, hub);
713 if (fillContents) {
714 zeroMemory(size, memory, constantSize, instanceHeaderSize(INJECTED_VMCONFIG), false, false, counters);
715 } else if (REPLACEMENTS_ASSERTIONS_ENABLED) {
716 fillWithGarbage(size, memory, constantSize, instanceHeaderSize(INJECTED_VMCONFIG), false, counters);
717 }
718 if (emitMemoryBarrier) {
719 MembarNode.memoryBarrier(MemoryBarriers.STORE_STORE, LocationIdentity.init());
720 }
721 return memory.toObjectNonNull();
722 }
723
724 @Snippet
725 private static void verifyHeap(@ConstantParameter Register threadRegister) {
726 Word thread = registerAsWord(threadRegister);
727 Word topValue = readTlabTop(thread);
728 if (!topValue.equal(WordFactory.zero())) {
729 Word topValueContents = topValue.readWord(0, MARK_WORD_LOCATION);
730 if (topValueContents.equal(WordFactory.zero())) {
731 AssertionSnippets.vmMessageC(AssertionSnippets.ASSERTION_VM_MESSAGE_C, true, cstring("overzeroing of TLAB detected"), 0L, 0L, 0L);
732 }
733 }
734 }
735
736 /**
737 * Formats some allocated memory with an object header and zeroes out the rest.
738 */
739 private static Object formatArray(KlassPointer hub,
740 long allocationSize,
741 int length,
742 int headerSize,
743 Word memory,
744 Word prototypeMarkWord,
745 boolean fillContents,
746 boolean emitMemoryBarrier,
747 boolean maybeUnroll,
748 boolean useBulkZeroing,
749 Counters counters) {
750 memory.writeInt(arrayLengthOffset(INJECTED_VMCONFIG), length, LocationIdentity.init());
751 /*
752 * store hub last as the concurrent garbage collectors assume length is valid if hub field
753 * is not null
754 */
755 initializeObjectHeader(memory, prototypeMarkWord, hub);
756 if (fillContents) {
757 zeroMemory(allocationSize, memory, false, headerSize, maybeUnroll, useBulkZeroing, counters);
758 } else if (REPLACEMENTS_ASSERTIONS_ENABLED) {
759 fillWithGarbage(allocationSize, memory, false, headerSize, maybeUnroll, counters);
760 }
761 if (emitMemoryBarrier) {
762 MembarNode.memoryBarrier(MemoryBarriers.STORE_STORE, LocationIdentity.init());
763 }
764 return memory.toObjectNonNull();
765 }
766
767 static class Counters {
768 Counters(SnippetCounter.Group.Factory factory) {
769 Group newInstance = factory.createSnippetCounterGroup("NewInstance");
770 Group newArray = factory.createSnippetCounterGroup("NewArray");
771 instanceSeqInit = new SnippetCounter(newInstance, "tlabSeqInit", "TLAB alloc with unrolled zeroing");
772 instanceLoopInit = new SnippetCounter(newInstance, "tlabLoopInit", "TLAB alloc with zeroing in a loop");
773 instanceBulkInit = new SnippetCounter(newArray, "tlabBulkInit", "TLAB alloc with bulk zeroing");
774 arrayLoopInit = new SnippetCounter(newArray, "tlabLoopInit", "TLAB alloc with zeroing in a loop");
775 stub = new SnippetCounter(newInstance, "stub", "alloc and zeroing via stub");
776 }
777
778 final SnippetCounter instanceSeqInit;
779 final SnippetCounter instanceLoopInit;
780 final SnippetCounter instanceBulkInit;
781 final SnippetCounter arrayLoopInit;
782 final SnippetCounter stub;
783 }
784
785 public static class Templates extends AbstractTemplates {
786
787 private final SnippetInfo allocateInstance = snippet(NewObjectSnippets.class, "allocateInstance", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION, TLAB_END_LOCATION,
788 PROTOTYPE_MARK_WORD_LOCATION);
789 private final SnippetInfo allocateInstancePIC = snippet(NewObjectSnippets.class, "allocateInstancePIC", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION,
790 TLAB_END_LOCATION, PROTOTYPE_MARK_WORD_LOCATION);
791 private final SnippetInfo allocateArray = snippet(NewObjectSnippets.class, "allocateArray", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION, TLAB_END_LOCATION);
792 private final SnippetInfo allocateArrayPIC = snippet(NewObjectSnippets.class, "allocateArrayPIC", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION, TLAB_END_LOCATION);
793 private final SnippetInfo allocatePrimitiveArrayPIC = snippet(NewObjectSnippets.class, "allocatePrimitiveArrayPIC", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION,
794 TLAB_END_LOCATION);
795 private final SnippetInfo allocateArrayDynamic = snippet(NewObjectSnippets.class, "allocateArrayDynamic", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION,
796 TLAB_END_LOCATION);
797 private final SnippetInfo allocateInstanceDynamic = snippet(NewObjectSnippets.class, "allocateInstanceDynamic", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION,
798 TLAB_END_LOCATION, PROTOTYPE_MARK_WORD_LOCATION, CLASS_STATE_LOCATION);
799 private final SnippetInfo newmultiarray = snippet(NewObjectSnippets.class, "newmultiarray", TLAB_TOP_LOCATION, TLAB_END_LOCATION);
800 private final SnippetInfo newmultiarrayPIC = snippet(NewObjectSnippets.class, "newmultiarrayPIC", TLAB_TOP_LOCATION, TLAB_END_LOCATION);
801 private final SnippetInfo verifyHeap = snippet(NewObjectSnippets.class, "verifyHeap");
802 private final GraalHotSpotVMConfig config;
803 private final Counters counters;
804
805 public Templates(OptionValues options, Iterable<DebugHandlersFactory> factories, SnippetCounter.Group.Factory factory, HotSpotProviders providers, TargetDescription target,
806 GraalHotSpotVMConfig config) {
807 super(options, factories, providers, providers.getSnippetReflection(), target);
808 this.config = config;
809 counters = new Counters(factory);
810 }
811
812 /**
813 * Lowers a {@link NewInstanceNode}.
814 */
815 public void lower(NewInstanceNode newInstanceNode, HotSpotRegistersProvider registers, LoweringTool tool) {
816 StructuredGraph graph = newInstanceNode.graph();
817 HotSpotResolvedObjectType type = (HotSpotResolvedObjectType) newInstanceNode.instanceClass();
818 assert !type.isArray();
819 ConstantNode hub = ConstantNode.forConstant(KlassPointerStamp.klassNonNull(), type.klass(), providers.getMetaAccess(), graph);
820 long size = instanceSize(type);
821
857 } else {
858 snippet = allocateArrayPIC;
859 }
860 } else {
861 snippet = allocateArray;
862 }
863
864 Arguments args = new Arguments(snippet, graph.getGuardsStage(), tool.getLoweringStage());
865 args.add("hub", hub);
866 ValueNode length = newArrayNode.length();
867 args.add("length", length.isAlive() ? length : graph.addOrUniqueWithInputs(length));
868 assert arrayType.prototypeMarkWord() == lookupArrayClass(tool, JavaKind.Object).prototypeMarkWord() : "all array types are assumed to have the same prototypeMarkWord";
869 args.add("prototypeMarkWord", arrayType.prototypeMarkWord());
870 args.addConst("headerSize", headerSize);
871 args.addConst("log2ElementSize", log2ElementSize);
872 args.addConst("fillContents", newArrayNode.fillContents());
873 args.addConst("emitMemoryBarrier", newArrayNode.emitMemoryBarrier());
874 args.addConst("threadRegister", registers.getThreadRegister());
875 args.addConst("maybeUnroll", length.isConstant());
876 args.addConst("typeContext", ProfileAllocations.getValue(localOptions) ? arrayType.toJavaName(false) : "");
877 args.addConst("useBulkZeroing", tool.getLowerer().supportBulkZeroing());
878 args.addConst("counters", counters);
879 SnippetTemplate template = template(newArrayNode, args);
880 graph.getDebug().log("Lowering allocateArray in %s: node=%s, template=%s, arguments=%s", graph, newArrayNode, template, args);
881 template.instantiate(providers.getMetaAccess(), newArrayNode, DEFAULT_REPLACER, args);
882 }
883
884 public void lower(DynamicNewInstanceNode newInstanceNode, HotSpotRegistersProvider registers, LoweringTool tool) {
885 Arguments args = new Arguments(allocateInstanceDynamic, newInstanceNode.graph().getGuardsStage(), tool.getLoweringStage());
886 args.add("type", newInstanceNode.getInstanceType());
887 ValueNode classClass = newInstanceNode.getClassClass();
888 assert classClass != null;
889 args.add("classClass", classClass);
890 args.addConst("fillContents", newInstanceNode.fillContents());
891 args.addConst("emitMemoryBarrier", newInstanceNode.emitMemoryBarrier());
892 args.addConst("threadRegister", registers.getThreadRegister());
893 args.addConst("counters", counters);
894
895 SnippetTemplate template = template(newInstanceNode, args);
896 template.instantiate(providers.getMetaAccess(), newInstanceNode, DEFAULT_REPLACER, args);
897 }
901 Arguments args = new Arguments(allocateArrayDynamic, graph.getGuardsStage(), tool.getLoweringStage());
902 args.add("elementType", newArrayNode.getElementType());
903 ValueNode voidClass = newArrayNode.getVoidClass();
904 assert voidClass != null;
905 args.add("voidClass", voidClass);
906 ValueNode length = newArrayNode.length();
907 args.add("length", length.isAlive() ? length : graph.addOrUniqueWithInputs(length));
908 args.addConst("fillContents", newArrayNode.fillContents());
909 args.addConst("emitMemoryBarrier", newArrayNode.emitMemoryBarrier());
910 args.addConst("threadRegister", registers.getThreadRegister());
911 /*
912 * We use Kind.Illegal as a marker value instead of null because constant snippet
913 * parameters cannot be null.
914 */
915 args.addConst("knownElementKind", newArrayNode.getKnownElementKind() == null ? JavaKind.Illegal : newArrayNode.getKnownElementKind());
916 if (newArrayNode.getKnownElementKind() != null) {
917 args.addConst("knownLayoutHelper", lookupArrayClass(tool, newArrayNode.getKnownElementKind()).layoutHelper());
918 } else {
919 args.addConst("knownLayoutHelper", 0);
920 }
921 args.addConst("useBulkZeroing", tool.getLowerer().supportBulkZeroing());
922 args.add("prototypeMarkWord", lookupArrayClass(tool, JavaKind.Object).prototypeMarkWord());
923 args.addConst("counters", counters);
924 SnippetTemplate template = template(newArrayNode, args);
925 template.instantiate(providers.getMetaAccess(), newArrayNode, DEFAULT_REPLACER, args);
926 }
927
928 private static HotSpotResolvedObjectType lookupArrayClass(LoweringTool tool, JavaKind kind) {
929 return (HotSpotResolvedObjectType) tool.getMetaAccess().lookupJavaType(kind == JavaKind.Object ? Object.class : kind.toJavaClass()).getArrayClass();
930 }
931
932 public void lower(NewMultiArrayNode newmultiarrayNode, LoweringTool tool) {
933 StructuredGraph graph = newmultiarrayNode.graph();
934 OptionValues localOptions = graph.getOptions();
935 int rank = newmultiarrayNode.dimensionCount();
936 ValueNode[] dims = new ValueNode[rank];
937 for (int i = 0; i < newmultiarrayNode.dimensionCount(); i++) {
938 dims[i] = newmultiarrayNode.dimension(i);
939 }
940 HotSpotResolvedObjectType type = (HotSpotResolvedObjectType) newmultiarrayNode.type();
941 ConstantNode hub = ConstantNode.forConstant(KlassPointerStamp.klassNonNull(), type.klass(), providers.getMetaAccess(), graph);
947 args.addVarargs("dimensions", int.class, StampFactory.forKind(JavaKind.Int), dims);
948 template(newmultiarrayNode, args).instantiate(providers.getMetaAccess(), newmultiarrayNode, DEFAULT_REPLACER, args);
949 }
950
951 private static long instanceSize(HotSpotResolvedObjectType type) {
952 long size = type.instanceSize();
953 assert size >= 0;
954 return size;
955 }
956
957 public void lower(VerifyHeapNode verifyHeapNode, HotSpotRegistersProvider registers, LoweringTool tool) {
958 if (config.cAssertions) {
959 Arguments args = new Arguments(verifyHeap, verifyHeapNode.graph().getGuardsStage(), tool.getLoweringStage());
960 args.addConst("threadRegister", registers.getThreadRegister());
961
962 SnippetTemplate template = template(verifyHeapNode, args);
963 template.instantiate(providers.getMetaAccess(), verifyHeapNode, DEFAULT_REPLACER, args);
964 } else {
965 GraphUtil.removeFixedWithUnusedInputs(verifyHeapNode);
966 }
967 }
968 }
969 }
|
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23
24
25 package org.graalvm.compiler.hotspot.replacements;
26
27 import static jdk.vm.ci.meta.DeoptimizationAction.InvalidateRecompile;
28 import static jdk.vm.ci.meta.DeoptimizationAction.None;
29 import static jdk.vm.ci.meta.DeoptimizationReason.RuntimeConstraint;
30 import static org.graalvm.compiler.core.common.GraalOptions.GeneratePIC;
31 import static org.graalvm.compiler.core.common.GraalOptions.MinimalBulkZeroingSize;
32 import static org.graalvm.compiler.core.common.calc.UnsignedMath.belowThan;
33 import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfigBase.INJECTED_OPTIONVALUES;
34 import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfigBase.INJECTED_VMCONFIG;
35 import static org.graalvm.compiler.hotspot.HotSpotBackend.NEW_ARRAY;
36 import static org.graalvm.compiler.hotspot.HotSpotBackend.NEW_ARRAY_OR_NULL;
37 import static org.graalvm.compiler.hotspot.HotSpotBackend.NEW_INSTANCE;
38 import static org.graalvm.compiler.hotspot.HotSpotBackend.NEW_INSTANCE_OR_NULL;
39 import static org.graalvm.compiler.hotspot.HotSpotBackend.NEW_MULTI_ARRAY;
40 import static org.graalvm.compiler.hotspot.HotSpotBackend.NEW_MULTI_ARRAY_OR_NULL;
41 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.CLASS_ARRAY_KLASS_LOCATION;
42 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.CLASS_INIT_STATE_LOCATION;
43 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.HUB_WRITE_LOCATION;
44 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.MARK_WORD_LOCATION;
45 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.PROTOTYPE_MARK_WORD_LOCATION;
46 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_END_LOCATION;
47 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_TOP_LOCATION;
48 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.allocateInstancePrefetchLines;
49 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.allocatePrefetchDistance;
50 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.allocatePrefetchLines;
51 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.allocatePrefetchStepSize;
52 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.allocatePrefetchStyle;
53 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayAllocationSize;
54 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayKlassOffset;
55 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayLengthOffset;
56 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.initializeObjectHeader;
57 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.instanceHeaderSize;
58 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.instanceKlassStateBeingInitialized;
59 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.isInstanceKlassFullyInitialized;
60 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperHeaderSizeMask;
61 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperHeaderSizeShift;
62 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperLog2ElementSizeMask;
63 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperLog2ElementSizeShift;
64 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.loadKlassFromObject;
65 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.prototypeMarkWordOffset;
66 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readInstanceKlassInitState;
67 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readInstanceKlassInitThread;
68 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readLayoutHelper;
69 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readTlabEnd;
70 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readTlabTop;
71 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.registerAsWord;
72 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.useBiasedLocking;
73 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.useTLAB;
74 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.verifyOop;
75 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.writeTlabTop;
76 import static org.graalvm.compiler.hotspot.replacements.HotspotSnippetsOptions.ProfileAllocations;
77 import static org.graalvm.compiler.hotspot.replacements.HotspotSnippetsOptions.ProfileAllocationsContext;
78 import static org.graalvm.compiler.nodes.PiArrayNode.piArrayCastToSnippetReplaceeStamp;
79 import static org.graalvm.compiler.nodes.PiNode.piCastToSnippetReplaceeStamp;
80 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.DEOPT_PROBABILITY;
81 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.FAST_PATH_PROBABILITY;
82 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.FREQUENT_PROBABILITY;
83 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.SLOW_PATH_PROBABILITY;
84 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.VERY_FAST_PATH_PROBABILITY;
85 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.probability;
86 import static org.graalvm.compiler.replacements.ReplacementsUtil.REPLACEMENTS_ASSERTIONS_ENABLED;
87 import static org.graalvm.compiler.replacements.ReplacementsUtil.runtimeAssert;
88 import static org.graalvm.compiler.replacements.ReplacementsUtil.staticAssert;
89 import static org.graalvm.compiler.replacements.SnippetTemplate.DEFAULT_REPLACER;
90 import static org.graalvm.compiler.replacements.nodes.CStringConstant.cstring;
91 import static org.graalvm.compiler.replacements.nodes.ExplodeLoopNode.explodeLoop;
92
93 import org.graalvm.compiler.api.replacements.Fold;
94 import org.graalvm.compiler.api.replacements.Fold.InjectedParameter;
95 import org.graalvm.compiler.api.replacements.Snippet;
96 import org.graalvm.compiler.api.replacements.Snippet.ConstantParameter;
97 import org.graalvm.compiler.api.replacements.Snippet.VarargsParameter;
98 import org.graalvm.compiler.core.common.spi.ForeignCallDescriptor;
99 import org.graalvm.compiler.core.common.type.StampFactory;
100 import org.graalvm.compiler.debug.DebugHandlersFactory;
101 import org.graalvm.compiler.debug.GraalError;
102 import org.graalvm.compiler.graph.Node.ConstantNodeParameter;
103 import org.graalvm.compiler.graph.Node.NodeIntrinsic;
104 import org.graalvm.compiler.hotspot.GraalHotSpotVMConfig;
105 import org.graalvm.compiler.hotspot.meta.HotSpotProviders;
106 import org.graalvm.compiler.hotspot.meta.HotSpotRegistersProvider;
107 import org.graalvm.compiler.hotspot.nodes.DimensionsNode;
108 import org.graalvm.compiler.hotspot.nodes.KlassBeingInitializedCheckNode;
109 import org.graalvm.compiler.hotspot.nodes.aot.LoadConstantIndirectlyFixedNode;
110 import org.graalvm.compiler.hotspot.nodes.aot.LoadConstantIndirectlyNode;
111 import org.graalvm.compiler.hotspot.nodes.type.KlassPointerStamp;
112 import org.graalvm.compiler.hotspot.word.KlassPointer;
113 import org.graalvm.compiler.nodes.ConstantNode;
114 import org.graalvm.compiler.nodes.DeoptimizeNode;
115 import org.graalvm.compiler.nodes.PiNode;
116 import org.graalvm.compiler.nodes.PrefetchAllocateNode;
117 import org.graalvm.compiler.nodes.SnippetAnchorNode;
118 import org.graalvm.compiler.nodes.StructuredGraph;
119 import org.graalvm.compiler.nodes.ValueNode;
120 import org.graalvm.compiler.nodes.debug.DynamicCounterNode;
121 import org.graalvm.compiler.nodes.debug.VerifyHeapNode;
122 import org.graalvm.compiler.nodes.extended.BranchProbabilityNode;
123 import org.graalvm.compiler.nodes.extended.ForeignCallNode;
124 import org.graalvm.compiler.nodes.extended.MembarNode;
125 import org.graalvm.compiler.nodes.java.DynamicNewArrayNode;
126 import org.graalvm.compiler.nodes.java.DynamicNewInstanceNode;
127 import org.graalvm.compiler.nodes.java.NewArrayNode;
128 import org.graalvm.compiler.nodes.java.NewInstanceNode;
129 import org.graalvm.compiler.nodes.java.NewMultiArrayNode;
130 import org.graalvm.compiler.nodes.memory.address.OffsetAddressNode;
131 import org.graalvm.compiler.nodes.spi.LoweringTool;
132 import org.graalvm.compiler.nodes.util.GraphUtil;
133 import org.graalvm.compiler.options.OptionValues;
134 import org.graalvm.compiler.replacements.ReplacementsUtil;
135 import org.graalvm.compiler.replacements.SnippetCounter;
136 import org.graalvm.compiler.replacements.SnippetCounter.Group;
137 import org.graalvm.compiler.replacements.SnippetTemplate;
138 import org.graalvm.compiler.replacements.SnippetTemplate.AbstractTemplates;
139 import org.graalvm.compiler.replacements.SnippetTemplate.Arguments;
140 import org.graalvm.compiler.replacements.SnippetTemplate.SnippetInfo;
141 import org.graalvm.compiler.replacements.Snippets;
142 import org.graalvm.compiler.replacements.nodes.ExplodeLoopNode;
281 Word prototypeMarkWord,
282 @ConstantParameter boolean fillContents,
283 @ConstantParameter boolean emitMemoryBarrier,
284 @ConstantParameter Register threadRegister,
285 @ConstantParameter boolean constantSize,
286 @ConstantParameter String typeContext,
287 @ConstantParameter Counters counters) {
288 // Klass must be initialized by the time the first instance is allocated, therefore we can
289 // just load it from the corresponding cell and avoid the resolution check. We have to use a
290 // fixed load though, to prevent it from floating above the initialization.
291 KlassPointer picHub = LoadConstantIndirectlyFixedNode.loadKlass(hub);
292 return piCastToSnippetReplaceeStamp(allocateInstanceHelper(size, picHub, prototypeMarkWord, fillContents, emitMemoryBarrier, threadRegister, constantSize, typeContext, counters));
293 }
294
295 @Snippet
296 public static Object allocateInstanceDynamic(Class<?> type, Class<?> classClass,
297 @ConstantParameter boolean fillContents,
298 @ConstantParameter boolean emitMemoryBarrier,
299 @ConstantParameter Register threadRegister,
300 @ConstantParameter Counters counters) {
301 if (probability(DEOPT_PROBABILITY, type == null)) {
302 DeoptimizeNode.deopt(None, RuntimeConstraint);
303 }
304 Class<?> nonNullType = PiNode.piCastNonNullClass(type, SnippetAnchorNode.anchor());
305
306 if (probability(DEOPT_PROBABILITY, DynamicNewInstanceNode.throwsInstantiationException(type, classClass))) {
307 DeoptimizeNode.deopt(None, RuntimeConstraint);
308 }
309
310 return PiNode.piCastToSnippetReplaceeStamp(allocateInstanceDynamicHelper(type, fillContents, emitMemoryBarrier, threadRegister, counters, nonNullType));
311 }
312
313 private static Object allocateInstanceDynamicHelper(Class<?> type,
314 boolean fillContents,
315 boolean emitMemoryBarrier,
316 Register threadRegister,
317 Counters counters,
318 Class<?> nonNullType) {
319 KlassPointer hub = ClassGetHubNode.readClass(nonNullType);
320 if (probability(FAST_PATH_PROBABILITY, !hub.isNull())) {
321 KlassPointer nonNullHub = ClassGetHubNode.piCastNonNull(hub, SnippetAnchorNode.anchor());
322
323 if (probability(VERY_FAST_PATH_PROBABILITY, isInstanceKlassFullyInitialized(nonNullHub))) {
324 int layoutHelper = readLayoutHelper(nonNullHub);
325 /*
326 * src/share/vm/oops/klass.hpp: For instances, layout helper is a positive number,
327 * the instance size. This size is already passed through align_object_size and
328 * scaled to bytes. The low order bit is set if instances of this class cannot be
329 * allocated using the fastpath.
330 */
331 if (probability(FAST_PATH_PROBABILITY, (layoutHelper & 1) == 0)) {
332 Word prototypeMarkWord = nonNullHub.readWord(prototypeMarkWordOffset(INJECTED_VMCONFIG), PROTOTYPE_MARK_WORD_LOCATION);
333 /*
334 * FIXME(je,ds): we should actually pass typeContext instead of "" but late
335 * binding of parameters is not yet supported by the GraphBuilderPlugin system.
336 */
337 return allocateInstanceHelper(layoutHelper, nonNullHub, prototypeMarkWord, fillContents, emitMemoryBarrier, threadRegister, false, "", counters);
338 }
339 } else {
340 DeoptimizeNode.deopt(None, RuntimeConstraint);
341 }
342 }
343 return dynamicNewInstanceStub(type);
344 }
345
346 /**
347 * Maximum array length for which fast path allocation is used.
348 */
349 public static final int MAX_ARRAY_FAST_PATH_ALLOCATION_LENGTH = 0x00FFFFFF;
350
351 @Snippet
352 public static Object allocatePrimitiveArrayPIC(KlassPointer hub,
353 int length,
354 Word prototypeMarkWord,
355 @ConstantParameter int headerSize,
356 @ConstantParameter int log2ElementSize,
357 @ConstantParameter boolean fillContents,
358 @ConstantParameter boolean emitMemoryBarrier,
359 @ConstantParameter Register threadRegister,
360 @ConstantParameter boolean maybeUnroll,
361 @ConstantParameter String typeContext,
362 @ConstantParameter int bulkZeroingStride,
363 @ConstantParameter Counters counters) {
364 // Primitive array types are eagerly pre-resolved. We can use a floating load.
365 KlassPointer picHub = LoadConstantIndirectlyNode.loadKlass(hub);
366 return allocateArrayImpl(picHub, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents,
367 emitMemoryBarrier, threadRegister, maybeUnroll, typeContext, bulkZeroingStride, counters);
368 }
369
370 @Snippet
371 public static Object allocateArrayPIC(KlassPointer hub,
372 int length,
373 Word prototypeMarkWord,
374 @ConstantParameter int headerSize,
375 @ConstantParameter int log2ElementSize,
376 @ConstantParameter boolean fillContents,
377 @ConstantParameter boolean emitMemoryBarrier,
378 @ConstantParameter Register threadRegister,
379 @ConstantParameter boolean maybeUnroll,
380 @ConstantParameter String typeContext,
381 @ConstantParameter int bulkZeroingStride,
382 @ConstantParameter Counters counters) {
383 // Array type would be resolved by dominating resolution.
384 KlassPointer picHub = LoadConstantIndirectlyFixedNode.loadKlass(hub);
385 return allocateArrayImpl(picHub, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents,
386 emitMemoryBarrier, threadRegister, maybeUnroll, typeContext, bulkZeroingStride, counters);
387 }
388
389 @Snippet
390 public static Object allocateArray(KlassPointer hub,
391 int length,
392 Word prototypeMarkWord,
393 @ConstantParameter int headerSize,
394 @ConstantParameter int log2ElementSize,
395 @ConstantParameter boolean fillContents,
396 @ConstantParameter boolean emitMemoryBarrier,
397 @ConstantParameter Register threadRegister,
398 @ConstantParameter boolean maybeUnroll,
399 @ConstantParameter String typeContext,
400 @ConstantParameter int bulkZeroingStride,
401 @ConstantParameter Counters counters) {
402 Object result = allocateArrayImpl(hub,
403 length,
404 prototypeMarkWord,
405 headerSize,
406 log2ElementSize,
407 fillContents,
408 emitMemoryBarrier, threadRegister,
409 maybeUnroll,
410 typeContext,
411 bulkZeroingStride,
412 counters);
413 return piArrayCastToSnippetReplaceeStamp(verifyOop(result), length);
414 }
415
416 /**
417 * When allocating on the slow path, determines whether to use a version of the runtime call
418 * that returns {@code null} on a failed allocation instead of raising an OutOfMemoryError.
419 */
420 @Fold
421 static boolean useNullAllocationStubs(@InjectedParameter GraalHotSpotVMConfig config) {
422 return config.areNullAllocationStubsAvailable();
423 }
424
425 private static Object allocateArrayImpl(KlassPointer hub,
426 int length,
427 Word prototypeMarkWord,
428 int headerSize,
429 int log2ElementSize,
430 boolean fillContents,
431 boolean emitMemoryBarrier,
432 Register threadRegister,
433 boolean maybeUnroll,
434 String typeContext,
435 int bulkZeroingStride,
436 Counters counters) {
437 Object result;
438 long allocationSize = arrayAllocationSize(length, headerSize, log2ElementSize);
439 Word thread = registerAsWord(threadRegister);
440 Word top = readTlabTop(thread);
441 Word end = readTlabEnd(thread);
442 Word newTop = top.add(WordFactory.unsigned(allocationSize));
443 if (probability(FREQUENT_PROBABILITY, belowThan(length, MAX_ARRAY_FAST_PATH_ALLOCATION_LENGTH)) && useTLAB(INJECTED_VMCONFIG) &&
444 probability(FAST_PATH_PROBABILITY, newTop.belowOrEqual(end))) {
445 writeTlabTop(thread, newTop);
446 emitPrefetchAllocate(newTop, true);
447 Counters theCounters = counters;
448 if (theCounters != null && theCounters.arrayLoopInit != null) {
449 theCounters.arrayLoopInit.inc();
450 }
451 result = formatArray(hub, allocationSize, length, headerSize, top, prototypeMarkWord, fillContents, emitMemoryBarrier, maybeUnroll, bulkZeroingStride, counters);
452 } else {
453 result = newArrayStub(hub, length);
454 }
455 profileAllocation("array", allocationSize, typeContext);
456 return result;
457 }
458
459 public static Object newArrayStub(KlassPointer hub, int length) {
460 if (useNullAllocationStubs(INJECTED_VMCONFIG)) {
461 return nonNullOrDeopt(newArrayOrNull(NEW_ARRAY_OR_NULL, hub, length));
462 } else {
463 return newArray(NEW_ARRAY, hub, length);
464 }
465 }
466
467 @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = true)
468 private static native Object newArray(@ConstantNodeParameter ForeignCallDescriptor descriptor, KlassPointer hub, int length);
469
470 @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = false)
471 private static native Object newArrayOrNull(@ConstantNodeParameter ForeignCallDescriptor descriptor, KlassPointer hub, int length);
475 */
476 public static final ForeignCallDescriptor DYNAMIC_NEW_INSTANCE = new ForeignCallDescriptor("dynamic_new_instance", Object.class, Class.class);
477
478 /**
479 * New dynamic array stub that returns null on allocation failure.
480 */
481 public static final ForeignCallDescriptor DYNAMIC_NEW_INSTANCE_OR_NULL = new ForeignCallDescriptor("dynamic_new_instance_or_null", Object.class, Class.class);
482
483 public static Object dynamicNewInstanceStub(Class<?> elementType) {
484 if (useNullAllocationStubs(INJECTED_VMCONFIG)) {
485 return nonNullOrDeopt(dynamicNewInstanceOrNull(DYNAMIC_NEW_INSTANCE_OR_NULL, elementType));
486 } else {
487 return dynamicNewInstance(DYNAMIC_NEW_INSTANCE, elementType);
488 }
489 }
490
491 /**
492 * Deoptimizes if {@code obj == null} otherwise returns {@code obj}.
493 */
494 private static Object nonNullOrDeopt(Object obj) {
495 if (BranchProbabilityNode.probability(BranchProbabilityNode.DEOPT_PROBABILITY, obj == null)) {
496 DeoptimizeNode.deopt(None, RuntimeConstraint);
497 }
498 return obj;
499 }
500
501 @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = true)
502 public static native Object dynamicNewInstance(@ConstantNodeParameter ForeignCallDescriptor descriptor, Class<?> elementType);
503
504 @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = false)
505 public static native Object dynamicNewInstanceOrNull(@ConstantNodeParameter ForeignCallDescriptor descriptor, Class<?> elementType);
506
507 @Snippet
508 public static Object allocateArrayDynamic(Class<?> elementType,
509 Class<?> voidClass,
510 int length,
511 @ConstantParameter boolean fillContents,
512 @ConstantParameter boolean emitMemoryBarrier,
513 @ConstantParameter Register threadRegister,
514 @ConstantParameter JavaKind knownElementKind,
515 @ConstantParameter int knownLayoutHelper,
516 @ConstantParameter int bulkZeroingStride,
517 Word prototypeMarkWord,
518 @ConstantParameter Counters counters) {
519 Object result = allocateArrayDynamicImpl(elementType, voidClass, length, fillContents, emitMemoryBarrier, threadRegister, knownElementKind,
520 knownLayoutHelper, bulkZeroingStride, prototypeMarkWord, counters);
521 return result;
522 }
523
524 private static Object allocateArrayDynamicImpl(Class<?> elementType,
525 Class<?> voidClass,
526 int length,
527 boolean fillContents,
528 boolean emitMemoryBarrier,
529 Register threadRegister,
530 JavaKind knownElementKind,
531 int knownLayoutHelper,
532 int bulkZeroingStride,
533 Word prototypeMarkWord,
534 Counters counters) {
535 /*
536 * We only need the dynamic check for void when we have no static information from
537 * knownElementKind.
538 */
539 staticAssert(knownElementKind != JavaKind.Void, "unsupported knownElementKind");
540 if (knownElementKind == JavaKind.Illegal && probability(SLOW_PATH_PROBABILITY, elementType == null || DynamicNewArrayNode.throwsIllegalArgumentException(elementType, voidClass))) {
541 DeoptimizeNode.deopt(None, RuntimeConstraint);
542 }
543
544 KlassPointer klass = loadKlassFromObject(elementType, arrayKlassOffset(INJECTED_VMCONFIG), CLASS_ARRAY_KLASS_LOCATION);
545 if (probability(DEOPT_PROBABILITY, klass.isNull())) {
546 DeoptimizeNode.deopt(None, RuntimeConstraint);
547 }
548 KlassPointer nonNullKlass = ClassGetHubNode.piCastNonNull(klass, SnippetAnchorNode.anchor());
549
550 if (probability(DEOPT_PROBABILITY, length < 0)) {
551 DeoptimizeNode.deopt(None, RuntimeConstraint);
552 }
553 int layoutHelper;
554 if (knownElementKind == JavaKind.Illegal) {
555 layoutHelper = readLayoutHelper(nonNullKlass);
556 } else {
557 runtimeAssert(knownLayoutHelper == readLayoutHelper(nonNullKlass), "layout mismatch");
558 layoutHelper = knownLayoutHelper;
559 }
560 //@formatter:off
561 // from src/share/vm/oops/klass.hpp:
562 //
563 // For arrays, layout helper is a negative number, containing four
564 // distinct bytes, as follows:
565 // MSB:[tag, hsz, ebt, log2(esz)]:LSB
566 // where:
567 // tag is 0x80 if the elements are oops, 0xC0 if non-oops
568 // hsz is array header size in bytes (i.e., offset of first element)
569 // ebt is the BasicType of the elements
570 // esz is the element size in bytes
571 //@formatter:on
572
573 int headerSize = (layoutHelper >> layoutHelperHeaderSizeShift(INJECTED_VMCONFIG)) & layoutHelperHeaderSizeMask(INJECTED_VMCONFIG);
574 int log2ElementSize = (layoutHelper >> layoutHelperLog2ElementSizeShift(INJECTED_VMCONFIG)) & layoutHelperLog2ElementSizeMask(INJECTED_VMCONFIG);
575
576 Object result = allocateArrayImpl(nonNullKlass, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents,
577 emitMemoryBarrier, threadRegister, false, "dynamic type", bulkZeroingStride, counters);
578 return piArrayCastToSnippetReplaceeStamp(verifyOop(result), length);
579 }
580
581 /**
582 * Calls the runtime stub for implementing MULTIANEWARRAY.
583 */
584 @Snippet
585 private static Object newmultiarray(KlassPointer hub, @ConstantParameter int rank, @VarargsParameter int[] dimensions) {
586 Word dims = DimensionsNode.allocaDimsArray(rank);
587 ExplodeLoopNode.explodeLoop();
588 for (int i = 0; i < rank; i++) {
589 dims.writeInt(i * 4, dimensions[i], LocationIdentity.init());
590 }
591 return newMultiArrayStub(hub, rank, dims);
592 }
593
594 private static Object newMultiArrayStub(KlassPointer hub, int rank, Word dims) {
595 if (useNullAllocationStubs(INJECTED_VMCONFIG)) {
596 return nonNullOrDeopt(newMultiArrayOrNull(NEW_MULTI_ARRAY_OR_NULL, hub, rank, dims));
597 } else {
605 KlassPointer picHub = LoadConstantIndirectlyFixedNode.loadKlass(hub);
606 return newmultiarray(picHub, rank, dimensions);
607 }
608
609 @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = true)
610 private static native Object newMultiArray(@ConstantNodeParameter ForeignCallDescriptor descriptor, KlassPointer hub, int rank, Word dims);
611
612 @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = false)
613 private static native Object newMultiArrayOrNull(@ConstantNodeParameter ForeignCallDescriptor descriptor, KlassPointer hub, int rank, Word dims);
614
615 /**
616 * Maximum number of long stores to emit when zeroing an object with a constant size. Larger
617 * objects have their bodies initialized in a loop.
618 */
619 private static final int MAX_UNROLLED_OBJECT_ZEROING_STORES = 8;
620
621 /**
622 * Zero uninitialized memory in a newly allocated object, unrolling as necessary and ensuring
623 * that stores are aligned.
624 *
625 * @param memory beginning of object which is being zeroed
626 * @param startOffset offset to begin zeroing (inclusive). May not be word aligned.
627 * @param endOffset offset to stop zeroing (exclusive). May not be word aligned.
628 * @param isEndOffsetConstant is {@code endOffset} known to be constant in the snippet
629 * @param manualUnroll maximally unroll zeroing
630 * @param bulkZeroingStride stride of bulk zeroing supported by the backend
631 */
632 private static void zeroMemory(Word memory, int startOffset, long endOffset, boolean isEndOffsetConstant, boolean manualUnroll,
633 int bulkZeroingStride, Counters counters) {
634 fillMemory(0, memory, startOffset, endOffset, isEndOffsetConstant, manualUnroll, bulkZeroingStride, counters);
635 }
636
637 private static void fillMemory(long value, Word memory, int startOffset, long offsetLimit, boolean constantOffsetLimit, boolean manualUnroll,
638 int bulkZeroingStride, Counters counters) {
639 ReplacementsUtil.runtimeAssert((offsetLimit & 0x7) == 0, "unaligned object size");
640 int offset = startOffset;
641 if ((offset & 0x7) != 0) {
642 memory.writeInt(offset, (int) value, LocationIdentity.init());
643 offset += 4;
644 }
645 ReplacementsUtil.runtimeAssert((offset & 0x7) == 0, "unaligned offset");
646 Counters theCounters = counters;
647 if (manualUnroll && ((offsetLimit - offset) / 8) <= MAX_UNROLLED_OBJECT_ZEROING_STORES) {
648 ReplacementsUtil.staticAssert(!constantOffsetLimit, "size shouldn't be constant at instantiation time");
649 // This case handles arrays of constant length. Instead of having a snippet variant for
650 // each length, generate a chain of stores of maximum length. Once it's inlined the
651 // break statement will trim excess stores.
652 if (theCounters != null && theCounters.instanceSeqInit != null) {
653 theCounters.instanceSeqInit.inc();
654 }
655
656 explodeLoop();
657 for (int i = 0; i < MAX_UNROLLED_OBJECT_ZEROING_STORES; i++, offset += 8) {
658 if (offset == offsetLimit) {
659 break;
660 }
661 memory.initializeLong(offset, value, LocationIdentity.init());
662 }
663 } else {
664 // Use Word instead of int to avoid extension to long in generated code
665 Word off = WordFactory.signed(offset);
666 if (bulkZeroingStride > 0 && value == 0 && probability(SLOW_PATH_PROBABILITY, (offsetLimit - offset) >= getMinimalBulkZeroingSize(INJECTED_OPTIONVALUES))) {
667 if (theCounters != null && theCounters.instanceBulkInit != null) {
668 theCounters.instanceBulkInit.inc();
669 }
670 ZeroMemoryNode.zero(memory.add(off), offsetLimit - offset, LocationIdentity.init());
671 } else {
672 if (constantOffsetLimit && ((offsetLimit - offset) / 8) <= MAX_UNROLLED_OBJECT_ZEROING_STORES) {
673 if (theCounters != null && theCounters.instanceSeqInit != null) {
674 theCounters.instanceSeqInit.inc();
675 }
676 explodeLoop();
677 } else {
678 if (theCounters != null && theCounters.instanceLoopInit != null) {
679 theCounters.instanceLoopInit.inc();
680 }
681 }
682 for (; off.rawValue() < offsetLimit; off = off.add(8)) {
683 memory.initializeLong(off, value, LocationIdentity.init());
684 }
685 }
686 }
687 }
688
689 @Fold
690 static int getMinimalBulkZeroingSize(@InjectedParameter OptionValues optionValues) {
691 return MinimalBulkZeroingSize.getValue(optionValues);
692 }
693
694 /**
695 * Fill uninitialized memory with garbage value in a newly allocated object, unrolling as
696 * necessary and ensuring that stores are aligned.
697 *
698 * @param memory beginning of object which is being zeroed
699 * @param startOffset offset to begin filling garbage value (inclusive). May not be word
700 * aligned.
701 * @param endOffset offset to stop filling garbage value (exclusive). May not be word aligned.
702 * @param isEndOffsetConstant is {@code endOffset} known to be constant in the snippet
703 * @param manualUnroll maximally unroll zeroing
704 */
705 private static void fillWithGarbage(Word memory, int startOffset, long endOffset, boolean isEndOffsetConstant, boolean manualUnroll, Counters counters) {
706 fillMemory(0xfefefefefefefefeL, memory, startOffset, endOffset, isEndOffsetConstant, manualUnroll, 0, counters);
707 }
708
709 /**
710 * Formats some allocated memory with an object header and zeroes out the rest.
711 */
712 private static Object formatObject(KlassPointer hub,
713 long size,
714 Word memory,
715 Word compileTimePrototypeMarkWord,
716 boolean fillContents,
717 boolean emitMemoryBarrier,
718 boolean constantSize,
719 Counters counters) {
720 Word prototypeMarkWord = useBiasedLocking(INJECTED_VMCONFIG) ? hub.readWord(prototypeMarkWordOffset(INJECTED_VMCONFIG), PROTOTYPE_MARK_WORD_LOCATION) : compileTimePrototypeMarkWord;
721 initializeObjectHeader(memory, prototypeMarkWord, hub);
722 if (fillContents) {
723 zeroMemory(memory, instanceHeaderSize(INJECTED_VMCONFIG), size, constantSize, false, 0, counters);
724 } else if (REPLACEMENTS_ASSERTIONS_ENABLED) {
725 fillWithGarbage(memory, instanceHeaderSize(INJECTED_VMCONFIG), size, constantSize, false, counters);
726 }
727 if (emitMemoryBarrier) {
728 MembarNode.memoryBarrier(MemoryBarriers.STORE_STORE, LocationIdentity.init());
729 }
730 return memory.toObjectNonNull();
731 }
732
733 @Snippet
734 private static void verifyHeap(@ConstantParameter Register threadRegister) {
735 Word thread = registerAsWord(threadRegister);
736 Word topValue = readTlabTop(thread);
737 if (!topValue.equal(WordFactory.zero())) {
738 Word topValueContents = topValue.readWord(0, MARK_WORD_LOCATION);
739 if (topValueContents.equal(WordFactory.zero())) {
740 AssertionSnippets.vmMessageC(AssertionSnippets.ASSERTION_VM_MESSAGE_C, true, cstring("overzeroing of TLAB detected"), 0L, 0L, 0L);
741 }
742 }
743 }
744
745 @Snippet
746 private static void threadBeingInitializedCheck(@ConstantParameter Register threadRegister, KlassPointer klass) {
747 int state = readInstanceKlassInitState(klass);
748 if (state != instanceKlassStateBeingInitialized(INJECTED_VMCONFIG)) {
749 // The klass is no longer being initialized so force recompilation
750 DeoptimizeNode.deopt(InvalidateRecompile, RuntimeConstraint);
751 } else if (registerAsWord(threadRegister) != readInstanceKlassInitThread(klass)) {
752 // The klass is being initialized but this isn't the initializing thread so
753 // so deopt and allow execution to resume in the interpreter where it should block.
754 DeoptimizeNode.deopt(None, RuntimeConstraint);
755 }
756 }
757
758 /**
759 * Formats some allocated memory with an object header and zeroes out the rest.
760 */
761 private static Object formatArray(KlassPointer hub,
762 long allocationSize,
763 int length,
764 int headerSize,
765 Word memory,
766 Word prototypeMarkWord,
767 boolean fillContents,
768 boolean emitMemoryBarrier,
769 boolean maybeUnroll,
770 int bulkZeroingStride,
771 Counters counters) {
772 memory.writeInt(arrayLengthOffset(INJECTED_VMCONFIG), length, LocationIdentity.init());
773 /*
774 * store hub last as the concurrent garbage collectors assume length is valid if hub field
775 * is not null
776 */
777 initializeObjectHeader(memory, prototypeMarkWord, hub);
778 if (fillContents) {
779 zeroMemory(memory, headerSize, allocationSize, false, maybeUnroll, bulkZeroingStride, counters);
780 } else if (REPLACEMENTS_ASSERTIONS_ENABLED) {
781 fillWithGarbage(memory, headerSize, allocationSize, false, maybeUnroll, counters);
782 }
783 if (emitMemoryBarrier) {
784 MembarNode.memoryBarrier(MemoryBarriers.STORE_STORE, LocationIdentity.init());
785 }
786 return memory.toObjectNonNull();
787 }
788
789 static class Counters {
790 Counters(SnippetCounter.Group.Factory factory) {
791 Group newInstance = factory.createSnippetCounterGroup("NewInstance");
792 Group newArray = factory.createSnippetCounterGroup("NewArray");
793 instanceSeqInit = new SnippetCounter(newInstance, "tlabSeqInit", "TLAB alloc with unrolled zeroing");
794 instanceLoopInit = new SnippetCounter(newInstance, "tlabLoopInit", "TLAB alloc with zeroing in a loop");
795 instanceBulkInit = new SnippetCounter(newArray, "tlabBulkInit", "TLAB alloc with bulk zeroing");
796 arrayLoopInit = new SnippetCounter(newArray, "tlabLoopInit", "TLAB alloc with zeroing in a loop");
797 stub = new SnippetCounter(newInstance, "stub", "alloc and zeroing via stub");
798 }
799
800 final SnippetCounter instanceSeqInit;
801 final SnippetCounter instanceLoopInit;
802 final SnippetCounter instanceBulkInit;
803 final SnippetCounter arrayLoopInit;
804 final SnippetCounter stub;
805 }
806
807 public static class Templates extends AbstractTemplates {
808
809 private final SnippetInfo allocateInstance = snippet(NewObjectSnippets.class, "allocateInstance", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION, TLAB_END_LOCATION,
810 PROTOTYPE_MARK_WORD_LOCATION);
811 private final SnippetInfo allocateInstancePIC = snippet(NewObjectSnippets.class, "allocateInstancePIC", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION,
812 TLAB_END_LOCATION, PROTOTYPE_MARK_WORD_LOCATION);
813 private final SnippetInfo allocateArray = snippet(NewObjectSnippets.class, "allocateArray", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION, TLAB_END_LOCATION);
814 private final SnippetInfo allocateArrayPIC = snippet(NewObjectSnippets.class, "allocateArrayPIC", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION, TLAB_END_LOCATION);
815 private final SnippetInfo allocatePrimitiveArrayPIC = snippet(NewObjectSnippets.class, "allocatePrimitiveArrayPIC", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION,
816 TLAB_END_LOCATION);
817 private final SnippetInfo allocateArrayDynamic = snippet(NewObjectSnippets.class, "allocateArrayDynamic", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION,
818 TLAB_END_LOCATION);
819 private final SnippetInfo allocateInstanceDynamic = snippet(NewObjectSnippets.class, "allocateInstanceDynamic", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION,
820 TLAB_END_LOCATION, PROTOTYPE_MARK_WORD_LOCATION, CLASS_INIT_STATE_LOCATION);
821 private final SnippetInfo newmultiarray = snippet(NewObjectSnippets.class, "newmultiarray", TLAB_TOP_LOCATION, TLAB_END_LOCATION);
822 private final SnippetInfo newmultiarrayPIC = snippet(NewObjectSnippets.class, "newmultiarrayPIC", TLAB_TOP_LOCATION, TLAB_END_LOCATION);
823 private final SnippetInfo verifyHeap = snippet(NewObjectSnippets.class, "verifyHeap");
824 private final SnippetInfo threadBeingInitializedCheck = snippet(NewObjectSnippets.class, "threadBeingInitializedCheck");
825 private final GraalHotSpotVMConfig config;
826 private final Counters counters;
827
828 public Templates(OptionValues options, Iterable<DebugHandlersFactory> factories, SnippetCounter.Group.Factory factory, HotSpotProviders providers, TargetDescription target,
829 GraalHotSpotVMConfig config) {
830 super(options, factories, providers, providers.getSnippetReflection(), target);
831 this.config = config;
832 counters = new Counters(factory);
833 }
834
835 /**
836 * Lowers a {@link NewInstanceNode}.
837 */
838 public void lower(NewInstanceNode newInstanceNode, HotSpotRegistersProvider registers, LoweringTool tool) {
839 StructuredGraph graph = newInstanceNode.graph();
840 HotSpotResolvedObjectType type = (HotSpotResolvedObjectType) newInstanceNode.instanceClass();
841 assert !type.isArray();
842 ConstantNode hub = ConstantNode.forConstant(KlassPointerStamp.klassNonNull(), type.klass(), providers.getMetaAccess(), graph);
843 long size = instanceSize(type);
844
880 } else {
881 snippet = allocateArrayPIC;
882 }
883 } else {
884 snippet = allocateArray;
885 }
886
887 Arguments args = new Arguments(snippet, graph.getGuardsStage(), tool.getLoweringStage());
888 args.add("hub", hub);
889 ValueNode length = newArrayNode.length();
890 args.add("length", length.isAlive() ? length : graph.addOrUniqueWithInputs(length));
891 assert arrayType.prototypeMarkWord() == lookupArrayClass(tool, JavaKind.Object).prototypeMarkWord() : "all array types are assumed to have the same prototypeMarkWord";
892 args.add("prototypeMarkWord", arrayType.prototypeMarkWord());
893 args.addConst("headerSize", headerSize);
894 args.addConst("log2ElementSize", log2ElementSize);
895 args.addConst("fillContents", newArrayNode.fillContents());
896 args.addConst("emitMemoryBarrier", newArrayNode.emitMemoryBarrier());
897 args.addConst("threadRegister", registers.getThreadRegister());
898 args.addConst("maybeUnroll", length.isConstant());
899 args.addConst("typeContext", ProfileAllocations.getValue(localOptions) ? arrayType.toJavaName(false) : "");
900 args.addConst("bulkZeroingStride", tool.getLowerer().bulkZeroingStride());
901 args.addConst("counters", counters);
902 SnippetTemplate template = template(newArrayNode, args);
903 graph.getDebug().log("Lowering allocateArray in %s: node=%s, template=%s, arguments=%s", graph, newArrayNode, template, args);
904 template.instantiate(providers.getMetaAccess(), newArrayNode, DEFAULT_REPLACER, args);
905 }
906
907 public void lower(DynamicNewInstanceNode newInstanceNode, HotSpotRegistersProvider registers, LoweringTool tool) {
908 Arguments args = new Arguments(allocateInstanceDynamic, newInstanceNode.graph().getGuardsStage(), tool.getLoweringStage());
909 args.add("type", newInstanceNode.getInstanceType());
910 ValueNode classClass = newInstanceNode.getClassClass();
911 assert classClass != null;
912 args.add("classClass", classClass);
913 args.addConst("fillContents", newInstanceNode.fillContents());
914 args.addConst("emitMemoryBarrier", newInstanceNode.emitMemoryBarrier());
915 args.addConst("threadRegister", registers.getThreadRegister());
916 args.addConst("counters", counters);
917
918 SnippetTemplate template = template(newInstanceNode, args);
919 template.instantiate(providers.getMetaAccess(), newInstanceNode, DEFAULT_REPLACER, args);
920 }
924 Arguments args = new Arguments(allocateArrayDynamic, graph.getGuardsStage(), tool.getLoweringStage());
925 args.add("elementType", newArrayNode.getElementType());
926 ValueNode voidClass = newArrayNode.getVoidClass();
927 assert voidClass != null;
928 args.add("voidClass", voidClass);
929 ValueNode length = newArrayNode.length();
930 args.add("length", length.isAlive() ? length : graph.addOrUniqueWithInputs(length));
931 args.addConst("fillContents", newArrayNode.fillContents());
932 args.addConst("emitMemoryBarrier", newArrayNode.emitMemoryBarrier());
933 args.addConst("threadRegister", registers.getThreadRegister());
934 /*
935 * We use Kind.Illegal as a marker value instead of null because constant snippet
936 * parameters cannot be null.
937 */
938 args.addConst("knownElementKind", newArrayNode.getKnownElementKind() == null ? JavaKind.Illegal : newArrayNode.getKnownElementKind());
939 if (newArrayNode.getKnownElementKind() != null) {
940 args.addConst("knownLayoutHelper", lookupArrayClass(tool, newArrayNode.getKnownElementKind()).layoutHelper());
941 } else {
942 args.addConst("knownLayoutHelper", 0);
943 }
944 args.addConst("bulkZeroingStride", tool.getLowerer().bulkZeroingStride());
945 args.add("prototypeMarkWord", lookupArrayClass(tool, JavaKind.Object).prototypeMarkWord());
946 args.addConst("counters", counters);
947 SnippetTemplate template = template(newArrayNode, args);
948 template.instantiate(providers.getMetaAccess(), newArrayNode, DEFAULT_REPLACER, args);
949 }
950
951 private static HotSpotResolvedObjectType lookupArrayClass(LoweringTool tool, JavaKind kind) {
952 return (HotSpotResolvedObjectType) tool.getMetaAccess().lookupJavaType(kind == JavaKind.Object ? Object.class : kind.toJavaClass()).getArrayClass();
953 }
954
955 public void lower(NewMultiArrayNode newmultiarrayNode, LoweringTool tool) {
956 StructuredGraph graph = newmultiarrayNode.graph();
957 OptionValues localOptions = graph.getOptions();
958 int rank = newmultiarrayNode.dimensionCount();
959 ValueNode[] dims = new ValueNode[rank];
960 for (int i = 0; i < newmultiarrayNode.dimensionCount(); i++) {
961 dims[i] = newmultiarrayNode.dimension(i);
962 }
963 HotSpotResolvedObjectType type = (HotSpotResolvedObjectType) newmultiarrayNode.type();
964 ConstantNode hub = ConstantNode.forConstant(KlassPointerStamp.klassNonNull(), type.klass(), providers.getMetaAccess(), graph);
970 args.addVarargs("dimensions", int.class, StampFactory.forKind(JavaKind.Int), dims);
971 template(newmultiarrayNode, args).instantiate(providers.getMetaAccess(), newmultiarrayNode, DEFAULT_REPLACER, args);
972 }
973
974 private static long instanceSize(HotSpotResolvedObjectType type) {
975 long size = type.instanceSize();
976 assert size >= 0;
977 return size;
978 }
979
980 public void lower(VerifyHeapNode verifyHeapNode, HotSpotRegistersProvider registers, LoweringTool tool) {
981 if (config.cAssertions) {
982 Arguments args = new Arguments(verifyHeap, verifyHeapNode.graph().getGuardsStage(), tool.getLoweringStage());
983 args.addConst("threadRegister", registers.getThreadRegister());
984
985 SnippetTemplate template = template(verifyHeapNode, args);
986 template.instantiate(providers.getMetaAccess(), verifyHeapNode, DEFAULT_REPLACER, args);
987 } else {
988 GraphUtil.removeFixedWithUnusedInputs(verifyHeapNode);
989 }
990 }
991
992 public void lower(KlassBeingInitializedCheckNode verifyHeapNode, HotSpotRegistersProvider registers, LoweringTool tool) {
993 Arguments args = new Arguments(threadBeingInitializedCheck, verifyHeapNode.graph().getGuardsStage(), tool.getLoweringStage());
994 args.addConst("threadRegister", registers.getThreadRegister());
995 args.add("klass", verifyHeapNode.getKlass());
996
997 SnippetTemplate template = template(verifyHeapNode, args);
998 template.instantiate(providers.getMetaAccess(), verifyHeapNode, DEFAULT_REPLACER, args);
999 }
1000 }
1001 }
|