1 /*
2 * Copyright (c) 2012, 2016, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23
24
25 package org.graalvm.compiler.hotspot.replacements;
26
27 import static jdk.vm.ci.meta.DeoptimizationAction.None;
28 import static jdk.vm.ci.meta.DeoptimizationReason.RuntimeConstraint;
29 import static org.graalvm.compiler.core.common.GraalOptions.GeneratePIC;
30 import static org.graalvm.compiler.core.common.calc.UnsignedMath.belowThan;
31 import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfigBase.INJECTED_VMCONFIG;
32 import static org.graalvm.compiler.hotspot.HotSpotBackend.NEW_ARRAY;
33 import static org.graalvm.compiler.hotspot.HotSpotBackend.NEW_ARRAY_OR_NULL;
34 import static org.graalvm.compiler.hotspot.HotSpotBackend.NEW_INSTANCE;
35 import static org.graalvm.compiler.hotspot.HotSpotBackend.NEW_INSTANCE_OR_NULL;
36 import static org.graalvm.compiler.hotspot.HotSpotBackend.NEW_MULTI_ARRAY;
37 import static org.graalvm.compiler.hotspot.HotSpotBackend.NEW_MULTI_ARRAY_OR_NULL;
38 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.CLASS_ARRAY_KLASS_LOCATION;
39 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.HUB_WRITE_LOCATION;
40 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.MARK_WORD_LOCATION;
41 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.PROTOTYPE_MARK_WORD_LOCATION;
42 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_END_LOCATION;
43 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_TOP_LOCATION;
44 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayAllocationSize;
45 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayKlassOffset;
46 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayLengthOffset;
47 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.config;
48 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.initializeObjectHeader;
49 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.instanceHeaderSize;
50 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.isInstanceKlassFullyInitialized;
51 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperHeaderSizeMask;
52 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperHeaderSizeShift;
53 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperLog2ElementSizeMask;
54 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperLog2ElementSizeShift;
55 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.loadKlassFromObject;
56 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.prototypeMarkWordOffset;
57 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readLayoutHelper;
58 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readTlabEnd;
59 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readTlabTop;
60 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.registerAsWord;
61 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.useBiasedLocking;
62 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.useTLAB;
63 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.verifyOop;
64 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.writeTlabTop;
65 import static org.graalvm.compiler.hotspot.replacements.HotspotSnippetsOptions.ProfileAllocations;
66 import static org.graalvm.compiler.hotspot.replacements.HotspotSnippetsOptions.ProfileAllocationsContext;
67 import static org.graalvm.compiler.nodes.PiArrayNode.piArrayCastToSnippetReplaceeStamp;
68 import static org.graalvm.compiler.nodes.PiNode.piCastToSnippetReplaceeStamp;
69 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.FAST_PATH_PROBABILITY;
70 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.FREQUENT_PROBABILITY;
71 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.SLOW_PATH_PROBABILITY;
72 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.probability;
73 import static org.graalvm.compiler.replacements.ReplacementsUtil.REPLACEMENTS_ASSERTIONS_ENABLED;
74 import static org.graalvm.compiler.replacements.ReplacementsUtil.runtimeAssert;
75 import static org.graalvm.compiler.replacements.ReplacementsUtil.staticAssert;
76 import static org.graalvm.compiler.replacements.SnippetTemplate.DEFAULT_REPLACER;
77 import static org.graalvm.compiler.replacements.nodes.CStringConstant.cstring;
78 import static org.graalvm.compiler.replacements.nodes.ExplodeLoopNode.explodeLoop;
79
80 import org.graalvm.compiler.api.replacements.Fold;
81 import org.graalvm.compiler.api.replacements.Fold.InjectedParameter;
82 import org.graalvm.compiler.api.replacements.Snippet;
83 import org.graalvm.compiler.api.replacements.Snippet.ConstantParameter;
84 import org.graalvm.compiler.api.replacements.Snippet.VarargsParameter;
85 import org.graalvm.compiler.core.common.spi.ForeignCallDescriptor;
86 import org.graalvm.compiler.core.common.type.StampFactory;
87 import org.graalvm.compiler.debug.DebugHandlersFactory;
88 import org.graalvm.compiler.debug.GraalError;
89 import org.graalvm.compiler.graph.Node.ConstantNodeParameter;
90 import org.graalvm.compiler.graph.Node.NodeIntrinsic;
91 import org.graalvm.compiler.hotspot.GraalHotSpotVMConfig;
92 import org.graalvm.compiler.hotspot.meta.HotSpotProviders;
93 import org.graalvm.compiler.hotspot.meta.HotSpotRegistersProvider;
94 import org.graalvm.compiler.hotspot.nodes.DimensionsNode;
95 import org.graalvm.compiler.hotspot.nodes.aot.LoadConstantIndirectlyFixedNode;
96 import org.graalvm.compiler.hotspot.nodes.aot.LoadConstantIndirectlyNode;
97 import org.graalvm.compiler.hotspot.nodes.type.KlassPointerStamp;
98 import org.graalvm.compiler.hotspot.word.KlassPointer;
99 import org.graalvm.compiler.nodes.ConstantNode;
100 import org.graalvm.compiler.nodes.DeoptimizeNode;
101 import org.graalvm.compiler.nodes.PiNode;
102 import org.graalvm.compiler.nodes.PrefetchAllocateNode;
103 import org.graalvm.compiler.nodes.SnippetAnchorNode;
104 import org.graalvm.compiler.nodes.StructuredGraph;
105 import org.graalvm.compiler.nodes.ValueNode;
106 import org.graalvm.compiler.nodes.debug.DynamicCounterNode;
107 import org.graalvm.compiler.nodes.debug.VerifyHeapNode;
108 import org.graalvm.compiler.nodes.extended.ForeignCallNode;
109 import org.graalvm.compiler.nodes.extended.MembarNode;
110 import org.graalvm.compiler.nodes.java.DynamicNewArrayNode;
111 import org.graalvm.compiler.nodes.java.DynamicNewInstanceNode;
112 import org.graalvm.compiler.nodes.java.NewArrayNode;
113 import org.graalvm.compiler.nodes.java.NewInstanceNode;
114 import org.graalvm.compiler.nodes.java.NewMultiArrayNode;
115 import org.graalvm.compiler.nodes.memory.address.OffsetAddressNode;
116 import org.graalvm.compiler.nodes.spi.LoweringTool;
117 import org.graalvm.compiler.nodes.util.GraphUtil;
118 import org.graalvm.compiler.options.OptionValues;
119 import org.graalvm.compiler.replacements.ReplacementsUtil;
120 import org.graalvm.compiler.replacements.SnippetCounter;
121 import org.graalvm.compiler.replacements.SnippetCounter.Group;
122 import org.graalvm.compiler.replacements.SnippetTemplate;
123 import org.graalvm.compiler.replacements.SnippetTemplate.AbstractTemplates;
124 import org.graalvm.compiler.replacements.SnippetTemplate.Arguments;
125 import org.graalvm.compiler.replacements.SnippetTemplate.SnippetInfo;
126 import org.graalvm.compiler.replacements.Snippets;
127 import org.graalvm.compiler.replacements.nodes.ExplodeLoopNode;
128 import org.graalvm.compiler.word.Word;
129 import jdk.internal.vm.compiler.word.LocationIdentity;
130 import jdk.internal.vm.compiler.word.WordFactory;
131
132 import jdk.vm.ci.code.CodeUtil;
133 import jdk.vm.ci.code.MemoryBarriers;
134 import jdk.vm.ci.code.Register;
135 import jdk.vm.ci.code.TargetDescription;
136 import jdk.vm.ci.hotspot.HotSpotResolvedObjectType;
137 import jdk.vm.ci.meta.JavaKind;
138 import jdk.vm.ci.meta.ResolvedJavaType;
139
140 /**
141 * Snippets used for implementing NEW, ANEWARRAY and NEWARRAY.
142 */
143 public class NewObjectSnippets implements Snippets {
144
145 enum ProfileContext {
146 AllocatingMethod,
147 InstanceOrArray,
148 AllocatedType,
149 AllocatedTypesInMethod,
150 Total
151 }
152
153 @Fold
154 static String createName(String path, String typeContext, OptionValues options) {
155 switch (ProfileAllocationsContext.getValue(options)) {
156 case AllocatingMethod:
157 return "";
158 case InstanceOrArray:
159 return path;
160 case AllocatedType:
161 case AllocatedTypesInMethod:
162 return typeContext;
163 case Total:
164 return "bytes";
165 default:
166 throw GraalError.shouldNotReachHere();
167 }
168 }
169
170 @Fold
171 static boolean doProfile(OptionValues options) {
172 return ProfileAllocations.getValue(options);
173 }
174
175 @Fold
176 static boolean withContext(OptionValues options) {
177 ProfileContext context = ProfileAllocationsContext.getValue(options);
178 return context == ProfileContext.AllocatingMethod || context == ProfileContext.AllocatedTypesInMethod;
179 }
180
181 protected static void profileAllocation(String path, long size, String typeContext, OptionValues options) {
182 if (doProfile(options)) {
183 String name = createName(path, typeContext, options);
184
185 boolean context = withContext(options);
186 DynamicCounterNode.counter("number of bytes allocated", name, size, context);
187 DynamicCounterNode.counter("number of allocations", name, 1, context);
188 }
189 }
190
191 public static void emitPrefetchAllocate(Word address, boolean isArray) {
192 GraalHotSpotVMConfig config = config(INJECTED_VMCONFIG);
193 if (config.allocatePrefetchStyle > 0) {
194 // Insert a prefetch for each allocation only on the fast-path
195 // Generate several prefetch instructions.
196 int lines = isArray ? config.allocatePrefetchLines : config.allocateInstancePrefetchLines;
197 int stepSize = config.allocatePrefetchStepSize;
198 int distance = config.allocatePrefetchDistance;
199 ExplodeLoopNode.explodeLoop();
200 for (int i = 0; i < lines; i++) {
201 PrefetchAllocateNode.prefetch(OffsetAddressNode.address(address, distance));
202 distance += stepSize;
203 }
204 }
205 }
206
207 @Snippet
208 public static Object allocateInstance(@ConstantParameter int size, KlassPointer hub, Word prototypeMarkWord, @ConstantParameter boolean fillContents,
209 @ConstantParameter Register threadRegister, @ConstantParameter boolean constantSize, @ConstantParameter String typeContext, @ConstantParameter OptionValues options,
210 @ConstantParameter Counters counters) {
211 return piCastToSnippetReplaceeStamp(allocateInstanceHelper(size, hub, prototypeMarkWord, fillContents, threadRegister, constantSize, typeContext, options, counters));
212 }
213
214 public static Object allocateInstanceHelper(int size, KlassPointer hub, Word prototypeMarkWord, boolean fillContents,
215 Register threadRegister, boolean constantSize, String typeContext, OptionValues options, Counters counters) {
216 Object result;
217 Word thread = registerAsWord(threadRegister);
218 Word top = readTlabTop(thread);
219 Word end = readTlabEnd(thread);
220 Word newTop = top.add(size);
221 if (useTLAB(INJECTED_VMCONFIG) && probability(FAST_PATH_PROBABILITY, newTop.belowOrEqual(end))) {
222 writeTlabTop(thread, newTop);
223 emitPrefetchAllocate(newTop, false);
224 result = formatObject(hub, size, top, prototypeMarkWord, fillContents, constantSize, counters);
225 } else {
226 if (counters != null && counters.stub != null) {
227 counters.stub.inc();
228 }
229 result = newInstanceStub(hub);
230 }
231 profileAllocation("instance", size, typeContext, options);
232 return verifyOop(result);
233 }
234
235 public static Object newInstanceStub(KlassPointer hub) {
236 if (useNullAllocationStubs(INJECTED_VMCONFIG)) {
237 return nonNullOrDeopt(newInstanceOrNull(NEW_INSTANCE_OR_NULL, hub));
238 } else {
239 return newInstance(NEW_INSTANCE, hub);
240 }
241 }
242
243 @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = true)
244 private static native Object newInstance(@ConstantNodeParameter ForeignCallDescriptor descriptor, KlassPointer hub);
245
246 @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = false)
247 private static native Object newInstanceOrNull(@ConstantNodeParameter ForeignCallDescriptor descriptor, KlassPointer hub);
248
249 @Snippet
250 public static Object allocateInstancePIC(@ConstantParameter int size, KlassPointer hub, Word prototypeMarkWord, @ConstantParameter boolean fillContents,
251 @ConstantParameter Register threadRegister, @ConstantParameter boolean constantSize, @ConstantParameter String typeContext, @ConstantParameter OptionValues options,
252 @ConstantParameter Counters counters) {
253 // Klass must be initialized by the time the first instance is allocated, therefore we can
254 // just load it from the corresponding cell and avoid the resolution check. We have to use a
255 // fixed load though, to prevent it from floating above the initialization.
256 KlassPointer picHub = LoadConstantIndirectlyFixedNode.loadKlass(hub);
257 return piCastToSnippetReplaceeStamp(allocateInstanceHelper(size, picHub, prototypeMarkWord, fillContents, threadRegister, constantSize, typeContext, options, counters));
258 }
259
260 @Snippet
261 public static Object allocateInstanceDynamic(Class<?> type, Class<?> classClass, @ConstantParameter boolean fillContents, @ConstantParameter Register threadRegister,
262 @ConstantParameter OptionValues options, @ConstantParameter Counters counters) {
263 if (probability(SLOW_PATH_PROBABILITY, type == null)) {
264 DeoptimizeNode.deopt(None, RuntimeConstraint);
265 }
266 Class<?> nonNullType = PiNode.piCastNonNullClass(type, SnippetAnchorNode.anchor());
267
268 if (probability(SLOW_PATH_PROBABILITY, DynamicNewInstanceNode.throwsInstantiationException(type, classClass))) {
269 DeoptimizeNode.deopt(None, RuntimeConstraint);
270 }
271
272 return PiNode.piCastToSnippetReplaceeStamp(allocateInstanceDynamicHelper(fillContents, threadRegister, options, counters, nonNullType));
273 }
274
275 private static Object allocateInstanceDynamicHelper(boolean fillContents, Register threadRegister, OptionValues options, Counters counters, Class<?> nonNullType) {
276 KlassPointer hub = ClassGetHubNode.readClass(nonNullType);
277 if (probability(FAST_PATH_PROBABILITY, !hub.isNull())) {
278 KlassPointer nonNullHub = ClassGetHubNode.piCastNonNull(hub, SnippetAnchorNode.anchor());
279
280 if (probability(FAST_PATH_PROBABILITY, isInstanceKlassFullyInitialized(nonNullHub))) {
281 int layoutHelper = readLayoutHelper(nonNullHub);
282 /*
283 * src/share/vm/oops/klass.hpp: For instances, layout helper is a positive number,
284 * the instance size. This size is already passed through align_object_size and
285 * scaled to bytes. The low order bit is set if instances of this class cannot be
286 * allocated using the fastpath.
287 */
288 if (probability(FAST_PATH_PROBABILITY, (layoutHelper & 1) == 0)) {
289 Word prototypeMarkWord = nonNullHub.readWord(prototypeMarkWordOffset(INJECTED_VMCONFIG), PROTOTYPE_MARK_WORD_LOCATION);
290 /*
291 * FIXME(je,ds): we should actually pass typeContext instead of "" but late
292 * binding of parameters is not yet supported by the GraphBuilderPlugin system.
293 */
294 return allocateInstanceHelper(layoutHelper, nonNullHub, prototypeMarkWord, fillContents, threadRegister, false, "", options, counters);
295 }
296 } else {
297 DeoptimizeNode.deopt(None, RuntimeConstraint);
298 }
299 }
300 DeoptimizeNode.deopt(None, RuntimeConstraint);
301 return null;
302 }
303
304 /**
305 * Maximum array length for which fast path allocation is used.
306 */
307 public static final int MAX_ARRAY_FAST_PATH_ALLOCATION_LENGTH = 0x00FFFFFF;
308
309 @Snippet
310 public static Object allocatePrimitiveArrayPIC(KlassPointer hub, int length, Word prototypeMarkWord, @ConstantParameter int headerSize, @ConstantParameter int log2ElementSize,
311 @ConstantParameter boolean fillContents, @ConstantParameter Register threadRegister, @ConstantParameter boolean maybeUnroll, @ConstantParameter String typeContext,
312 @ConstantParameter OptionValues options, @ConstantParameter Counters counters) {
313 // Primitive array types are eagerly pre-resolved. We can use a floating load.
314 KlassPointer picHub = LoadConstantIndirectlyNode.loadKlass(hub);
315 return allocateArrayImpl(picHub, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents, threadRegister, maybeUnroll, typeContext, false, options, counters);
316 }
317
318 @Snippet
319 public static Object allocateArrayPIC(KlassPointer hub, int length, Word prototypeMarkWord, @ConstantParameter int headerSize, @ConstantParameter int log2ElementSize,
320 @ConstantParameter boolean fillContents, @ConstantParameter Register threadRegister, @ConstantParameter boolean maybeUnroll, @ConstantParameter String typeContext,
321 @ConstantParameter OptionValues options, @ConstantParameter Counters counters) {
322 // Array type would be resolved by dominating resolution.
323 KlassPointer picHub = LoadConstantIndirectlyFixedNode.loadKlass(hub);
324 return allocateArrayImpl(picHub, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents, threadRegister, maybeUnroll, typeContext, false, options, counters);
325 }
326
327 @Snippet
328 public static Object allocateArray(KlassPointer hub,
329 int length,
330 Word prototypeMarkWord,
331 @ConstantParameter int headerSize,
332 @ConstantParameter int log2ElementSize,
333 @ConstantParameter boolean fillContents,
334 @ConstantParameter Register threadRegister,
335 @ConstantParameter boolean maybeUnroll,
336 @ConstantParameter String typeContext,
337 @ConstantParameter OptionValues options,
338 @ConstantParameter Counters counters) {
339 Object result = allocateArrayImpl(hub, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents, threadRegister, maybeUnroll, typeContext, false, options, counters);
340 return piArrayCastToSnippetReplaceeStamp(verifyOop(result), length);
341 }
342
343 /**
344 * When allocating on the slow path, determines whether to use a version of the runtime call
345 * that returns {@code null} on a failed allocation instead of raising an OutOfMemoryError.
346 */
347 @Fold
348 static boolean useNullAllocationStubs(@InjectedParameter GraalHotSpotVMConfig config) {
349 return config.areNullAllocationStubsAvailable();
350 }
351
352 private static Object allocateArrayImpl(KlassPointer hub, int length, Word prototypeMarkWord, int headerSize, int log2ElementSize, boolean fillContents, Register threadRegister,
353 boolean maybeUnroll, String typeContext, boolean skipNegativeCheck, OptionValues options, Counters counters) {
354 Object result;
355 int allocationSize = arrayAllocationSize(length, headerSize, log2ElementSize);
356 Word thread = registerAsWord(threadRegister);
357 Word top = readTlabTop(thread);
358 Word end = readTlabEnd(thread);
359 Word newTop = top.add(allocationSize);
360 if (probability(FREQUENT_PROBABILITY, skipNegativeCheck || belowThan(length, MAX_ARRAY_FAST_PATH_ALLOCATION_LENGTH)) && useTLAB(INJECTED_VMCONFIG) &&
361 probability(FAST_PATH_PROBABILITY, newTop.belowOrEqual(end))) {
362 writeTlabTop(thread, newTop);
363 emitPrefetchAllocate(newTop, true);
364 if (counters != null && counters.arrayLoopInit != null) {
365 counters.arrayLoopInit.inc();
366 }
367 result = formatArray(hub, allocationSize, length, headerSize, top, prototypeMarkWord, fillContents, maybeUnroll, counters);
368 } else {
369 result = newArrayStub(hub, length);
370 }
371 profileAllocation("array", allocationSize, typeContext, options);
372 return result;
373 }
374
375 public static Object newArrayStub(KlassPointer hub, int length) {
376 if (useNullAllocationStubs(INJECTED_VMCONFIG)) {
377 return nonNullOrDeopt(newArrayOrNull(NEW_ARRAY_OR_NULL, hub, length));
378 } else {
379 return newArray(NEW_ARRAY, hub, length);
380 }
381 }
382
383 @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = true)
384 private static native Object newArray(@ConstantNodeParameter ForeignCallDescriptor descriptor, KlassPointer hub, int length);
385
386 @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = false)
387 private static native Object newArrayOrNull(@ConstantNodeParameter ForeignCallDescriptor descriptor, KlassPointer hub, int length);
388
389 /**
390 * Deoptimizes if {@code obj == null} otherwise returns {@code obj}.
391 */
392 private static Object nonNullOrDeopt(Object obj) {
393 if (obj == null) {
394 DeoptimizeNode.deopt(None, RuntimeConstraint);
395 }
396 return obj;
397 }
398
399 @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = true)
400 public static native Object dynamicNewInstance(@ConstantNodeParameter ForeignCallDescriptor descriptor, Class<?> elementType);
401
402 @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = false)
403 public static native Object dynamicNewInstanceOrNull(@ConstantNodeParameter ForeignCallDescriptor descriptor, Class<?> elementType);
404
405 @Snippet
406 public static Object allocateArrayDynamic(Class<?> elementType, Class<?> voidClass, int length, @ConstantParameter boolean fillContents, @ConstantParameter Register threadRegister,
407 @ConstantParameter JavaKind knownElementKind, @ConstantParameter int knownLayoutHelper, Word prototypeMarkWord, @ConstantParameter OptionValues options,
408 @ConstantParameter Counters counters) {
409 Object result = allocateArrayDynamicImpl(elementType, voidClass, length, fillContents, threadRegister, knownElementKind, knownLayoutHelper, prototypeMarkWord, options, counters);
410 return result;
411 }
412
413 private static Object allocateArrayDynamicImpl(Class<?> elementType, Class<?> voidClass, int length, boolean fillContents, Register threadRegister, JavaKind knownElementKind,
414 int knownLayoutHelper, Word prototypeMarkWord, OptionValues options, Counters counters) {
415 /*
416 * We only need the dynamic check for void when we have no static information from
417 * knownElementKind.
418 */
419 staticAssert(knownElementKind != JavaKind.Void, "unsupported knownElementKind");
420 if (knownElementKind == JavaKind.Illegal && probability(SLOW_PATH_PROBABILITY, elementType == null || DynamicNewArrayNode.throwsIllegalArgumentException(elementType, voidClass))) {
421 DeoptimizeNode.deopt(None, RuntimeConstraint);
422 }
423
424 KlassPointer klass = loadKlassFromObject(elementType, arrayKlassOffset(INJECTED_VMCONFIG), CLASS_ARRAY_KLASS_LOCATION);
425 if (klass.isNull()) {
426 DeoptimizeNode.deopt(None, RuntimeConstraint);
427 }
428 KlassPointer nonNullKlass = ClassGetHubNode.piCastNonNull(klass, SnippetAnchorNode.anchor());
429
430 if (length < 0) {
431 DeoptimizeNode.deopt(None, RuntimeConstraint);
432 }
433 int layoutHelper;
434 if (knownElementKind == JavaKind.Illegal) {
435 layoutHelper = readLayoutHelper(nonNullKlass);
436 } else {
437 runtimeAssert(knownLayoutHelper == readLayoutHelper(nonNullKlass), "layout mismatch");
438 layoutHelper = knownLayoutHelper;
439 }
440 //@formatter:off
441 // from src/share/vm/oops/klass.hpp:
442 //
443 // For arrays, layout helper is a negative number, containing four
444 // distinct bytes, as follows:
445 // MSB:[tag, hsz, ebt, log2(esz)]:LSB
446 // where:
447 // tag is 0x80 if the elements are oops, 0xC0 if non-oops
448 // hsz is array header size in bytes (i.e., offset of first element)
449 // ebt is the BasicType of the elements
450 // esz is the element size in bytes
451 //@formatter:on
452
453 int headerSize = (layoutHelper >> layoutHelperHeaderSizeShift(INJECTED_VMCONFIG)) & layoutHelperHeaderSizeMask(INJECTED_VMCONFIG);
454 int log2ElementSize = (layoutHelper >> layoutHelperLog2ElementSizeShift(INJECTED_VMCONFIG)) & layoutHelperLog2ElementSizeMask(INJECTED_VMCONFIG);
455
456 Object result = allocateArrayImpl(nonNullKlass, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents, threadRegister, false, "dynamic type", true, options, counters);
457 return piArrayCastToSnippetReplaceeStamp(verifyOop(result), length);
458 }
459
460 /**
461 * Calls the runtime stub for implementing MULTIANEWARRAY.
462 */
463 @Snippet
464 private static Object newmultiarray(KlassPointer hub, @ConstantParameter int rank, @VarargsParameter int[] dimensions) {
465 Word dims = DimensionsNode.allocaDimsArray(rank);
466 ExplodeLoopNode.explodeLoop();
467 for (int i = 0; i < rank; i++) {
468 dims.writeInt(i * 4, dimensions[i], LocationIdentity.init());
469 }
470 return newMultiArrayStub(hub, rank, dims);
471 }
472
473 private static Object newMultiArrayStub(KlassPointer hub, int rank, Word dims) {
474 if (useNullAllocationStubs(INJECTED_VMCONFIG)) {
475 return nonNullOrDeopt(newMultiArrayOrNull(NEW_MULTI_ARRAY_OR_NULL, hub, rank, dims));
476 } else {
477 return newMultiArray(NEW_MULTI_ARRAY, hub, rank, dims);
478 }
479 }
480
481 @Snippet
482 private static Object newmultiarrayPIC(KlassPointer hub, @ConstantParameter int rank, @VarargsParameter int[] dimensions) {
483 // Array type would be resolved by dominating resolution.
484 KlassPointer picHub = LoadConstantIndirectlyFixedNode.loadKlass(hub);
485 return newmultiarray(picHub, rank, dimensions);
486 }
487
488 @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = true)
489 private static native Object newMultiArray(@ConstantNodeParameter ForeignCallDescriptor descriptor, KlassPointer hub, int rank, Word dims);
490
491 @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = false)
492 private static native Object newMultiArrayOrNull(@ConstantNodeParameter ForeignCallDescriptor descriptor, KlassPointer hub, int rank, Word dims);
493
494 /**
495 * Maximum number of long stores to emit when zeroing an object with a constant size. Larger
496 * objects have their bodies initialized in a loop.
497 */
498 private static final int MAX_UNROLLED_OBJECT_ZEROING_STORES = 8;
499
500 /**
501 * Zero uninitialized memory in a newly allocated object, unrolling as necessary and ensuring
502 * that stores are aligned.
503 *
504 * @param size number of bytes to zero
505 * @param memory beginning of object which is being zeroed
506 * @param constantSize is {@code size} known to be constant in the snippet
507 * @param startOffset offset to begin zeroing. May not be word aligned.
508 * @param manualUnroll maximally unroll zeroing
509 */
510 private static void zeroMemory(int size, Word memory, boolean constantSize, int startOffset, boolean manualUnroll, Counters counters) {
511 fillMemory(0, size, memory, constantSize, startOffset, manualUnroll, counters);
512 }
513
514 private static void fillMemory(long value, int size, Word memory, boolean constantSize, int startOffset, boolean manualUnroll, Counters counters) {
515 ReplacementsUtil.runtimeAssert((size & 0x7) == 0, "unaligned object size");
516 int offset = startOffset;
517 if ((offset & 0x7) != 0) {
518 memory.writeInt(offset, (int) value, LocationIdentity.init());
519 offset += 4;
520 }
521 ReplacementsUtil.runtimeAssert((offset & 0x7) == 0, "unaligned offset");
522 if (manualUnroll && ((size - offset) / 8) <= MAX_UNROLLED_OBJECT_ZEROING_STORES) {
523 ReplacementsUtil.staticAssert(!constantSize, "size shouldn't be constant at instantiation time");
524 // This case handles arrays of constant length. Instead of having a snippet variant for
525 // each length, generate a chain of stores of maximum length. Once it's inlined the
526 // break statement will trim excess stores.
527 if (counters != null && counters.instanceSeqInit != null) {
528 counters.instanceSeqInit.inc();
529 }
530
531 explodeLoop();
532 for (int i = 0; i < MAX_UNROLLED_OBJECT_ZEROING_STORES; i++, offset += 8) {
533 if (offset == size) {
534 break;
535 }
536 memory.initializeLong(offset, value, LocationIdentity.init());
537 }
538 } else {
539 // Use Word instead of int to avoid extension to long in generated code
540 Word off = WordFactory.signed(offset);
541 if (constantSize && ((size - offset) / 8) <= MAX_UNROLLED_OBJECT_ZEROING_STORES) {
542 if (counters != null && counters.instanceSeqInit != null) {
543 counters.instanceSeqInit.inc();
544 }
545 explodeLoop();
546 } else {
547 if (counters != null && counters.instanceLoopInit != null) {
548 counters.instanceLoopInit.inc();
549 }
550 }
551 for (; off.rawValue() < size; off = off.add(8)) {
552 memory.initializeLong(off, value, LocationIdentity.init());
553 }
554 }
555 }
556
557 /**
558 * Fill uninitialized memory with garbage value in a newly allocated object, unrolling as
559 * necessary and ensuring that stores are aligned.
560 *
561 * @param size number of bytes to zero
562 * @param memory beginning of object which is being zeroed
563 * @param constantSize is {@code size} known to be constant in the snippet
564 * @param startOffset offset to begin zeroing. May not be word aligned.
565 * @param manualUnroll maximally unroll zeroing
566 */
567 private static void fillWithGarbage(int size, Word memory, boolean constantSize, int startOffset, boolean manualUnroll, Counters counters) {
568 fillMemory(0xfefefefefefefefeL, size, memory, constantSize, startOffset, manualUnroll, counters);
569 }
570
571 /**
572 * Formats some allocated memory with an object header and zeroes out the rest.
573 */
574 private static Object formatObject(KlassPointer hub, int size, Word memory, Word compileTimePrototypeMarkWord, boolean fillContents, boolean constantSize, Counters counters) {
575 Word prototypeMarkWord = useBiasedLocking(INJECTED_VMCONFIG) ? hub.readWord(prototypeMarkWordOffset(INJECTED_VMCONFIG), PROTOTYPE_MARK_WORD_LOCATION) : compileTimePrototypeMarkWord;
576 initializeObjectHeader(memory, prototypeMarkWord, hub);
577 if (fillContents) {
578 zeroMemory(size, memory, constantSize, instanceHeaderSize(INJECTED_VMCONFIG), false, counters);
579 } else if (REPLACEMENTS_ASSERTIONS_ENABLED) {
580 fillWithGarbage(size, memory, constantSize, instanceHeaderSize(INJECTED_VMCONFIG), false, counters);
581 }
582 MembarNode.memoryBarrier(MemoryBarriers.STORE_STORE, LocationIdentity.init());
583 return memory.toObjectNonNull();
584 }
585
586 @Snippet
587 private static void verifyHeap(@ConstantParameter Register threadRegister) {
588 Word thread = registerAsWord(threadRegister);
589 Word topValue = readTlabTop(thread);
590 if (!topValue.equal(WordFactory.zero())) {
591 Word topValueContents = topValue.readWord(0, MARK_WORD_LOCATION);
592 if (topValueContents.equal(WordFactory.zero())) {
593 AssertionSnippets.vmMessageC(AssertionSnippets.ASSERTION_VM_MESSAGE_C, true, cstring("overzeroing of TLAB detected"), 0L, 0L, 0L);
594 }
595 }
596 }
597
598 /**
599 * Formats some allocated memory with an object header and zeroes out the rest.
600 */
601 private static Object formatArray(KlassPointer hub, int allocationSize, int length, int headerSize, Word memory, Word prototypeMarkWord, boolean fillContents, boolean maybeUnroll,
602 Counters counters) {
603 memory.writeInt(arrayLengthOffset(INJECTED_VMCONFIG), length, LocationIdentity.init());
604 /*
605 * store hub last as the concurrent garbage collectors assume length is valid if hub field
606 * is not null
607 */
608 initializeObjectHeader(memory, prototypeMarkWord, hub);
609 if (fillContents) {
610 zeroMemory(allocationSize, memory, false, headerSize, maybeUnroll, counters);
611 } else if (REPLACEMENTS_ASSERTIONS_ENABLED) {
612 fillWithGarbage(allocationSize, memory, false, headerSize, maybeUnroll, counters);
613 }
614 MembarNode.memoryBarrier(MemoryBarriers.STORE_STORE, LocationIdentity.init());
615 return memory.toObjectNonNull();
616 }
617
618 static class Counters {
619 Counters(SnippetCounter.Group.Factory factory) {
620 Group newInstance = factory.createSnippetCounterGroup("NewInstance");
621 Group newArray = factory.createSnippetCounterGroup("NewArray");
622 instanceSeqInit = new SnippetCounter(newInstance, "tlabSeqInit", "TLAB alloc with unrolled zeroing");
623 instanceLoopInit = new SnippetCounter(newInstance, "tlabLoopInit", "TLAB alloc with zeroing in a loop");
624 arrayLoopInit = new SnippetCounter(newArray, "tlabLoopInit", "TLAB alloc with zeroing in a loop");
625 stub = new SnippetCounter(newInstance, "stub", "alloc and zeroing via stub");
626 }
627
628 final SnippetCounter instanceSeqInit;
629 final SnippetCounter instanceLoopInit;
630 final SnippetCounter arrayLoopInit;
631 final SnippetCounter stub;
632 }
633
634 public static class Templates extends AbstractTemplates {
635
636 private final SnippetInfo allocateInstance = snippet(NewObjectSnippets.class, "allocateInstance", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION, TLAB_END_LOCATION);
637 private final SnippetInfo allocateInstancePIC = snippet(NewObjectSnippets.class, "allocateInstancePIC", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION,
638 TLAB_END_LOCATION);
639 private final SnippetInfo allocateArray = snippet(NewObjectSnippets.class, "allocateArray", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION, TLAB_END_LOCATION);
640 private final SnippetInfo allocateArrayPIC = snippet(NewObjectSnippets.class, "allocateArrayPIC", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION, TLAB_END_LOCATION);
641 private final SnippetInfo allocatePrimitiveArrayPIC = snippet(NewObjectSnippets.class, "allocatePrimitiveArrayPIC", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION,
642 TLAB_END_LOCATION);
643 private final SnippetInfo allocateArrayDynamic = snippet(NewObjectSnippets.class, "allocateArrayDynamic", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION,
644 TLAB_END_LOCATION);
645 private final SnippetInfo allocateInstanceDynamic = snippet(NewObjectSnippets.class, "allocateInstanceDynamic", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION,
646 TLAB_END_LOCATION);
647 private final SnippetInfo newmultiarray = snippet(NewObjectSnippets.class, "newmultiarray", TLAB_TOP_LOCATION, TLAB_END_LOCATION);
648 private final SnippetInfo newmultiarrayPIC = snippet(NewObjectSnippets.class, "newmultiarrayPIC", TLAB_TOP_LOCATION, TLAB_END_LOCATION);
649 private final SnippetInfo verifyHeap = snippet(NewObjectSnippets.class, "verifyHeap");
650 private final GraalHotSpotVMConfig config;
651 private final Counters counters;
652
653 public Templates(OptionValues options, Iterable<DebugHandlersFactory> factories, SnippetCounter.Group.Factory factory, HotSpotProviders providers, TargetDescription target,
654 GraalHotSpotVMConfig config) {
655 super(options, factories, providers, providers.getSnippetReflection(), target);
656 this.config = config;
657 counters = new Counters(factory);
658 }
659
660 /**
661 * Lowers a {@link NewInstanceNode}.
662 */
663 public void lower(NewInstanceNode newInstanceNode, HotSpotRegistersProvider registers, LoweringTool tool) {
664 StructuredGraph graph = newInstanceNode.graph();
665 HotSpotResolvedObjectType type = (HotSpotResolvedObjectType) newInstanceNode.instanceClass();
666 assert !type.isArray();
667 ConstantNode hub = ConstantNode.forConstant(KlassPointerStamp.klassNonNull(), type.klass(), providers.getMetaAccess(), graph);
668 int size = instanceSize(type);
669
670 OptionValues localOptions = graph.getOptions();
671 SnippetInfo snippet = GeneratePIC.getValue(localOptions) ? allocateInstancePIC : allocateInstance;
672 Arguments args = new Arguments(snippet, graph.getGuardsStage(), tool.getLoweringStage());
673 args.addConst("size", size);
674 args.add("hub", hub);
675 args.add("prototypeMarkWord", type.prototypeMarkWord());
676 args.addConst("fillContents", newInstanceNode.fillContents());
677 args.addConst("threadRegister", registers.getThreadRegister());
678 args.addConst("constantSize", true);
679 args.addConst("typeContext", ProfileAllocations.getValue(localOptions) ? type.toJavaName(false) : "");
680 args.addConst("options", localOptions);
681 args.addConst("counters", counters);
682
683 SnippetTemplate template = template(newInstanceNode, args);
684 graph.getDebug().log("Lowering allocateInstance in %s: node=%s, template=%s, arguments=%s", graph, newInstanceNode, template, args);
685 template.instantiate(providers.getMetaAccess(), newInstanceNode, DEFAULT_REPLACER, args);
686 }
687
688 /**
689 * Lowers a {@link NewArrayNode}.
690 */
691 public void lower(NewArrayNode newArrayNode, HotSpotRegistersProvider registers, LoweringTool tool) {
692 StructuredGraph graph = newArrayNode.graph();
693 ResolvedJavaType elementType = newArrayNode.elementType();
694 HotSpotResolvedObjectType arrayType = (HotSpotResolvedObjectType) elementType.getArrayClass();
695 JavaKind elementKind = elementType.getJavaKind();
696 ConstantNode hub = ConstantNode.forConstant(KlassPointerStamp.klassNonNull(), arrayType.klass(), providers.getMetaAccess(), graph);
697 final int headerSize = tool.getMetaAccess().getArrayBaseOffset(elementKind);
698 int log2ElementSize = CodeUtil.log2(tool.getMetaAccess().getArrayIndexScale(elementKind));
699
700 OptionValues localOptions = graph.getOptions();
701 SnippetInfo snippet;
702 if (GeneratePIC.getValue(localOptions)) {
703 if (elementType.isPrimitive()) {
704 snippet = allocatePrimitiveArrayPIC;
705 } else {
706 snippet = allocateArrayPIC;
707 }
708 } else {
709 snippet = allocateArray;
710 }
711
712 Arguments args = new Arguments(snippet, graph.getGuardsStage(), tool.getLoweringStage());
713 args.add("hub", hub);
714 ValueNode length = newArrayNode.length();
715 args.add("length", length.isAlive() ? length : graph.addOrUniqueWithInputs(length));
716 assert arrayType.prototypeMarkWord() == lookupArrayClass(tool, JavaKind.Object).prototypeMarkWord() : "all array types are assumed to have the same prototypeMarkWord";
717 args.add("prototypeMarkWord", arrayType.prototypeMarkWord());
718 args.addConst("headerSize", headerSize);
719 args.addConst("log2ElementSize", log2ElementSize);
720 args.addConst("fillContents", newArrayNode.fillContents());
721 args.addConst("threadRegister", registers.getThreadRegister());
722 args.addConst("maybeUnroll", length.isConstant());
723 args.addConst("typeContext", ProfileAllocations.getValue(localOptions) ? arrayType.toJavaName(false) : "");
724 args.addConst("options", localOptions);
725 args.addConst("counters", counters);
726 SnippetTemplate template = template(newArrayNode, args);
727 graph.getDebug().log("Lowering allocateArray in %s: node=%s, template=%s, arguments=%s", graph, newArrayNode, template, args);
728 template.instantiate(providers.getMetaAccess(), newArrayNode, DEFAULT_REPLACER, args);
729 }
730
731 public void lower(DynamicNewInstanceNode newInstanceNode, HotSpotRegistersProvider registers, LoweringTool tool) {
732 Arguments args = new Arguments(allocateInstanceDynamic, newInstanceNode.graph().getGuardsStage(), tool.getLoweringStage());
733 OptionValues localOptions = newInstanceNode.getOptions();
734 args.add("type", newInstanceNode.getInstanceType());
735 ValueNode classClass = newInstanceNode.getClassClass();
736 assert classClass != null;
737 args.add("classClass", classClass);
738 args.addConst("fillContents", newInstanceNode.fillContents());
739 args.addConst("threadRegister", registers.getThreadRegister());
740 args.addConst("options", localOptions);
741 args.addConst("counters", counters);
742
743 SnippetTemplate template = template(newInstanceNode, args);
744 template.instantiate(providers.getMetaAccess(), newInstanceNode, DEFAULT_REPLACER, args);
745 }
746
747 public void lower(DynamicNewArrayNode newArrayNode, HotSpotRegistersProvider registers, LoweringTool tool) {
748 StructuredGraph graph = newArrayNode.graph();
749 OptionValues localOptions = graph.getOptions();
750 Arguments args = new Arguments(allocateArrayDynamic, newArrayNode.graph().getGuardsStage(), tool.getLoweringStage());
751 args.add("elementType", newArrayNode.getElementType());
752 ValueNode voidClass = newArrayNode.getVoidClass();
753 assert voidClass != null;
754 args.add("voidClass", voidClass);
755 ValueNode length = newArrayNode.length();
756 args.add("length", length.isAlive() ? length : graph.addOrUniqueWithInputs(length));
757 args.addConst("fillContents", newArrayNode.fillContents());
758 args.addConst("threadRegister", registers.getThreadRegister());
759 /*
760 * We use Kind.Illegal as a marker value instead of null because constant snippet
761 * parameters cannot be null.
762 */
763 args.addConst("knownElementKind", newArrayNode.getKnownElementKind() == null ? JavaKind.Illegal : newArrayNode.getKnownElementKind());
764 if (newArrayNode.getKnownElementKind() != null) {
765 args.addConst("knownLayoutHelper", lookupArrayClass(tool, newArrayNode.getKnownElementKind()).layoutHelper());
766 } else {
767 args.addConst("knownLayoutHelper", 0);
768 }
769 args.add("prototypeMarkWord", lookupArrayClass(tool, JavaKind.Object).prototypeMarkWord());
770 args.addConst("options", localOptions);
771 args.addConst("counters", counters);
772 SnippetTemplate template = template(newArrayNode, args);
773 template.instantiate(providers.getMetaAccess(), newArrayNode, DEFAULT_REPLACER, args);
774 }
775
776 private static HotSpotResolvedObjectType lookupArrayClass(LoweringTool tool, JavaKind kind) {
777 return (HotSpotResolvedObjectType) tool.getMetaAccess().lookupJavaType(kind == JavaKind.Object ? Object.class : kind.toJavaClass()).getArrayClass();
778 }
779
780 public void lower(NewMultiArrayNode newmultiarrayNode, LoweringTool tool) {
781 StructuredGraph graph = newmultiarrayNode.graph();
782 OptionValues localOptions = graph.getOptions();
783 int rank = newmultiarrayNode.dimensionCount();
784 ValueNode[] dims = new ValueNode[rank];
785 for (int i = 0; i < newmultiarrayNode.dimensionCount(); i++) {
786 dims[i] = newmultiarrayNode.dimension(i);
787 }
788 HotSpotResolvedObjectType type = (HotSpotResolvedObjectType) newmultiarrayNode.type();
789 ConstantNode hub = ConstantNode.forConstant(KlassPointerStamp.klassNonNull(), type.klass(), providers.getMetaAccess(), graph);
790
791 SnippetInfo snippet = GeneratePIC.getValue(localOptions) ? newmultiarrayPIC : newmultiarray;
792 Arguments args = new Arguments(snippet, graph.getGuardsStage(), tool.getLoweringStage());
793 args.add("hub", hub);
794 args.addConst("rank", rank);
795 args.addVarargs("dimensions", int.class, StampFactory.forKind(JavaKind.Int), dims);
796 template(newmultiarrayNode, args).instantiate(providers.getMetaAccess(), newmultiarrayNode, DEFAULT_REPLACER, args);
797 }
798
799 private static int instanceSize(HotSpotResolvedObjectType type) {
800 int size = type.instanceSize();
801 assert size >= 0;
802 return size;
803 }
804
805 public void lower(VerifyHeapNode verifyHeapNode, HotSpotRegistersProvider registers, LoweringTool tool) {
806 if (config.cAssertions) {
807 Arguments args = new Arguments(verifyHeap, verifyHeapNode.graph().getGuardsStage(), tool.getLoweringStage());
808 args.addConst("threadRegister", registers.getThreadRegister());
809
810 SnippetTemplate template = template(verifyHeapNode, args);
811 template.instantiate(providers.getMetaAccess(), verifyHeapNode, DEFAULT_REPLACER, args);
812 } else {
813 GraphUtil.removeFixedWithUnusedInputs(verifyHeapNode);
814 }
815 }
816 }
817 }