1 /*
2 * Copyright (c) 2012, 2019, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23
24
25 package org.graalvm.compiler.hotspot.replacements;
26
27 import static jdk.vm.ci.meta.DeoptimizationAction.None;
28 import static jdk.vm.ci.meta.DeoptimizationReason.RuntimeConstraint;
29 import static org.graalvm.compiler.core.common.GraalOptions.GeneratePIC;
30 import static org.graalvm.compiler.core.common.GraalOptions.MinimalBulkZeroingSize;
31 import static org.graalvm.compiler.core.common.calc.UnsignedMath.belowThan;
32 import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfigBase.INJECTED_OPTIONVALUES;
33 import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfigBase.INJECTED_VMCONFIG;
34 import static org.graalvm.compiler.hotspot.HotSpotBackend.NEW_ARRAY;
35 import static org.graalvm.compiler.hotspot.HotSpotBackend.NEW_ARRAY_OR_NULL;
36 import static org.graalvm.compiler.hotspot.HotSpotBackend.NEW_INSTANCE;
37 import static org.graalvm.compiler.hotspot.HotSpotBackend.NEW_INSTANCE_OR_NULL;
38 import static org.graalvm.compiler.hotspot.HotSpotBackend.NEW_MULTI_ARRAY;
39 import static org.graalvm.compiler.hotspot.HotSpotBackend.NEW_MULTI_ARRAY_OR_NULL;
40 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.CLASS_ARRAY_KLASS_LOCATION;
41 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.HUB_WRITE_LOCATION;
42 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.MARK_WORD_LOCATION;
43 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.PROTOTYPE_MARK_WORD_LOCATION;
44 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_END_LOCATION;
45 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_TOP_LOCATION;
46 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.allocateInstancePrefetchLines;
47 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.allocatePrefetchDistance;
48 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.allocatePrefetchLines;
49 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.allocatePrefetchStepSize;
50 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.allocatePrefetchStyle;
51 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayAllocationSize;
52 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayKlassOffset;
53 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayLengthOffset;
54 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.initializeObjectHeader;
55 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.instanceHeaderSize;
56 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.isInstanceKlassFullyInitialized;
57 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperHeaderSizeMask;
58 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperHeaderSizeShift;
59 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperLog2ElementSizeMask;
60 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperLog2ElementSizeShift;
61 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.loadKlassFromObject;
62 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.prototypeMarkWordOffset;
63 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readLayoutHelper;
64 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readTlabEnd;
65 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readTlabTop;
66 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.registerAsWord;
67 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.useBiasedLocking;
68 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.useTLAB;
69 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.verifyOop;
70 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.writeTlabTop;
71 import static org.graalvm.compiler.hotspot.replacements.HotspotSnippetsOptions.ProfileAllocations;
72 import static org.graalvm.compiler.hotspot.replacements.HotspotSnippetsOptions.ProfileAllocationsContext;
73 import static org.graalvm.compiler.nodes.PiArrayNode.piArrayCastToSnippetReplaceeStamp;
74 import static org.graalvm.compiler.nodes.PiNode.piCastToSnippetReplaceeStamp;
75 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.FAST_PATH_PROBABILITY;
76 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.FREQUENT_PROBABILITY;
77 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.SLOW_PATH_PROBABILITY;
78 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.probability;
79 import static org.graalvm.compiler.replacements.ReplacementsUtil.REPLACEMENTS_ASSERTIONS_ENABLED;
80 import static org.graalvm.compiler.replacements.ReplacementsUtil.runtimeAssert;
81 import static org.graalvm.compiler.replacements.ReplacementsUtil.staticAssert;
82 import static org.graalvm.compiler.replacements.SnippetTemplate.DEFAULT_REPLACER;
83 import static org.graalvm.compiler.replacements.nodes.CStringConstant.cstring;
84 import static org.graalvm.compiler.replacements.nodes.ExplodeLoopNode.explodeLoop;
85
86 import org.graalvm.compiler.api.replacements.Fold;
87 import org.graalvm.compiler.api.replacements.Fold.InjectedParameter;
88 import org.graalvm.compiler.api.replacements.Snippet;
89 import org.graalvm.compiler.api.replacements.Snippet.ConstantParameter;
90 import org.graalvm.compiler.api.replacements.Snippet.VarargsParameter;
91 import org.graalvm.compiler.core.common.spi.ForeignCallDescriptor;
92 import org.graalvm.compiler.core.common.type.StampFactory;
93 import org.graalvm.compiler.debug.DebugHandlersFactory;
94 import org.graalvm.compiler.debug.GraalError;
95 import org.graalvm.compiler.graph.Node.ConstantNodeParameter;
96 import org.graalvm.compiler.graph.Node.NodeIntrinsic;
97 import org.graalvm.compiler.hotspot.GraalHotSpotVMConfig;
98 import org.graalvm.compiler.hotspot.meta.HotSpotProviders;
99 import org.graalvm.compiler.hotspot.meta.HotSpotRegistersProvider;
100 import org.graalvm.compiler.hotspot.nodes.DimensionsNode;
101 import org.graalvm.compiler.hotspot.nodes.aot.LoadConstantIndirectlyFixedNode;
102 import org.graalvm.compiler.hotspot.nodes.aot.LoadConstantIndirectlyNode;
103 import org.graalvm.compiler.hotspot.nodes.type.KlassPointerStamp;
104 import org.graalvm.compiler.hotspot.word.KlassPointer;
105 import org.graalvm.compiler.nodes.ConstantNode;
106 import org.graalvm.compiler.nodes.DeoptimizeNode;
107 import org.graalvm.compiler.nodes.PiNode;
108 import org.graalvm.compiler.nodes.PrefetchAllocateNode;
109 import org.graalvm.compiler.nodes.SnippetAnchorNode;
110 import org.graalvm.compiler.nodes.StructuredGraph;
111 import org.graalvm.compiler.nodes.ValueNode;
112 import org.graalvm.compiler.nodes.debug.DynamicCounterNode;
113 import org.graalvm.compiler.nodes.debug.VerifyHeapNode;
114 import org.graalvm.compiler.nodes.extended.ForeignCallNode;
115 import org.graalvm.compiler.nodes.extended.MembarNode;
116 import org.graalvm.compiler.nodes.java.DynamicNewArrayNode;
117 import org.graalvm.compiler.nodes.java.DynamicNewInstanceNode;
118 import org.graalvm.compiler.nodes.java.NewArrayNode;
119 import org.graalvm.compiler.nodes.java.NewInstanceNode;
120 import org.graalvm.compiler.nodes.java.NewMultiArrayNode;
121 import org.graalvm.compiler.nodes.memory.address.OffsetAddressNode;
122 import org.graalvm.compiler.nodes.spi.LoweringTool;
123 import org.graalvm.compiler.nodes.util.GraphUtil;
124 import org.graalvm.compiler.options.OptionValues;
125 import org.graalvm.compiler.replacements.ReplacementsUtil;
126 import org.graalvm.compiler.replacements.SnippetCounter;
127 import org.graalvm.compiler.replacements.SnippetCounter.Group;
128 import org.graalvm.compiler.replacements.SnippetTemplate;
129 import org.graalvm.compiler.replacements.SnippetTemplate.AbstractTemplates;
130 import org.graalvm.compiler.replacements.SnippetTemplate.Arguments;
131 import org.graalvm.compiler.replacements.SnippetTemplate.SnippetInfo;
132 import org.graalvm.compiler.replacements.Snippets;
133 import org.graalvm.compiler.replacements.nodes.ExplodeLoopNode;
134 import org.graalvm.compiler.replacements.nodes.ZeroMemoryNode;
135 import org.graalvm.compiler.word.Word;
136 import jdk.internal.vm.compiler.word.LocationIdentity;
137 import jdk.internal.vm.compiler.word.WordFactory;
138
139 import jdk.vm.ci.code.CodeUtil;
140 import jdk.vm.ci.code.MemoryBarriers;
141 import jdk.vm.ci.code.Register;
142 import jdk.vm.ci.code.TargetDescription;
143 import jdk.vm.ci.hotspot.HotSpotResolvedObjectType;
144 import jdk.vm.ci.meta.JavaKind;
145 import jdk.vm.ci.meta.ResolvedJavaType;
146
147 /**
148 * Snippets used for implementing NEW, ANEWARRAY and NEWARRAY.
149 */
150 public class NewObjectSnippets implements Snippets {
151
152 enum ProfileContext {
153 AllocatingMethod,
154 InstanceOrArray,
155 AllocatedType,
156 AllocatedTypesInMethod,
157 Total
158 }
159
160 @Fold
161 static String createName(@Fold.InjectedParameter OptionValues options, String path, String typeContext) {
162 switch (ProfileAllocationsContext.getValue(options)) {
163 case AllocatingMethod:
164 return "";
165 case InstanceOrArray:
166 return path;
167 case AllocatedType:
168 case AllocatedTypesInMethod:
169 return typeContext;
170 case Total:
171 return "bytes";
172 default:
173 throw GraalError.shouldNotReachHere();
174 }
175 }
176
177 @Fold
178 static boolean doProfile(@Fold.InjectedParameter OptionValues options) {
179 return ProfileAllocations.getValue(options);
180 }
181
182 @Fold
183 static boolean withContext(@Fold.InjectedParameter OptionValues options) {
184 ProfileContext context = ProfileAllocationsContext.getValue(options);
185 return context == ProfileContext.AllocatingMethod || context == ProfileContext.AllocatedTypesInMethod;
186 }
187
188 protected static void profileAllocation(String path, long size, String typeContext) {
189 if (doProfile(INJECTED_OPTIONVALUES)) {
190 String name = createName(INJECTED_OPTIONVALUES, path, typeContext);
191
192 boolean context = withContext(INJECTED_OPTIONVALUES);
193 DynamicCounterNode.counter("number of bytes allocated", name, size, context);
194 DynamicCounterNode.counter("number of allocations", name, 1, context);
195 }
196 }
197
198 public static void emitPrefetchAllocate(Word address, boolean isArray) {
199 if (allocatePrefetchStyle(INJECTED_VMCONFIG) > 0) {
200 // Insert a prefetch for each allocation only on the fast-path
201 // Generate several prefetch instructions.
202 int lines = isArray ? allocatePrefetchLines(INJECTED_VMCONFIG) : allocateInstancePrefetchLines(INJECTED_VMCONFIG);
203 int stepSize = allocatePrefetchStepSize(INJECTED_VMCONFIG);
204 int distance = allocatePrefetchDistance(INJECTED_VMCONFIG);
205 ExplodeLoopNode.explodeLoop();
206 for (int i = 0; i < lines; i++) {
207 PrefetchAllocateNode.prefetch(OffsetAddressNode.address(address, distance));
208 distance += stepSize;
209 }
210 }
211 }
212
213 @Snippet
214 public static Object allocateInstance(@ConstantParameter long size, KlassPointer hub, Word prototypeMarkWord, @ConstantParameter boolean fillContents,
215 @ConstantParameter Register threadRegister, @ConstantParameter boolean constantSize, @ConstantParameter String typeContext,
216 @ConstantParameter Counters counters) {
217 return piCastToSnippetReplaceeStamp(allocateInstanceHelper(size, hub, prototypeMarkWord, fillContents, threadRegister, constantSize, typeContext, counters));
218 }
219
220 public static Object allocateInstanceHelper(long size, KlassPointer hub, Word prototypeMarkWord, boolean fillContents,
221 Register threadRegister, boolean constantSize, String typeContext, Counters counters) {
222 Object result;
223 Word thread = registerAsWord(threadRegister);
224 Word top = readTlabTop(thread);
225 Word end = readTlabEnd(thread);
226 Word newTop = top.add(WordFactory.unsigned(size));
227 if (useTLAB(INJECTED_VMCONFIG) && probability(FAST_PATH_PROBABILITY, newTop.belowOrEqual(end))) {
228 writeTlabTop(thread, newTop);
229 emitPrefetchAllocate(newTop, false);
230 result = formatObject(hub, size, top, prototypeMarkWord, fillContents, constantSize, counters);
231 } else {
232 Counters theCounters = counters;
233 if (theCounters != null && theCounters.stub != null) {
234 theCounters.stub.inc();
235 }
236 result = newInstanceStub(hub);
237 }
238 profileAllocation("instance", size, typeContext);
239 return verifyOop(result);
240 }
241
242 public static Object newInstanceStub(KlassPointer hub) {
243 if (useNullAllocationStubs(INJECTED_VMCONFIG)) {
244 return nonNullOrDeopt(newInstanceOrNull(NEW_INSTANCE_OR_NULL, hub));
245 } else {
246 return newInstance(NEW_INSTANCE, hub);
247 }
248 }
249
250 @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = true)
251 private static native Object newInstance(@ConstantNodeParameter ForeignCallDescriptor descriptor, KlassPointer hub);
252
253 @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = false)
254 private static native Object newInstanceOrNull(@ConstantNodeParameter ForeignCallDescriptor descriptor, KlassPointer hub);
255
256 @Snippet
257 public static Object allocateInstancePIC(@ConstantParameter long size, KlassPointer hub, Word prototypeMarkWord, @ConstantParameter boolean fillContents,
258 @ConstantParameter Register threadRegister, @ConstantParameter boolean constantSize, @ConstantParameter String typeContext,
259 @ConstantParameter Counters counters) {
260 // Klass must be initialized by the time the first instance is allocated, therefore we can
261 // just load it from the corresponding cell and avoid the resolution check. We have to use a
262 // fixed load though, to prevent it from floating above the initialization.
263 KlassPointer picHub = LoadConstantIndirectlyFixedNode.loadKlass(hub);
264 return piCastToSnippetReplaceeStamp(allocateInstanceHelper(size, picHub, prototypeMarkWord, fillContents, threadRegister, constantSize, typeContext, counters));
265 }
266
267 @Snippet
268 public static Object allocateInstanceDynamic(Class<?> type, Class<?> classClass, @ConstantParameter boolean fillContents, @ConstantParameter Register threadRegister,
269 @ConstantParameter Counters counters) {
270 if (probability(SLOW_PATH_PROBABILITY, type == null)) {
271 DeoptimizeNode.deopt(None, RuntimeConstraint);
272 }
273 Class<?> nonNullType = PiNode.piCastNonNullClass(type, SnippetAnchorNode.anchor());
274
275 if (probability(SLOW_PATH_PROBABILITY, DynamicNewInstanceNode.throwsInstantiationException(type, classClass))) {
276 DeoptimizeNode.deopt(None, RuntimeConstraint);
277 }
278
279 return PiNode.piCastToSnippetReplaceeStamp(allocateInstanceDynamicHelper(type, fillContents, threadRegister, counters, nonNullType));
280 }
281
282 private static Object allocateInstanceDynamicHelper(Class<?> type, boolean fillContents, Register threadRegister, Counters counters, Class<?> nonNullType) {
283 KlassPointer hub = ClassGetHubNode.readClass(nonNullType);
284 if (probability(FAST_PATH_PROBABILITY, !hub.isNull())) {
285 KlassPointer nonNullHub = ClassGetHubNode.piCastNonNull(hub, SnippetAnchorNode.anchor());
286
287 if (probability(FAST_PATH_PROBABILITY, isInstanceKlassFullyInitialized(nonNullHub))) {
288 int layoutHelper = readLayoutHelper(nonNullHub);
289 /*
290 * src/share/vm/oops/klass.hpp: For instances, layout helper is a positive number,
291 * the instance size. This size is already passed through align_object_size and
292 * scaled to bytes. The low order bit is set if instances of this class cannot be
293 * allocated using the fastpath.
294 */
295 if (probability(FAST_PATH_PROBABILITY, (layoutHelper & 1) == 0)) {
296 Word prototypeMarkWord = nonNullHub.readWord(prototypeMarkWordOffset(INJECTED_VMCONFIG), PROTOTYPE_MARK_WORD_LOCATION);
297 /*
298 * FIXME(je,ds): we should actually pass typeContext instead of "" but late
299 * binding of parameters is not yet supported by the GraphBuilderPlugin system.
300 */
301 return allocateInstanceHelper(layoutHelper, nonNullHub, prototypeMarkWord, fillContents, threadRegister, false, "", counters);
302 }
303 } else {
304 DeoptimizeNode.deopt(None, RuntimeConstraint);
305 }
306 }
307 return dynamicNewInstanceStub(type);
308 }
309
310 /**
311 * Maximum array length for which fast path allocation is used.
312 */
313 public static final int MAX_ARRAY_FAST_PATH_ALLOCATION_LENGTH = 0x00FFFFFF;
314
315 @Snippet
316 public static Object allocatePrimitiveArrayPIC(KlassPointer hub,
317 int length,
318 Word prototypeMarkWord,
319 @ConstantParameter int headerSize,
320 @ConstantParameter int log2ElementSize,
321 @ConstantParameter boolean fillContents,
322 @ConstantParameter Register threadRegister,
323 @ConstantParameter boolean maybeUnroll,
324 @ConstantParameter String typeContext,
325 @ConstantParameter boolean useBulkZeroing,
326 @ConstantParameter Counters counters) {
327 // Primitive array types are eagerly pre-resolved. We can use a floating load.
328 KlassPointer picHub = LoadConstantIndirectlyNode.loadKlass(hub);
329 return allocateArrayImpl(picHub, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents,
330 threadRegister, maybeUnroll, typeContext, useBulkZeroing, counters);
331 }
332
333 @Snippet
334 public static Object allocateArrayPIC(KlassPointer hub,
335 int length,
336 Word prototypeMarkWord,
337 @ConstantParameter int headerSize,
338 @ConstantParameter int log2ElementSize,
339 @ConstantParameter boolean fillContents,
340 @ConstantParameter Register threadRegister,
341 @ConstantParameter boolean maybeUnroll,
342 @ConstantParameter String typeContext,
343 @ConstantParameter boolean useBulkZeroing,
344 @ConstantParameter Counters counters) {
345 // Array type would be resolved by dominating resolution.
346 KlassPointer picHub = LoadConstantIndirectlyFixedNode.loadKlass(hub);
347 return allocateArrayImpl(picHub, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents,
348 threadRegister, maybeUnroll, typeContext, useBulkZeroing, counters);
349 }
350
351 @Snippet
352 public static Object allocateArray(KlassPointer hub,
353 int length,
354 Word prototypeMarkWord,
355 @ConstantParameter int headerSize,
356 @ConstantParameter int log2ElementSize,
357 @ConstantParameter boolean fillContents,
358 @ConstantParameter Register threadRegister,
359 @ConstantParameter boolean maybeUnroll,
360 @ConstantParameter String typeContext,
361 @ConstantParameter boolean useBulkZeroing,
362 @ConstantParameter Counters counters) {
363 Object result = allocateArrayImpl(hub,
364 length,
365 prototypeMarkWord,
366 headerSize,
367 log2ElementSize,
368 fillContents,
369 threadRegister,
370 maybeUnroll,
371 typeContext,
372 useBulkZeroing,
373 counters);
374 return piArrayCastToSnippetReplaceeStamp(verifyOop(result), length);
375 }
376
377 /**
378 * When allocating on the slow path, determines whether to use a version of the runtime call
379 * that returns {@code null} on a failed allocation instead of raising an OutOfMemoryError.
380 */
381 @Fold
382 static boolean useNullAllocationStubs(@InjectedParameter GraalHotSpotVMConfig config) {
383 return config.areNullAllocationStubsAvailable();
384 }
385
386 private static Object allocateArrayImpl(KlassPointer hub, int length, Word prototypeMarkWord, int headerSize, int log2ElementSize, boolean fillContents, Register threadRegister,
387 boolean maybeUnroll, String typeContext, boolean useBulkZeroing, Counters counters) {
388 Object result;
389 long allocationSize = arrayAllocationSize(length, headerSize, log2ElementSize);
390 Word thread = registerAsWord(threadRegister);
391 Word top = readTlabTop(thread);
392 Word end = readTlabEnd(thread);
393 Word newTop = top.add(WordFactory.unsigned(allocationSize));
394 if (probability(FREQUENT_PROBABILITY, belowThan(length, MAX_ARRAY_FAST_PATH_ALLOCATION_LENGTH)) && useTLAB(INJECTED_VMCONFIG) &&
395 probability(FAST_PATH_PROBABILITY, newTop.belowOrEqual(end))) {
396 writeTlabTop(thread, newTop);
397 emitPrefetchAllocate(newTop, true);
398 Counters theCounters = counters;
399 if (theCounters != null && theCounters.arrayLoopInit != null) {
400 theCounters.arrayLoopInit.inc();
401 }
402 result = formatArray(hub, allocationSize, length, headerSize, top, prototypeMarkWord, fillContents, maybeUnroll, useBulkZeroing, counters);
403 } else {
404 result = newArrayStub(hub, length);
405 }
406 profileAllocation("array", allocationSize, typeContext);
407 return result;
408 }
409
410 public static Object newArrayStub(KlassPointer hub, int length) {
411 if (useNullAllocationStubs(INJECTED_VMCONFIG)) {
412 return nonNullOrDeopt(newArrayOrNull(NEW_ARRAY_OR_NULL, hub, length));
413 } else {
414 return newArray(NEW_ARRAY, hub, length);
415 }
416 }
417
418 @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = true)
419 private static native Object newArray(@ConstantNodeParameter ForeignCallDescriptor descriptor, KlassPointer hub, int length);
420
421 @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = false)
422 private static native Object newArrayOrNull(@ConstantNodeParameter ForeignCallDescriptor descriptor, KlassPointer hub, int length);
423
424 /**
425 * New dynamic array stub that throws an {@link OutOfMemoryError} on allocation failure.
426 */
427 public static final ForeignCallDescriptor DYNAMIC_NEW_INSTANCE = new ForeignCallDescriptor("dynamic_new_instance", Object.class, Class.class);
428
429 /**
430 * New dynamic array stub that returns null on allocation failure.
431 */
432 public static final ForeignCallDescriptor DYNAMIC_NEW_INSTANCE_OR_NULL = new ForeignCallDescriptor("dynamic_new_instance_or_null", Object.class, Class.class);
433
434 public static Object dynamicNewInstanceStub(Class<?> elementType) {
435 if (useNullAllocationStubs(INJECTED_VMCONFIG)) {
436 return nonNullOrDeopt(dynamicNewInstanceOrNull(DYNAMIC_NEW_INSTANCE_OR_NULL, elementType));
437 } else {
438 return dynamicNewInstance(DYNAMIC_NEW_INSTANCE, elementType);
439 }
440 }
441
442 /**
443 * Deoptimizes if {@code obj == null} otherwise returns {@code obj}.
444 */
445 private static Object nonNullOrDeopt(Object obj) {
446 if (obj == null) {
447 DeoptimizeNode.deopt(None, RuntimeConstraint);
448 }
449 return obj;
450 }
451
452 @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = true)
453 public static native Object dynamicNewInstance(@ConstantNodeParameter ForeignCallDescriptor descriptor, Class<?> elementType);
454
455 @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = false)
456 public static native Object dynamicNewInstanceOrNull(@ConstantNodeParameter ForeignCallDescriptor descriptor, Class<?> elementType);
457
458 @Snippet
459 public static Object allocateArrayDynamic(Class<?> elementType,
460 Class<?> voidClass,
461 int length,
462 @ConstantParameter boolean fillContents,
463 @ConstantParameter Register threadRegister,
464 @ConstantParameter JavaKind knownElementKind,
465 @ConstantParameter int knownLayoutHelper,
466 @ConstantParameter boolean useBulkZeroing,
467 Word prototypeMarkWord,
468 @ConstantParameter Counters counters) {
469 Object result = allocateArrayDynamicImpl(elementType, voidClass, length, fillContents, threadRegister, knownElementKind,
470 knownLayoutHelper, useBulkZeroing, prototypeMarkWord, counters);
471 return result;
472 }
473
474 private static Object allocateArrayDynamicImpl(Class<?> elementType, Class<?> voidClass, int length, boolean fillContents, Register threadRegister, JavaKind knownElementKind,
475 int knownLayoutHelper, boolean useBulkZeroing, Word prototypeMarkWord, Counters counters) {
476 /*
477 * We only need the dynamic check for void when we have no static information from
478 * knownElementKind.
479 */
480 staticAssert(knownElementKind != JavaKind.Void, "unsupported knownElementKind");
481 if (knownElementKind == JavaKind.Illegal && probability(SLOW_PATH_PROBABILITY, elementType == null || DynamicNewArrayNode.throwsIllegalArgumentException(elementType, voidClass))) {
482 DeoptimizeNode.deopt(None, RuntimeConstraint);
483 }
484
485 KlassPointer klass = loadKlassFromObject(elementType, arrayKlassOffset(INJECTED_VMCONFIG), CLASS_ARRAY_KLASS_LOCATION);
486 if (klass.isNull()) {
487 DeoptimizeNode.deopt(None, RuntimeConstraint);
488 }
489 KlassPointer nonNullKlass = ClassGetHubNode.piCastNonNull(klass, SnippetAnchorNode.anchor());
490
491 if (length < 0) {
492 DeoptimizeNode.deopt(None, RuntimeConstraint);
493 }
494 int layoutHelper;
495 if (knownElementKind == JavaKind.Illegal) {
496 layoutHelper = readLayoutHelper(nonNullKlass);
497 } else {
498 runtimeAssert(knownLayoutHelper == readLayoutHelper(nonNullKlass), "layout mismatch");
499 layoutHelper = knownLayoutHelper;
500 }
501 //@formatter:off
502 // from src/share/vm/oops/klass.hpp:
503 //
504 // For arrays, layout helper is a negative number, containing four
505 // distinct bytes, as follows:
506 // MSB:[tag, hsz, ebt, log2(esz)]:LSB
507 // where:
508 // tag is 0x80 if the elements are oops, 0xC0 if non-oops
509 // hsz is array header size in bytes (i.e., offset of first element)
510 // ebt is the BasicType of the elements
511 // esz is the element size in bytes
512 //@formatter:on
513
514 int headerSize = (layoutHelper >> layoutHelperHeaderSizeShift(INJECTED_VMCONFIG)) & layoutHelperHeaderSizeMask(INJECTED_VMCONFIG);
515 int log2ElementSize = (layoutHelper >> layoutHelperLog2ElementSizeShift(INJECTED_VMCONFIG)) & layoutHelperLog2ElementSizeMask(INJECTED_VMCONFIG);
516
517 Object result = allocateArrayImpl(nonNullKlass, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents,
518 threadRegister, false, "dynamic type", useBulkZeroing, counters);
519 return piArrayCastToSnippetReplaceeStamp(verifyOop(result), length);
520 }
521
522 /**
523 * Calls the runtime stub for implementing MULTIANEWARRAY.
524 */
525 @Snippet
526 private static Object newmultiarray(KlassPointer hub, @ConstantParameter int rank, @VarargsParameter int[] dimensions) {
527 Word dims = DimensionsNode.allocaDimsArray(rank);
528 ExplodeLoopNode.explodeLoop();
529 for (int i = 0; i < rank; i++) {
530 dims.writeInt(i * 4, dimensions[i], LocationIdentity.init());
531 }
532 return newMultiArrayStub(hub, rank, dims);
533 }
534
535 private static Object newMultiArrayStub(KlassPointer hub, int rank, Word dims) {
536 if (useNullAllocationStubs(INJECTED_VMCONFIG)) {
537 return nonNullOrDeopt(newMultiArrayOrNull(NEW_MULTI_ARRAY_OR_NULL, hub, rank, dims));
538 } else {
539 return newMultiArray(NEW_MULTI_ARRAY, hub, rank, dims);
540 }
541 }
542
543 @Snippet
544 private static Object newmultiarrayPIC(KlassPointer hub, @ConstantParameter int rank, @VarargsParameter int[] dimensions) {
545 // Array type would be resolved by dominating resolution.
546 KlassPointer picHub = LoadConstantIndirectlyFixedNode.loadKlass(hub);
547 return newmultiarray(picHub, rank, dimensions);
548 }
549
550 @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = true)
551 private static native Object newMultiArray(@ConstantNodeParameter ForeignCallDescriptor descriptor, KlassPointer hub, int rank, Word dims);
552
553 @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = false)
554 private static native Object newMultiArrayOrNull(@ConstantNodeParameter ForeignCallDescriptor descriptor, KlassPointer hub, int rank, Word dims);
555
556 /**
557 * Maximum number of long stores to emit when zeroing an object with a constant size. Larger
558 * objects have their bodies initialized in a loop.
559 */
560 private static final int MAX_UNROLLED_OBJECT_ZEROING_STORES = 8;
561
562 /**
563 * Zero uninitialized memory in a newly allocated object, unrolling as necessary and ensuring
564 * that stores are aligned.
565 *
566 * @param size number of bytes to zero
567 * @param memory beginning of object which is being zeroed
568 * @param constantSize is {@code size} known to be constant in the snippet
569 * @param startOffset offset to begin zeroing. May not be word aligned.
570 * @param manualUnroll maximally unroll zeroing
571 * @param useBulkZeroing apply bulk zeroing
572 */
573 private static void zeroMemory(long size, Word memory, boolean constantSize, int startOffset, boolean manualUnroll,
574 boolean useBulkZeroing, Counters counters) {
575 fillMemory(0, size, memory, constantSize, startOffset, manualUnroll, useBulkZeroing, counters);
576 }
577
578 private static void fillMemory(long value, long size, Word memory, boolean constantSize, int startOffset, boolean manualUnroll,
579 boolean useBulkZeroing, Counters counters) {
580 ReplacementsUtil.runtimeAssert((size & 0x7) == 0, "unaligned object size");
581 int offset = startOffset;
582 if ((offset & 0x7) != 0) {
583 memory.writeInt(offset, (int) value, LocationIdentity.init());
584 offset += 4;
585 }
586 ReplacementsUtil.runtimeAssert((offset & 0x7) == 0, "unaligned offset");
587 Counters theCounters = counters;
588 if (manualUnroll && ((size - offset) / 8) <= MAX_UNROLLED_OBJECT_ZEROING_STORES) {
589 ReplacementsUtil.staticAssert(!constantSize, "size shouldn't be constant at instantiation time");
590 // This case handles arrays of constant length. Instead of having a snippet variant for
591 // each length, generate a chain of stores of maximum length. Once it's inlined the
592 // break statement will trim excess stores.
593 if (theCounters != null && theCounters.instanceSeqInit != null) {
594 theCounters.instanceSeqInit.inc();
595 }
596
597 explodeLoop();
598 for (int i = 0; i < MAX_UNROLLED_OBJECT_ZEROING_STORES; i++, offset += 8) {
599 if (offset == size) {
600 break;
601 }
602 memory.initializeLong(offset, value, LocationIdentity.init());
603 }
604 } else {
605 // Use Word instead of int to avoid extension to long in generated code
606 Word off = WordFactory.signed(offset);
607 if (useBulkZeroing && probability(SLOW_PATH_PROBABILITY, size >= getMinimalBulkZeroingSize(INJECTED_OPTIONVALUES))) {
608 if (theCounters != null && theCounters.instanceBulkInit != null) {
609 theCounters.instanceBulkInit.inc();
610 }
611 ZeroMemoryNode.zero(memory.add(off), size - offset, LocationIdentity.init());
612 } else {
613 if (constantSize && ((size - offset) / 8) <= MAX_UNROLLED_OBJECT_ZEROING_STORES) {
614 if (theCounters != null && theCounters.instanceSeqInit != null) {
615 theCounters.instanceSeqInit.inc();
616 }
617 explodeLoop();
618 } else {
619 if (theCounters != null && theCounters.instanceLoopInit != null) {
620 theCounters.instanceLoopInit.inc();
621 }
622 }
623 for (; off.rawValue() < size; off = off.add(8)) {
624 memory.initializeLong(off, value, LocationIdentity.init());
625 }
626
627 }
628 }
629 }
630
631 @Fold
632 static int getMinimalBulkZeroingSize(@InjectedParameter OptionValues optionValues) {
633 return MinimalBulkZeroingSize.getValue(optionValues);
634 }
635
636 /**
637 * Fill uninitialized memory with garbage value in a newly allocated object, unrolling as
638 * necessary and ensuring that stores are aligned.
639 *
640 * @param size number of bytes to zero
641 * @param memory beginning of object which is being zeroed
642 * @param constantSize is {@code size} known to be constant in the snippet
643 * @param startOffset offset to begin zeroing. May not be word aligned.
644 * @param manualUnroll maximally unroll zeroing
645 */
646 private static void fillWithGarbage(long size, Word memory, boolean constantSize, int startOffset, boolean manualUnroll, Counters counters) {
647 fillMemory(0xfefefefefefefefeL, size, memory, constantSize, startOffset, manualUnroll, false, counters);
648 }
649
650 /**
651 * Formats some allocated memory with an object header and zeroes out the rest.
652 */
653 private static Object formatObject(KlassPointer hub, long size, Word memory, Word compileTimePrototypeMarkWord, boolean fillContents, boolean constantSize, Counters counters) {
654 Word prototypeMarkWord = useBiasedLocking(INJECTED_VMCONFIG) ? hub.readWord(prototypeMarkWordOffset(INJECTED_VMCONFIG), PROTOTYPE_MARK_WORD_LOCATION) : compileTimePrototypeMarkWord;
655 initializeObjectHeader(memory, prototypeMarkWord, hub);
656 if (fillContents) {
657 zeroMemory(size, memory, constantSize, instanceHeaderSize(INJECTED_VMCONFIG), false, false, counters);
658 } else if (REPLACEMENTS_ASSERTIONS_ENABLED) {
659 fillWithGarbage(size, memory, constantSize, instanceHeaderSize(INJECTED_VMCONFIG), false, counters);
660 }
661 MembarNode.memoryBarrier(MemoryBarriers.STORE_STORE, LocationIdentity.init());
662 return memory.toObjectNonNull();
663 }
664
665 @Snippet
666 private static void verifyHeap(@ConstantParameter Register threadRegister) {
667 Word thread = registerAsWord(threadRegister);
668 Word topValue = readTlabTop(thread);
669 if (!topValue.equal(WordFactory.zero())) {
670 Word topValueContents = topValue.readWord(0, MARK_WORD_LOCATION);
671 if (topValueContents.equal(WordFactory.zero())) {
672 AssertionSnippets.vmMessageC(AssertionSnippets.ASSERTION_VM_MESSAGE_C, true, cstring("overzeroing of TLAB detected"), 0L, 0L, 0L);
673 }
674 }
675 }
676
677 /**
678 * Formats some allocated memory with an object header and zeroes out the rest.
679 */
680 private static Object formatArray(KlassPointer hub, long allocationSize, int length, int headerSize, Word memory, Word prototypeMarkWord, boolean fillContents, boolean maybeUnroll,
681 boolean useBulkZeroing, Counters counters) {
682 memory.writeInt(arrayLengthOffset(INJECTED_VMCONFIG), length, LocationIdentity.init());
683 /*
684 * store hub last as the concurrent garbage collectors assume length is valid if hub field
685 * is not null
686 */
687 initializeObjectHeader(memory, prototypeMarkWord, hub);
688 if (fillContents) {
689 zeroMemory(allocationSize, memory, false, headerSize, maybeUnroll, useBulkZeroing, counters);
690 } else if (REPLACEMENTS_ASSERTIONS_ENABLED) {
691 fillWithGarbage(allocationSize, memory, false, headerSize, maybeUnroll, counters);
692 }
693 MembarNode.memoryBarrier(MemoryBarriers.STORE_STORE, LocationIdentity.init());
694 return memory.toObjectNonNull();
695 }
696
697 static class Counters {
698 Counters(SnippetCounter.Group.Factory factory) {
699 Group newInstance = factory.createSnippetCounterGroup("NewInstance");
700 Group newArray = factory.createSnippetCounterGroup("NewArray");
701 instanceSeqInit = new SnippetCounter(newInstance, "tlabSeqInit", "TLAB alloc with unrolled zeroing");
702 instanceLoopInit = new SnippetCounter(newInstance, "tlabLoopInit", "TLAB alloc with zeroing in a loop");
703 instanceBulkInit = new SnippetCounter(newArray, "tlabBulkInit", "TLAB alloc with bulk zeroing");
704 arrayLoopInit = new SnippetCounter(newArray, "tlabLoopInit", "TLAB alloc with zeroing in a loop");
705 stub = new SnippetCounter(newInstance, "stub", "alloc and zeroing via stub");
706 }
707
708 final SnippetCounter instanceSeqInit;
709 final SnippetCounter instanceLoopInit;
710 final SnippetCounter instanceBulkInit;
711 final SnippetCounter arrayLoopInit;
712 final SnippetCounter stub;
713 }
714
715 public static class Templates extends AbstractTemplates {
716
717 private final SnippetInfo allocateInstance = snippet(NewObjectSnippets.class, "allocateInstance", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION, TLAB_END_LOCATION);
718 private final SnippetInfo allocateInstancePIC = snippet(NewObjectSnippets.class, "allocateInstancePIC", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION,
719 TLAB_END_LOCATION);
720 private final SnippetInfo allocateArray = snippet(NewObjectSnippets.class, "allocateArray", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION, TLAB_END_LOCATION);
721 private final SnippetInfo allocateArrayPIC = snippet(NewObjectSnippets.class, "allocateArrayPIC", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION, TLAB_END_LOCATION);
722 private final SnippetInfo allocatePrimitiveArrayPIC = snippet(NewObjectSnippets.class, "allocatePrimitiveArrayPIC", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION,
723 TLAB_END_LOCATION);
724 private final SnippetInfo allocateArrayDynamic = snippet(NewObjectSnippets.class, "allocateArrayDynamic", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION,
725 TLAB_END_LOCATION);
726 private final SnippetInfo allocateInstanceDynamic = snippet(NewObjectSnippets.class, "allocateInstanceDynamic", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION,
727 TLAB_END_LOCATION);
728 private final SnippetInfo newmultiarray = snippet(NewObjectSnippets.class, "newmultiarray", TLAB_TOP_LOCATION, TLAB_END_LOCATION);
729 private final SnippetInfo newmultiarrayPIC = snippet(NewObjectSnippets.class, "newmultiarrayPIC", TLAB_TOP_LOCATION, TLAB_END_LOCATION);
730 private final SnippetInfo verifyHeap = snippet(NewObjectSnippets.class, "verifyHeap");
731 private final GraalHotSpotVMConfig config;
732 private final Counters counters;
733
734 public Templates(OptionValues options, Iterable<DebugHandlersFactory> factories, SnippetCounter.Group.Factory factory, HotSpotProviders providers, TargetDescription target,
735 GraalHotSpotVMConfig config) {
736 super(options, factories, providers, providers.getSnippetReflection(), target);
737 this.config = config;
738 counters = new Counters(factory);
739 }
740
741 /**
742 * Lowers a {@link NewInstanceNode}.
743 */
744 public void lower(NewInstanceNode newInstanceNode, HotSpotRegistersProvider registers, LoweringTool tool) {
745 StructuredGraph graph = newInstanceNode.graph();
746 HotSpotResolvedObjectType type = (HotSpotResolvedObjectType) newInstanceNode.instanceClass();
747 assert !type.isArray();
748 ConstantNode hub = ConstantNode.forConstant(KlassPointerStamp.klassNonNull(), type.klass(), providers.getMetaAccess(), graph);
749 long size = instanceSize(type);
750
751 OptionValues localOptions = graph.getOptions();
752 SnippetInfo snippet = GeneratePIC.getValue(localOptions) ? allocateInstancePIC : allocateInstance;
753 Arguments args = new Arguments(snippet, graph.getGuardsStage(), tool.getLoweringStage());
754 args.addConst("size", size);
755 args.add("hub", hub);
756 args.add("prototypeMarkWord", type.prototypeMarkWord());
757 args.addConst("fillContents", newInstanceNode.fillContents());
758 args.addConst("threadRegister", registers.getThreadRegister());
759 args.addConst("constantSize", true);
760 args.addConst("typeContext", ProfileAllocations.getValue(localOptions) ? type.toJavaName(false) : "");
761 args.addConst("counters", counters);
762
763 SnippetTemplate template = template(newInstanceNode, args);
764 graph.getDebug().log("Lowering allocateInstance in %s: node=%s, template=%s, arguments=%s", graph, newInstanceNode, template, args);
765 template.instantiate(providers.getMetaAccess(), newInstanceNode, DEFAULT_REPLACER, args);
766 }
767
768 /**
769 * Lowers a {@link NewArrayNode}.
770 */
771 public void lower(NewArrayNode newArrayNode, HotSpotRegistersProvider registers, LoweringTool tool) {
772 StructuredGraph graph = newArrayNode.graph();
773 ResolvedJavaType elementType = newArrayNode.elementType();
774 HotSpotResolvedObjectType arrayType = (HotSpotResolvedObjectType) elementType.getArrayClass();
775 JavaKind elementKind = elementType.getJavaKind();
776 ConstantNode hub = ConstantNode.forConstant(KlassPointerStamp.klassNonNull(), arrayType.klass(), providers.getMetaAccess(), graph);
777 final int headerSize = tool.getMetaAccess().getArrayBaseOffset(elementKind);
778 int log2ElementSize = CodeUtil.log2(tool.getMetaAccess().getArrayIndexScale(elementKind));
779
780 OptionValues localOptions = graph.getOptions();
781 SnippetInfo snippet;
782 if (GeneratePIC.getValue(localOptions)) {
783 if (elementType.isPrimitive()) {
784 snippet = allocatePrimitiveArrayPIC;
785 } else {
786 snippet = allocateArrayPIC;
787 }
788 } else {
789 snippet = allocateArray;
790 }
791
792 Arguments args = new Arguments(snippet, graph.getGuardsStage(), tool.getLoweringStage());
793 args.add("hub", hub);
794 ValueNode length = newArrayNode.length();
795 args.add("length", length.isAlive() ? length : graph.addOrUniqueWithInputs(length));
796 assert arrayType.prototypeMarkWord() == lookupArrayClass(tool, JavaKind.Object).prototypeMarkWord() : "all array types are assumed to have the same prototypeMarkWord";
797 args.add("prototypeMarkWord", arrayType.prototypeMarkWord());
798 args.addConst("headerSize", headerSize);
799 args.addConst("log2ElementSize", log2ElementSize);
800 args.addConst("fillContents", newArrayNode.fillContents());
801 args.addConst("threadRegister", registers.getThreadRegister());
802 args.addConst("maybeUnroll", length.isConstant());
803 args.addConst("typeContext", ProfileAllocations.getValue(localOptions) ? arrayType.toJavaName(false) : "");
804 args.addConst("useBulkZeroing", tool.getLowerer().supportBulkZeroing());
805 args.addConst("counters", counters);
806 SnippetTemplate template = template(newArrayNode, args);
807 graph.getDebug().log("Lowering allocateArray in %s: node=%s, template=%s, arguments=%s", graph, newArrayNode, template, args);
808 template.instantiate(providers.getMetaAccess(), newArrayNode, DEFAULT_REPLACER, args);
809 }
810
811 public void lower(DynamicNewInstanceNode newInstanceNode, HotSpotRegistersProvider registers, LoweringTool tool) {
812 Arguments args = new Arguments(allocateInstanceDynamic, newInstanceNode.graph().getGuardsStage(), tool.getLoweringStage());
813 args.add("type", newInstanceNode.getInstanceType());
814 ValueNode classClass = newInstanceNode.getClassClass();
815 assert classClass != null;
816 args.add("classClass", classClass);
817 args.addConst("fillContents", newInstanceNode.fillContents());
818 args.addConst("threadRegister", registers.getThreadRegister());
819 args.addConst("counters", counters);
820
821 SnippetTemplate template = template(newInstanceNode, args);
822 template.instantiate(providers.getMetaAccess(), newInstanceNode, DEFAULT_REPLACER, args);
823 }
824
825 public void lower(DynamicNewArrayNode newArrayNode, HotSpotRegistersProvider registers, LoweringTool tool) {
826 StructuredGraph graph = newArrayNode.graph();
827 Arguments args = new Arguments(allocateArrayDynamic, graph.getGuardsStage(), tool.getLoweringStage());
828 args.add("elementType", newArrayNode.getElementType());
829 ValueNode voidClass = newArrayNode.getVoidClass();
830 assert voidClass != null;
831 args.add("voidClass", voidClass);
832 ValueNode length = newArrayNode.length();
833 args.add("length", length.isAlive() ? length : graph.addOrUniqueWithInputs(length));
834 args.addConst("fillContents", newArrayNode.fillContents());
835 args.addConst("threadRegister", registers.getThreadRegister());
836 /*
837 * We use Kind.Illegal as a marker value instead of null because constant snippet
838 * parameters cannot be null.
839 */
840 args.addConst("knownElementKind", newArrayNode.getKnownElementKind() == null ? JavaKind.Illegal : newArrayNode.getKnownElementKind());
841 if (newArrayNode.getKnownElementKind() != null) {
842 args.addConst("knownLayoutHelper", lookupArrayClass(tool, newArrayNode.getKnownElementKind()).layoutHelper());
843 } else {
844 args.addConst("knownLayoutHelper", 0);
845 }
846 args.addConst("useBulkZeroing", tool.getLowerer().supportBulkZeroing());
847 args.add("prototypeMarkWord", lookupArrayClass(tool, JavaKind.Object).prototypeMarkWord());
848 args.addConst("counters", counters);
849 SnippetTemplate template = template(newArrayNode, args);
850 template.instantiate(providers.getMetaAccess(), newArrayNode, DEFAULT_REPLACER, args);
851 }
852
853 private static HotSpotResolvedObjectType lookupArrayClass(LoweringTool tool, JavaKind kind) {
854 return (HotSpotResolvedObjectType) tool.getMetaAccess().lookupJavaType(kind == JavaKind.Object ? Object.class : kind.toJavaClass()).getArrayClass();
855 }
856
857 public void lower(NewMultiArrayNode newmultiarrayNode, LoweringTool tool) {
858 StructuredGraph graph = newmultiarrayNode.graph();
859 OptionValues localOptions = graph.getOptions();
860 int rank = newmultiarrayNode.dimensionCount();
861 ValueNode[] dims = new ValueNode[rank];
862 for (int i = 0; i < newmultiarrayNode.dimensionCount(); i++) {
863 dims[i] = newmultiarrayNode.dimension(i);
864 }
865 HotSpotResolvedObjectType type = (HotSpotResolvedObjectType) newmultiarrayNode.type();
866 ConstantNode hub = ConstantNode.forConstant(KlassPointerStamp.klassNonNull(), type.klass(), providers.getMetaAccess(), graph);
867
868 SnippetInfo snippet = GeneratePIC.getValue(localOptions) ? newmultiarrayPIC : newmultiarray;
869 Arguments args = new Arguments(snippet, graph.getGuardsStage(), tool.getLoweringStage());
870 args.add("hub", hub);
871 args.addConst("rank", rank);
872 args.addVarargs("dimensions", int.class, StampFactory.forKind(JavaKind.Int), dims);
873 template(newmultiarrayNode, args).instantiate(providers.getMetaAccess(), newmultiarrayNode, DEFAULT_REPLACER, args);
874 }
875
876 private static long instanceSize(HotSpotResolvedObjectType type) {
877 long size = type.instanceSize();
878 assert size >= 0;
879 return size;
880 }
881
882 public void lower(VerifyHeapNode verifyHeapNode, HotSpotRegistersProvider registers, LoweringTool tool) {
883 if (config.cAssertions) {
884 Arguments args = new Arguments(verifyHeap, verifyHeapNode.graph().getGuardsStage(), tool.getLoweringStage());
885 args.addConst("threadRegister", registers.getThreadRegister());
886
887 SnippetTemplate template = template(verifyHeapNode, args);
888 template.instantiate(providers.getMetaAccess(), verifyHeapNode, DEFAULT_REPLACER, args);
889 } else {
890 GraphUtil.removeFixedWithUnusedInputs(verifyHeapNode);
891 }
892 }
893 }
894 }