1 /*
2 * Copyright (c) 2012, 2019, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23
24
25 package org.graalvm.compiler.hotspot.replacements;
26
27 import static jdk.vm.ci.meta.DeoptimizationAction.None;
28 import static jdk.vm.ci.meta.DeoptimizationReason.RuntimeConstraint;
29 import static org.graalvm.compiler.core.common.GraalOptions.GeneratePIC;
30 import static org.graalvm.compiler.core.common.GraalOptions.MinimalBulkZeroingSize;
31 import static org.graalvm.compiler.core.common.calc.UnsignedMath.belowThan;
32 import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfigBase.INJECTED_OPTIONVALUES;
33 import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfigBase.INJECTED_VMCONFIG;
34 import static org.graalvm.compiler.hotspot.HotSpotBackend.NEW_ARRAY;
35 import static org.graalvm.compiler.hotspot.HotSpotBackend.NEW_ARRAY_OR_NULL;
36 import static org.graalvm.compiler.hotspot.HotSpotBackend.NEW_INSTANCE;
37 import static org.graalvm.compiler.hotspot.HotSpotBackend.NEW_INSTANCE_OR_NULL;
38 import static org.graalvm.compiler.hotspot.HotSpotBackend.NEW_MULTI_ARRAY;
39 import static org.graalvm.compiler.hotspot.HotSpotBackend.NEW_MULTI_ARRAY_OR_NULL;
40 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.CLASS_ARRAY_KLASS_LOCATION;
41 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.CLASS_STATE_LOCATION;
42 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.HUB_WRITE_LOCATION;
43 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.MARK_WORD_LOCATION;
44 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.PROTOTYPE_MARK_WORD_LOCATION;
45 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_END_LOCATION;
46 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_TOP_LOCATION;
47 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.allocateInstancePrefetchLines;
48 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.allocatePrefetchDistance;
49 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.allocatePrefetchLines;
50 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.allocatePrefetchStepSize;
51 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.allocatePrefetchStyle;
52 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayAllocationSize;
53 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayKlassOffset;
54 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayLengthOffset;
55 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.initializeObjectHeader;
56 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.instanceHeaderSize;
57 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.isInstanceKlassFullyInitialized;
58 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperHeaderSizeMask;
59 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperHeaderSizeShift;
60 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperLog2ElementSizeMask;
61 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperLog2ElementSizeShift;
62 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.loadKlassFromObject;
63 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.prototypeMarkWordOffset;
64 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readLayoutHelper;
65 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readTlabEnd;
66 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readTlabTop;
67 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.registerAsWord;
68 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.useBiasedLocking;
69 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.useTLAB;
70 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.verifyOop;
71 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.writeTlabTop;
72 import static org.graalvm.compiler.hotspot.replacements.HotspotSnippetsOptions.ProfileAllocations;
73 import static org.graalvm.compiler.hotspot.replacements.HotspotSnippetsOptions.ProfileAllocationsContext;
74 import static org.graalvm.compiler.nodes.PiArrayNode.piArrayCastToSnippetReplaceeStamp;
75 import static org.graalvm.compiler.nodes.PiNode.piCastToSnippetReplaceeStamp;
76 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.FAST_PATH_PROBABILITY;
77 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.FREQUENT_PROBABILITY;
78 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.SLOW_PATH_PROBABILITY;
79 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.probability;
80 import static org.graalvm.compiler.replacements.ReplacementsUtil.REPLACEMENTS_ASSERTIONS_ENABLED;
81 import static org.graalvm.compiler.replacements.ReplacementsUtil.runtimeAssert;
82 import static org.graalvm.compiler.replacements.ReplacementsUtil.staticAssert;
83 import static org.graalvm.compiler.replacements.SnippetTemplate.DEFAULT_REPLACER;
84 import static org.graalvm.compiler.replacements.nodes.CStringConstant.cstring;
85 import static org.graalvm.compiler.replacements.nodes.ExplodeLoopNode.explodeLoop;
86
87 import org.graalvm.compiler.api.replacements.Fold;
88 import org.graalvm.compiler.api.replacements.Fold.InjectedParameter;
89 import org.graalvm.compiler.api.replacements.Snippet;
90 import org.graalvm.compiler.api.replacements.Snippet.ConstantParameter;
91 import org.graalvm.compiler.api.replacements.Snippet.VarargsParameter;
92 import org.graalvm.compiler.core.common.spi.ForeignCallDescriptor;
93 import org.graalvm.compiler.core.common.type.StampFactory;
94 import org.graalvm.compiler.debug.DebugHandlersFactory;
95 import org.graalvm.compiler.debug.GraalError;
96 import org.graalvm.compiler.graph.Node.ConstantNodeParameter;
97 import org.graalvm.compiler.graph.Node.NodeIntrinsic;
98 import org.graalvm.compiler.hotspot.GraalHotSpotVMConfig;
99 import org.graalvm.compiler.hotspot.meta.HotSpotProviders;
100 import org.graalvm.compiler.hotspot.meta.HotSpotRegistersProvider;
101 import org.graalvm.compiler.hotspot.nodes.DimensionsNode;
102 import org.graalvm.compiler.hotspot.nodes.aot.LoadConstantIndirectlyFixedNode;
103 import org.graalvm.compiler.hotspot.nodes.aot.LoadConstantIndirectlyNode;
104 import org.graalvm.compiler.hotspot.nodes.type.KlassPointerStamp;
105 import org.graalvm.compiler.hotspot.word.KlassPointer;
106 import org.graalvm.compiler.nodes.ConstantNode;
107 import org.graalvm.compiler.nodes.DeoptimizeNode;
108 import org.graalvm.compiler.nodes.PiNode;
109 import org.graalvm.compiler.nodes.PrefetchAllocateNode;
110 import org.graalvm.compiler.nodes.SnippetAnchorNode;
111 import org.graalvm.compiler.nodes.StructuredGraph;
112 import org.graalvm.compiler.nodes.ValueNode;
113 import org.graalvm.compiler.nodes.debug.DynamicCounterNode;
114 import org.graalvm.compiler.nodes.debug.VerifyHeapNode;
115 import org.graalvm.compiler.nodes.extended.ForeignCallNode;
116 import org.graalvm.compiler.nodes.extended.MembarNode;
117 import org.graalvm.compiler.nodes.java.DynamicNewArrayNode;
118 import org.graalvm.compiler.nodes.java.DynamicNewInstanceNode;
119 import org.graalvm.compiler.nodes.java.NewArrayNode;
120 import org.graalvm.compiler.nodes.java.NewInstanceNode;
121 import org.graalvm.compiler.nodes.java.NewMultiArrayNode;
122 import org.graalvm.compiler.nodes.memory.address.OffsetAddressNode;
123 import org.graalvm.compiler.nodes.spi.LoweringTool;
124 import org.graalvm.compiler.nodes.util.GraphUtil;
125 import org.graalvm.compiler.options.OptionValues;
126 import org.graalvm.compiler.replacements.ReplacementsUtil;
127 import org.graalvm.compiler.replacements.SnippetCounter;
128 import org.graalvm.compiler.replacements.SnippetCounter.Group;
129 import org.graalvm.compiler.replacements.SnippetTemplate;
130 import org.graalvm.compiler.replacements.SnippetTemplate.AbstractTemplates;
131 import org.graalvm.compiler.replacements.SnippetTemplate.Arguments;
132 import org.graalvm.compiler.replacements.SnippetTemplate.SnippetInfo;
133 import org.graalvm.compiler.replacements.Snippets;
134 import org.graalvm.compiler.replacements.nodes.ExplodeLoopNode;
135 import org.graalvm.compiler.replacements.nodes.ZeroMemoryNode;
136 import org.graalvm.compiler.word.Word;
137 import jdk.internal.vm.compiler.word.LocationIdentity;
138 import jdk.internal.vm.compiler.word.WordFactory;
139
140 import jdk.vm.ci.code.CodeUtil;
141 import jdk.vm.ci.code.MemoryBarriers;
142 import jdk.vm.ci.code.Register;
143 import jdk.vm.ci.code.TargetDescription;
144 import jdk.vm.ci.hotspot.HotSpotResolvedObjectType;
145 import jdk.vm.ci.meta.JavaKind;
146 import jdk.vm.ci.meta.ResolvedJavaType;
147
148 /**
149 * Snippets used for implementing NEW, ANEWARRAY and NEWARRAY.
150 */
151 public class NewObjectSnippets implements Snippets {
152
153 enum ProfileContext {
154 AllocatingMethod,
155 InstanceOrArray,
156 AllocatedType,
157 AllocatedTypesInMethod,
158 Total
159 }
160
161 @Fold
162 static String createName(@Fold.InjectedParameter OptionValues options, String path, String typeContext) {
163 switch (ProfileAllocationsContext.getValue(options)) {
164 case AllocatingMethod:
165 return "";
166 case InstanceOrArray:
167 return path;
168 case AllocatedType:
169 case AllocatedTypesInMethod:
170 return typeContext;
171 case Total:
172 return "bytes";
173 default:
174 throw GraalError.shouldNotReachHere();
175 }
176 }
177
178 @Fold
179 static boolean doProfile(@Fold.InjectedParameter OptionValues options) {
180 return ProfileAllocations.getValue(options);
181 }
182
183 @Fold
184 static boolean withContext(@Fold.InjectedParameter OptionValues options) {
185 ProfileContext context = ProfileAllocationsContext.getValue(options);
186 return context == ProfileContext.AllocatingMethod || context == ProfileContext.AllocatedTypesInMethod;
187 }
188
189 protected static void profileAllocation(String path, long size, String typeContext) {
190 if (doProfile(INJECTED_OPTIONVALUES)) {
191 String name = createName(INJECTED_OPTIONVALUES, path, typeContext);
192
193 boolean context = withContext(INJECTED_OPTIONVALUES);
194 DynamicCounterNode.counter("number of bytes allocated", name, size, context);
195 DynamicCounterNode.counter("number of allocations", name, 1, context);
196 }
197 }
198
199 public static void emitPrefetchAllocate(Word address, boolean isArray) {
200 if (allocatePrefetchStyle(INJECTED_VMCONFIG) > 0) {
201 // Insert a prefetch for each allocation only on the fast-path
202 // Generate several prefetch instructions.
203 int lines = isArray ? allocatePrefetchLines(INJECTED_VMCONFIG) : allocateInstancePrefetchLines(INJECTED_VMCONFIG);
204 int stepSize = allocatePrefetchStepSize(INJECTED_VMCONFIG);
205 int distance = allocatePrefetchDistance(INJECTED_VMCONFIG);
206 ExplodeLoopNode.explodeLoop();
207 for (int i = 0; i < lines; i++) {
208 PrefetchAllocateNode.prefetch(OffsetAddressNode.address(address, distance));
209 distance += stepSize;
210 }
211 }
212 }
213
214 @Snippet
215 public static Object allocateInstance(@ConstantParameter long size,
216 KlassPointer hub,
217 Word prototypeMarkWord,
218 @ConstantParameter boolean fillContents,
219 @ConstantParameter boolean emitMemoryBarrier,
220 @ConstantParameter Register threadRegister,
221 @ConstantParameter boolean constantSize,
222 @ConstantParameter String typeContext,
223 @ConstantParameter Counters counters) {
224 return piCastToSnippetReplaceeStamp(allocateInstanceHelper(size, hub, prototypeMarkWord, fillContents, emitMemoryBarrier, threadRegister, constantSize, typeContext, counters));
225 }
226
227 public static Object allocateInstanceHelper(long size,
228 KlassPointer hub,
229 Word prototypeMarkWord,
230 boolean fillContents,
231 boolean emitMemoryBarrier,
232 Register threadRegister,
233 boolean constantSize,
234 String typeContext,
235 Counters counters) {
236 Object result;
237 Word thread = registerAsWord(threadRegister);
238 Word top = readTlabTop(thread);
239 Word end = readTlabEnd(thread);
240 Word newTop = top.add(WordFactory.unsigned(size));
241 if (useTLAB(INJECTED_VMCONFIG) && probability(FAST_PATH_PROBABILITY, newTop.belowOrEqual(end))) {
242 writeTlabTop(thread, newTop);
243 emitPrefetchAllocate(newTop, false);
244 result = formatObject(hub, size, top, prototypeMarkWord, fillContents, emitMemoryBarrier, constantSize, counters);
245 } else {
246 Counters theCounters = counters;
247 if (theCounters != null && theCounters.stub != null) {
248 theCounters.stub.inc();
249 }
250 result = newInstanceStub(hub);
251 }
252 profileAllocation("instance", size, typeContext);
253 return verifyOop(result);
254 }
255
256 public static Object newInstanceStub(KlassPointer hub) {
257 if (useNullAllocationStubs(INJECTED_VMCONFIG)) {
258 return nonNullOrDeopt(newInstanceOrNull(NEW_INSTANCE_OR_NULL, hub));
259 } else {
260 return newInstance(NEW_INSTANCE, hub);
261 }
262 }
263
264 @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = true)
265 private static native Object newInstance(@ConstantNodeParameter ForeignCallDescriptor descriptor, KlassPointer hub);
266
267 @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = false)
268 private static native Object newInstanceOrNull(@ConstantNodeParameter ForeignCallDescriptor descriptor, KlassPointer hub);
269
270 @Snippet
271 public static Object allocateInstancePIC(@ConstantParameter long size,
272 KlassPointer hub,
273 Word prototypeMarkWord,
274 @ConstantParameter boolean fillContents,
275 @ConstantParameter boolean emitMemoryBarrier,
276 @ConstantParameter Register threadRegister,
277 @ConstantParameter boolean constantSize,
278 @ConstantParameter String typeContext,
279 @ConstantParameter Counters counters) {
280 // Klass must be initialized by the time the first instance is allocated, therefore we can
281 // just load it from the corresponding cell and avoid the resolution check. We have to use a
282 // fixed load though, to prevent it from floating above the initialization.
283 KlassPointer picHub = LoadConstantIndirectlyFixedNode.loadKlass(hub);
284 return piCastToSnippetReplaceeStamp(allocateInstanceHelper(size, picHub, prototypeMarkWord, fillContents, emitMemoryBarrier, threadRegister, constantSize, typeContext, counters));
285 }
286
287 @Snippet
288 public static Object allocateInstanceDynamic(Class<?> type, Class<?> classClass,
289 @ConstantParameter boolean fillContents,
290 @ConstantParameter boolean emitMemoryBarrier,
291 @ConstantParameter Register threadRegister,
292 @ConstantParameter Counters counters) {
293 if (probability(SLOW_PATH_PROBABILITY, type == null)) {
294 DeoptimizeNode.deopt(None, RuntimeConstraint);
295 }
296 Class<?> nonNullType = PiNode.piCastNonNullClass(type, SnippetAnchorNode.anchor());
297
298 if (probability(SLOW_PATH_PROBABILITY, DynamicNewInstanceNode.throwsInstantiationException(type, classClass))) {
299 DeoptimizeNode.deopt(None, RuntimeConstraint);
300 }
301
302 return PiNode.piCastToSnippetReplaceeStamp(allocateInstanceDynamicHelper(type, fillContents, emitMemoryBarrier, threadRegister, counters, nonNullType));
303 }
304
305 private static Object allocateInstanceDynamicHelper(Class<?> type,
306 boolean fillContents,
307 boolean emitMemoryBarrier,
308 Register threadRegister,
309 Counters counters,
310 Class<?> nonNullType) {
311 KlassPointer hub = ClassGetHubNode.readClass(nonNullType);
312 if (probability(FAST_PATH_PROBABILITY, !hub.isNull())) {
313 KlassPointer nonNullHub = ClassGetHubNode.piCastNonNull(hub, SnippetAnchorNode.anchor());
314
315 if (probability(FAST_PATH_PROBABILITY, isInstanceKlassFullyInitialized(nonNullHub))) {
316 int layoutHelper = readLayoutHelper(nonNullHub);
317 /*
318 * src/share/vm/oops/klass.hpp: For instances, layout helper is a positive number,
319 * the instance size. This size is already passed through align_object_size and
320 * scaled to bytes. The low order bit is set if instances of this class cannot be
321 * allocated using the fastpath.
322 */
323 if (probability(FAST_PATH_PROBABILITY, (layoutHelper & 1) == 0)) {
324 Word prototypeMarkWord = nonNullHub.readWord(prototypeMarkWordOffset(INJECTED_VMCONFIG), PROTOTYPE_MARK_WORD_LOCATION);
325 /*
326 * FIXME(je,ds): we should actually pass typeContext instead of "" but late
327 * binding of parameters is not yet supported by the GraphBuilderPlugin system.
328 */
329 return allocateInstanceHelper(layoutHelper, nonNullHub, prototypeMarkWord, fillContents, emitMemoryBarrier, threadRegister, false, "", counters);
330 }
331 } else {
332 DeoptimizeNode.deopt(None, RuntimeConstraint);
333 }
334 }
335 return dynamicNewInstanceStub(type);
336 }
337
338 /**
339 * Maximum array length for which fast path allocation is used.
340 */
341 public static final int MAX_ARRAY_FAST_PATH_ALLOCATION_LENGTH = 0x00FFFFFF;
342
343 @Snippet
344 public static Object allocatePrimitiveArrayPIC(KlassPointer hub,
345 int length,
346 Word prototypeMarkWord,
347 @ConstantParameter int headerSize,
348 @ConstantParameter int log2ElementSize,
349 @ConstantParameter boolean fillContents,
350 @ConstantParameter boolean emitMemoryBarrier,
351 @ConstantParameter Register threadRegister,
352 @ConstantParameter boolean maybeUnroll,
353 @ConstantParameter String typeContext,
354 @ConstantParameter boolean useBulkZeroing,
355 @ConstantParameter Counters counters) {
356 // Primitive array types are eagerly pre-resolved. We can use a floating load.
357 KlassPointer picHub = LoadConstantIndirectlyNode.loadKlass(hub);
358 return allocateArrayImpl(picHub, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents,
359 emitMemoryBarrier, threadRegister, maybeUnroll, typeContext, useBulkZeroing, counters);
360 }
361
362 @Snippet
363 public static Object allocateArrayPIC(KlassPointer hub,
364 int length,
365 Word prototypeMarkWord,
366 @ConstantParameter int headerSize,
367 @ConstantParameter int log2ElementSize,
368 @ConstantParameter boolean fillContents,
369 @ConstantParameter boolean emitMemoryBarrier,
370 @ConstantParameter Register threadRegister,
371 @ConstantParameter boolean maybeUnroll,
372 @ConstantParameter String typeContext,
373 @ConstantParameter boolean useBulkZeroing,
374 @ConstantParameter Counters counters) {
375 // Array type would be resolved by dominating resolution.
376 KlassPointer picHub = LoadConstantIndirectlyFixedNode.loadKlass(hub);
377 return allocateArrayImpl(picHub, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents,
378 emitMemoryBarrier, threadRegister, maybeUnroll, typeContext, useBulkZeroing, counters);
379 }
380
381 @Snippet
382 public static Object allocateArray(KlassPointer hub,
383 int length,
384 Word prototypeMarkWord,
385 @ConstantParameter int headerSize,
386 @ConstantParameter int log2ElementSize,
387 @ConstantParameter boolean fillContents,
388 @ConstantParameter boolean emitMemoryBarrier,
389 @ConstantParameter Register threadRegister,
390 @ConstantParameter boolean maybeUnroll,
391 @ConstantParameter String typeContext,
392 @ConstantParameter boolean useBulkZeroing,
393 @ConstantParameter Counters counters) {
394 Object result = allocateArrayImpl(hub,
395 length,
396 prototypeMarkWord,
397 headerSize,
398 log2ElementSize,
399 fillContents,
400 emitMemoryBarrier, threadRegister,
401 maybeUnroll,
402 typeContext,
403 useBulkZeroing,
404 counters);
405 return piArrayCastToSnippetReplaceeStamp(verifyOop(result), length);
406 }
407
408 /**
409 * When allocating on the slow path, determines whether to use a version of the runtime call
410 * that returns {@code null} on a failed allocation instead of raising an OutOfMemoryError.
411 */
412 @Fold
413 static boolean useNullAllocationStubs(@InjectedParameter GraalHotSpotVMConfig config) {
414 return config.areNullAllocationStubsAvailable();
415 }
416
417 private static Object allocateArrayImpl(KlassPointer hub,
418 int length,
419 Word prototypeMarkWord,
420 int headerSize,
421 int log2ElementSize,
422 boolean fillContents,
423 boolean emitMemoryBarrier,
424 Register threadRegister,
425 boolean maybeUnroll,
426 String typeContext,
427 boolean useBulkZeroing,
428 Counters counters) {
429 Object result;
430 long allocationSize = arrayAllocationSize(length, headerSize, log2ElementSize);
431 Word thread = registerAsWord(threadRegister);
432 Word top = readTlabTop(thread);
433 Word end = readTlabEnd(thread);
434 Word newTop = top.add(WordFactory.unsigned(allocationSize));
435 if (probability(FREQUENT_PROBABILITY, belowThan(length, MAX_ARRAY_FAST_PATH_ALLOCATION_LENGTH)) && useTLAB(INJECTED_VMCONFIG) &&
436 probability(FAST_PATH_PROBABILITY, newTop.belowOrEqual(end))) {
437 writeTlabTop(thread, newTop);
438 emitPrefetchAllocate(newTop, true);
439 Counters theCounters = counters;
440 if (theCounters != null && theCounters.arrayLoopInit != null) {
441 theCounters.arrayLoopInit.inc();
442 }
443 result = formatArray(hub, allocationSize, length, headerSize, top, prototypeMarkWord, fillContents, emitMemoryBarrier, maybeUnroll, useBulkZeroing, counters);
444 } else {
445 result = newArrayStub(hub, length);
446 }
447 profileAllocation("array", allocationSize, typeContext);
448 return result;
449 }
450
451 public static Object newArrayStub(KlassPointer hub, int length) {
452 if (useNullAllocationStubs(INJECTED_VMCONFIG)) {
453 return nonNullOrDeopt(newArrayOrNull(NEW_ARRAY_OR_NULL, hub, length));
454 } else {
455 return newArray(NEW_ARRAY, hub, length);
456 }
457 }
458
459 @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = true)
460 private static native Object newArray(@ConstantNodeParameter ForeignCallDescriptor descriptor, KlassPointer hub, int length);
461
462 @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = false)
463 private static native Object newArrayOrNull(@ConstantNodeParameter ForeignCallDescriptor descriptor, KlassPointer hub, int length);
464
465 /**
466 * New dynamic array stub that throws an {@link OutOfMemoryError} on allocation failure.
467 */
468 public static final ForeignCallDescriptor DYNAMIC_NEW_INSTANCE = new ForeignCallDescriptor("dynamic_new_instance", Object.class, Class.class);
469
470 /**
471 * New dynamic array stub that returns null on allocation failure.
472 */
473 public static final ForeignCallDescriptor DYNAMIC_NEW_INSTANCE_OR_NULL = new ForeignCallDescriptor("dynamic_new_instance_or_null", Object.class, Class.class);
474
475 public static Object dynamicNewInstanceStub(Class<?> elementType) {
476 if (useNullAllocationStubs(INJECTED_VMCONFIG)) {
477 return nonNullOrDeopt(dynamicNewInstanceOrNull(DYNAMIC_NEW_INSTANCE_OR_NULL, elementType));
478 } else {
479 return dynamicNewInstance(DYNAMIC_NEW_INSTANCE, elementType);
480 }
481 }
482
483 /**
484 * Deoptimizes if {@code obj == null} otherwise returns {@code obj}.
485 */
486 private static Object nonNullOrDeopt(Object obj) {
487 if (obj == null) {
488 DeoptimizeNode.deopt(None, RuntimeConstraint);
489 }
490 return obj;
491 }
492
493 @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = true)
494 public static native Object dynamicNewInstance(@ConstantNodeParameter ForeignCallDescriptor descriptor, Class<?> elementType);
495
496 @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = false)
497 public static native Object dynamicNewInstanceOrNull(@ConstantNodeParameter ForeignCallDescriptor descriptor, Class<?> elementType);
498
499 @Snippet
500 public static Object allocateArrayDynamic(Class<?> elementType,
501 Class<?> voidClass,
502 int length,
503 @ConstantParameter boolean fillContents,
504 @ConstantParameter boolean emitMemoryBarrier,
505 @ConstantParameter Register threadRegister,
506 @ConstantParameter JavaKind knownElementKind,
507 @ConstantParameter int knownLayoutHelper,
508 @ConstantParameter boolean useBulkZeroing,
509 Word prototypeMarkWord,
510 @ConstantParameter Counters counters) {
511 Object result = allocateArrayDynamicImpl(elementType, voidClass, length, fillContents, emitMemoryBarrier, threadRegister, knownElementKind,
512 knownLayoutHelper, useBulkZeroing, prototypeMarkWord, counters);
513 return result;
514 }
515
516 private static Object allocateArrayDynamicImpl(Class<?> elementType,
517 Class<?> voidClass,
518 int length,
519 boolean fillContents,
520 boolean emitMemoryBarrier,
521 Register threadRegister,
522 JavaKind knownElementKind,
523 int knownLayoutHelper,
524 boolean useBulkZeroing,
525 Word prototypeMarkWord,
526 Counters counters) {
527 /*
528 * We only need the dynamic check for void when we have no static information from
529 * knownElementKind.
530 */
531 staticAssert(knownElementKind != JavaKind.Void, "unsupported knownElementKind");
532 if (knownElementKind == JavaKind.Illegal && probability(SLOW_PATH_PROBABILITY, elementType == null || DynamicNewArrayNode.throwsIllegalArgumentException(elementType, voidClass))) {
533 DeoptimizeNode.deopt(None, RuntimeConstraint);
534 }
535
536 KlassPointer klass = loadKlassFromObject(elementType, arrayKlassOffset(INJECTED_VMCONFIG), CLASS_ARRAY_KLASS_LOCATION);
537 if (klass.isNull()) {
538 DeoptimizeNode.deopt(None, RuntimeConstraint);
539 }
540 KlassPointer nonNullKlass = ClassGetHubNode.piCastNonNull(klass, SnippetAnchorNode.anchor());
541
542 if (length < 0) {
543 DeoptimizeNode.deopt(None, RuntimeConstraint);
544 }
545 int layoutHelper;
546 if (knownElementKind == JavaKind.Illegal) {
547 layoutHelper = readLayoutHelper(nonNullKlass);
548 } else {
549 runtimeAssert(knownLayoutHelper == readLayoutHelper(nonNullKlass), "layout mismatch");
550 layoutHelper = knownLayoutHelper;
551 }
552 //@formatter:off
553 // from src/share/vm/oops/klass.hpp:
554 //
555 // For arrays, layout helper is a negative number, containing four
556 // distinct bytes, as follows:
557 // MSB:[tag, hsz, ebt, log2(esz)]:LSB
558 // where:
559 // tag is 0x80 if the elements are oops, 0xC0 if non-oops
560 // hsz is array header size in bytes (i.e., offset of first element)
561 // ebt is the BasicType of the elements
562 // esz is the element size in bytes
563 //@formatter:on
564
565 int headerSize = (layoutHelper >> layoutHelperHeaderSizeShift(INJECTED_VMCONFIG)) & layoutHelperHeaderSizeMask(INJECTED_VMCONFIG);
566 int log2ElementSize = (layoutHelper >> layoutHelperLog2ElementSizeShift(INJECTED_VMCONFIG)) & layoutHelperLog2ElementSizeMask(INJECTED_VMCONFIG);
567
568 Object result = allocateArrayImpl(nonNullKlass, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents,
569 emitMemoryBarrier, threadRegister, false, "dynamic type", useBulkZeroing, counters);
570 return piArrayCastToSnippetReplaceeStamp(verifyOop(result), length);
571 }
572
573 /**
574 * Calls the runtime stub for implementing MULTIANEWARRAY.
575 */
576 @Snippet
577 private static Object newmultiarray(KlassPointer hub, @ConstantParameter int rank, @VarargsParameter int[] dimensions) {
578 Word dims = DimensionsNode.allocaDimsArray(rank);
579 ExplodeLoopNode.explodeLoop();
580 for (int i = 0; i < rank; i++) {
581 dims.writeInt(i * 4, dimensions[i], LocationIdentity.init());
582 }
583 return newMultiArrayStub(hub, rank, dims);
584 }
585
586 private static Object newMultiArrayStub(KlassPointer hub, int rank, Word dims) {
587 if (useNullAllocationStubs(INJECTED_VMCONFIG)) {
588 return nonNullOrDeopt(newMultiArrayOrNull(NEW_MULTI_ARRAY_OR_NULL, hub, rank, dims));
589 } else {
590 return newMultiArray(NEW_MULTI_ARRAY, hub, rank, dims);
591 }
592 }
593
594 @Snippet
595 private static Object newmultiarrayPIC(KlassPointer hub, @ConstantParameter int rank, @VarargsParameter int[] dimensions) {
596 // Array type would be resolved by dominating resolution.
597 KlassPointer picHub = LoadConstantIndirectlyFixedNode.loadKlass(hub);
598 return newmultiarray(picHub, rank, dimensions);
599 }
600
601 @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = true)
602 private static native Object newMultiArray(@ConstantNodeParameter ForeignCallDescriptor descriptor, KlassPointer hub, int rank, Word dims);
603
604 @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = false)
605 private static native Object newMultiArrayOrNull(@ConstantNodeParameter ForeignCallDescriptor descriptor, KlassPointer hub, int rank, Word dims);
606
607 /**
608 * Maximum number of long stores to emit when zeroing an object with a constant size. Larger
609 * objects have their bodies initialized in a loop.
610 */
611 private static final int MAX_UNROLLED_OBJECT_ZEROING_STORES = 8;
612
613 /**
614 * Zero uninitialized memory in a newly allocated object, unrolling as necessary and ensuring
615 * that stores are aligned.
616 *
617 * @param size number of bytes to zero
618 * @param memory beginning of object which is being zeroed
619 * @param constantSize is {@code size} known to be constant in the snippet
620 * @param startOffset offset to begin zeroing. May not be word aligned.
621 * @param manualUnroll maximally unroll zeroing
622 * @param useBulkZeroing apply bulk zeroing
623 */
624 private static void zeroMemory(long size, Word memory, boolean constantSize, int startOffset, boolean manualUnroll,
625 boolean useBulkZeroing, Counters counters) {
626 fillMemory(0, size, memory, constantSize, startOffset, manualUnroll, useBulkZeroing, counters);
627 }
628
629 private static void fillMemory(long value, long size, Word memory, boolean constantSize, int startOffset, boolean manualUnroll,
630 boolean useBulkZeroing, Counters counters) {
631 ReplacementsUtil.runtimeAssert((size & 0x7) == 0, "unaligned object size");
632 int offset = startOffset;
633 if ((offset & 0x7) != 0) {
634 memory.writeInt(offset, (int) value, LocationIdentity.init());
635 offset += 4;
636 }
637 ReplacementsUtil.runtimeAssert((offset & 0x7) == 0, "unaligned offset");
638 Counters theCounters = counters;
639 if (manualUnroll && ((size - offset) / 8) <= MAX_UNROLLED_OBJECT_ZEROING_STORES) {
640 ReplacementsUtil.staticAssert(!constantSize, "size shouldn't be constant at instantiation time");
641 // This case handles arrays of constant length. Instead of having a snippet variant for
642 // each length, generate a chain of stores of maximum length. Once it's inlined the
643 // break statement will trim excess stores.
644 if (theCounters != null && theCounters.instanceSeqInit != null) {
645 theCounters.instanceSeqInit.inc();
646 }
647
648 explodeLoop();
649 for (int i = 0; i < MAX_UNROLLED_OBJECT_ZEROING_STORES; i++, offset += 8) {
650 if (offset == size) {
651 break;
652 }
653 memory.initializeLong(offset, value, LocationIdentity.init());
654 }
655 } else {
656 // Use Word instead of int to avoid extension to long in generated code
657 Word off = WordFactory.signed(offset);
658 if (useBulkZeroing && value == 0 && probability(SLOW_PATH_PROBABILITY, (size - offset) >= getMinimalBulkZeroingSize(INJECTED_OPTIONVALUES))) {
659 if (theCounters != null && theCounters.instanceBulkInit != null) {
660 theCounters.instanceBulkInit.inc();
661 }
662 ZeroMemoryNode.zero(memory.add(off), size - offset, LocationIdentity.init());
663 } else {
664 if (constantSize && ((size - offset) / 8) <= MAX_UNROLLED_OBJECT_ZEROING_STORES) {
665 if (theCounters != null && theCounters.instanceSeqInit != null) {
666 theCounters.instanceSeqInit.inc();
667 }
668 explodeLoop();
669 } else {
670 if (theCounters != null && theCounters.instanceLoopInit != null) {
671 theCounters.instanceLoopInit.inc();
672 }
673 }
674 for (; off.rawValue() < size; off = off.add(8)) {
675 memory.initializeLong(off, value, LocationIdentity.init());
676 }
677 }
678 }
679 }
680
681 @Fold
682 static int getMinimalBulkZeroingSize(@InjectedParameter OptionValues optionValues) {
683 return MinimalBulkZeroingSize.getValue(optionValues);
684 }
685
686 /**
687 * Fill uninitialized memory with garbage value in a newly allocated object, unrolling as
688 * necessary and ensuring that stores are aligned.
689 *
690 * @param size number of bytes to zero
691 * @param memory beginning of object which is being zeroed
692 * @param constantSize is {@code size} known to be constant in the snippet
693 * @param startOffset offset to begin zeroing. May not be word aligned.
694 * @param manualUnroll maximally unroll zeroing
695 */
696 private static void fillWithGarbage(long size, Word memory, boolean constantSize, int startOffset, boolean manualUnroll, Counters counters) {
697 fillMemory(0xfefefefefefefefeL, size, memory, constantSize, startOffset, manualUnroll, false, counters);
698 }
699
700 /**
701 * Formats some allocated memory with an object header and zeroes out the rest.
702 */
703 private static Object formatObject(KlassPointer hub,
704 long size,
705 Word memory,
706 Word compileTimePrototypeMarkWord,
707 boolean fillContents,
708 boolean emitMemoryBarrier,
709 boolean constantSize,
710 Counters counters) {
711 Word prototypeMarkWord = useBiasedLocking(INJECTED_VMCONFIG) ? hub.readWord(prototypeMarkWordOffset(INJECTED_VMCONFIG), PROTOTYPE_MARK_WORD_LOCATION) : compileTimePrototypeMarkWord;
712 initializeObjectHeader(memory, prototypeMarkWord, hub);
713 if (fillContents) {
714 zeroMemory(size, memory, constantSize, instanceHeaderSize(INJECTED_VMCONFIG), false, false, counters);
715 } else if (REPLACEMENTS_ASSERTIONS_ENABLED) {
716 fillWithGarbage(size, memory, constantSize, instanceHeaderSize(INJECTED_VMCONFIG), false, counters);
717 }
718 if (emitMemoryBarrier) {
719 MembarNode.memoryBarrier(MemoryBarriers.STORE_STORE, LocationIdentity.init());
720 }
721 return memory.toObjectNonNull();
722 }
723
724 @Snippet
725 private static void verifyHeap(@ConstantParameter Register threadRegister) {
726 Word thread = registerAsWord(threadRegister);
727 Word topValue = readTlabTop(thread);
728 if (!topValue.equal(WordFactory.zero())) {
729 Word topValueContents = topValue.readWord(0, MARK_WORD_LOCATION);
730 if (topValueContents.equal(WordFactory.zero())) {
731 AssertionSnippets.vmMessageC(AssertionSnippets.ASSERTION_VM_MESSAGE_C, true, cstring("overzeroing of TLAB detected"), 0L, 0L, 0L);
732 }
733 }
734 }
735
736 /**
737 * Formats some allocated memory with an object header and zeroes out the rest.
738 */
739 private static Object formatArray(KlassPointer hub,
740 long allocationSize,
741 int length,
742 int headerSize,
743 Word memory,
744 Word prototypeMarkWord,
745 boolean fillContents,
746 boolean emitMemoryBarrier,
747 boolean maybeUnroll,
748 boolean useBulkZeroing,
749 Counters counters) {
750 memory.writeInt(arrayLengthOffset(INJECTED_VMCONFIG), length, LocationIdentity.init());
751 /*
752 * store hub last as the concurrent garbage collectors assume length is valid if hub field
753 * is not null
754 */
755 initializeObjectHeader(memory, prototypeMarkWord, hub);
756 if (fillContents) {
757 zeroMemory(allocationSize, memory, false, headerSize, maybeUnroll, useBulkZeroing, counters);
758 } else if (REPLACEMENTS_ASSERTIONS_ENABLED) {
759 fillWithGarbage(allocationSize, memory, false, headerSize, maybeUnroll, counters);
760 }
761 if (emitMemoryBarrier) {
762 MembarNode.memoryBarrier(MemoryBarriers.STORE_STORE, LocationIdentity.init());
763 }
764 return memory.toObjectNonNull();
765 }
766
767 static class Counters {
768 Counters(SnippetCounter.Group.Factory factory) {
769 Group newInstance = factory.createSnippetCounterGroup("NewInstance");
770 Group newArray = factory.createSnippetCounterGroup("NewArray");
771 instanceSeqInit = new SnippetCounter(newInstance, "tlabSeqInit", "TLAB alloc with unrolled zeroing");
772 instanceLoopInit = new SnippetCounter(newInstance, "tlabLoopInit", "TLAB alloc with zeroing in a loop");
773 instanceBulkInit = new SnippetCounter(newArray, "tlabBulkInit", "TLAB alloc with bulk zeroing");
774 arrayLoopInit = new SnippetCounter(newArray, "tlabLoopInit", "TLAB alloc with zeroing in a loop");
775 stub = new SnippetCounter(newInstance, "stub", "alloc and zeroing via stub");
776 }
777
778 final SnippetCounter instanceSeqInit;
779 final SnippetCounter instanceLoopInit;
780 final SnippetCounter instanceBulkInit;
781 final SnippetCounter arrayLoopInit;
782 final SnippetCounter stub;
783 }
784
785 public static class Templates extends AbstractTemplates {
786
787 private final SnippetInfo allocateInstance = snippet(NewObjectSnippets.class, "allocateInstance", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION, TLAB_END_LOCATION,
788 PROTOTYPE_MARK_WORD_LOCATION);
789 private final SnippetInfo allocateInstancePIC = snippet(NewObjectSnippets.class, "allocateInstancePIC", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION,
790 TLAB_END_LOCATION, PROTOTYPE_MARK_WORD_LOCATION);
791 private final SnippetInfo allocateArray = snippet(NewObjectSnippets.class, "allocateArray", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION, TLAB_END_LOCATION);
792 private final SnippetInfo allocateArrayPIC = snippet(NewObjectSnippets.class, "allocateArrayPIC", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION, TLAB_END_LOCATION);
793 private final SnippetInfo allocatePrimitiveArrayPIC = snippet(NewObjectSnippets.class, "allocatePrimitiveArrayPIC", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION,
794 TLAB_END_LOCATION);
795 private final SnippetInfo allocateArrayDynamic = snippet(NewObjectSnippets.class, "allocateArrayDynamic", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION,
796 TLAB_END_LOCATION);
797 private final SnippetInfo allocateInstanceDynamic = snippet(NewObjectSnippets.class, "allocateInstanceDynamic", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION,
798 TLAB_END_LOCATION, PROTOTYPE_MARK_WORD_LOCATION, CLASS_STATE_LOCATION);
799 private final SnippetInfo newmultiarray = snippet(NewObjectSnippets.class, "newmultiarray", TLAB_TOP_LOCATION, TLAB_END_LOCATION);
800 private final SnippetInfo newmultiarrayPIC = snippet(NewObjectSnippets.class, "newmultiarrayPIC", TLAB_TOP_LOCATION, TLAB_END_LOCATION);
801 private final SnippetInfo verifyHeap = snippet(NewObjectSnippets.class, "verifyHeap");
802 private final GraalHotSpotVMConfig config;
803 private final Counters counters;
804
805 public Templates(OptionValues options, Iterable<DebugHandlersFactory> factories, SnippetCounter.Group.Factory factory, HotSpotProviders providers, TargetDescription target,
806 GraalHotSpotVMConfig config) {
807 super(options, factories, providers, providers.getSnippetReflection(), target);
808 this.config = config;
809 counters = new Counters(factory);
810 }
811
812 /**
813 * Lowers a {@link NewInstanceNode}.
814 */
815 public void lower(NewInstanceNode newInstanceNode, HotSpotRegistersProvider registers, LoweringTool tool) {
816 StructuredGraph graph = newInstanceNode.graph();
817 HotSpotResolvedObjectType type = (HotSpotResolvedObjectType) newInstanceNode.instanceClass();
818 assert !type.isArray();
819 ConstantNode hub = ConstantNode.forConstant(KlassPointerStamp.klassNonNull(), type.klass(), providers.getMetaAccess(), graph);
820 long size = instanceSize(type);
821
822 OptionValues localOptions = graph.getOptions();
823 SnippetInfo snippet = GeneratePIC.getValue(localOptions) ? allocateInstancePIC : allocateInstance;
824 Arguments args = new Arguments(snippet, graph.getGuardsStage(), tool.getLoweringStage());
825 args.addConst("size", size);
826 args.add("hub", hub);
827 args.add("prototypeMarkWord", type.prototypeMarkWord());
828 args.addConst("fillContents", newInstanceNode.fillContents());
829 args.addConst("emitMemoryBarrier", newInstanceNode.emitMemoryBarrier());
830 args.addConst("threadRegister", registers.getThreadRegister());
831 args.addConst("constantSize", true);
832 args.addConst("typeContext", ProfileAllocations.getValue(localOptions) ? type.toJavaName(false) : "");
833 args.addConst("counters", counters);
834
835 SnippetTemplate template = template(newInstanceNode, args);
836 graph.getDebug().log("Lowering allocateInstance in %s: node=%s, template=%s, arguments=%s", graph, newInstanceNode, template, args);
837 template.instantiate(providers.getMetaAccess(), newInstanceNode, DEFAULT_REPLACER, args);
838 }
839
840 /**
841 * Lowers a {@link NewArrayNode}.
842 */
843 public void lower(NewArrayNode newArrayNode, HotSpotRegistersProvider registers, LoweringTool tool) {
844 StructuredGraph graph = newArrayNode.graph();
845 ResolvedJavaType elementType = newArrayNode.elementType();
846 HotSpotResolvedObjectType arrayType = (HotSpotResolvedObjectType) elementType.getArrayClass();
847 JavaKind elementKind = elementType.getJavaKind();
848 ConstantNode hub = ConstantNode.forConstant(KlassPointerStamp.klassNonNull(), arrayType.klass(), providers.getMetaAccess(), graph);
849 final int headerSize = tool.getMetaAccess().getArrayBaseOffset(elementKind);
850 int log2ElementSize = CodeUtil.log2(tool.getMetaAccess().getArrayIndexScale(elementKind));
851
852 OptionValues localOptions = graph.getOptions();
853 SnippetInfo snippet;
854 if (GeneratePIC.getValue(localOptions)) {
855 if (elementType.isPrimitive()) {
856 snippet = allocatePrimitiveArrayPIC;
857 } else {
858 snippet = allocateArrayPIC;
859 }
860 } else {
861 snippet = allocateArray;
862 }
863
864 Arguments args = new Arguments(snippet, graph.getGuardsStage(), tool.getLoweringStage());
865 args.add("hub", hub);
866 ValueNode length = newArrayNode.length();
867 args.add("length", length.isAlive() ? length : graph.addOrUniqueWithInputs(length));
868 assert arrayType.prototypeMarkWord() == lookupArrayClass(tool, JavaKind.Object).prototypeMarkWord() : "all array types are assumed to have the same prototypeMarkWord";
869 args.add("prototypeMarkWord", arrayType.prototypeMarkWord());
870 args.addConst("headerSize", headerSize);
871 args.addConst("log2ElementSize", log2ElementSize);
872 args.addConst("fillContents", newArrayNode.fillContents());
873 args.addConst("emitMemoryBarrier", newArrayNode.emitMemoryBarrier());
874 args.addConst("threadRegister", registers.getThreadRegister());
875 args.addConst("maybeUnroll", length.isConstant());
876 args.addConst("typeContext", ProfileAllocations.getValue(localOptions) ? arrayType.toJavaName(false) : "");
877 args.addConst("useBulkZeroing", tool.getLowerer().supportBulkZeroing());
878 args.addConst("counters", counters);
879 SnippetTemplate template = template(newArrayNode, args);
880 graph.getDebug().log("Lowering allocateArray in %s: node=%s, template=%s, arguments=%s", graph, newArrayNode, template, args);
881 template.instantiate(providers.getMetaAccess(), newArrayNode, DEFAULT_REPLACER, args);
882 }
883
884 public void lower(DynamicNewInstanceNode newInstanceNode, HotSpotRegistersProvider registers, LoweringTool tool) {
885 Arguments args = new Arguments(allocateInstanceDynamic, newInstanceNode.graph().getGuardsStage(), tool.getLoweringStage());
886 args.add("type", newInstanceNode.getInstanceType());
887 ValueNode classClass = newInstanceNode.getClassClass();
888 assert classClass != null;
889 args.add("classClass", classClass);
890 args.addConst("fillContents", newInstanceNode.fillContents());
891 args.addConst("emitMemoryBarrier", newInstanceNode.emitMemoryBarrier());
892 args.addConst("threadRegister", registers.getThreadRegister());
893 args.addConst("counters", counters);
894
895 SnippetTemplate template = template(newInstanceNode, args);
896 template.instantiate(providers.getMetaAccess(), newInstanceNode, DEFAULT_REPLACER, args);
897 }
898
899 public void lower(DynamicNewArrayNode newArrayNode, HotSpotRegistersProvider registers, LoweringTool tool) {
900 StructuredGraph graph = newArrayNode.graph();
901 Arguments args = new Arguments(allocateArrayDynamic, graph.getGuardsStage(), tool.getLoweringStage());
902 args.add("elementType", newArrayNode.getElementType());
903 ValueNode voidClass = newArrayNode.getVoidClass();
904 assert voidClass != null;
905 args.add("voidClass", voidClass);
906 ValueNode length = newArrayNode.length();
907 args.add("length", length.isAlive() ? length : graph.addOrUniqueWithInputs(length));
908 args.addConst("fillContents", newArrayNode.fillContents());
909 args.addConst("emitMemoryBarrier", newArrayNode.emitMemoryBarrier());
910 args.addConst("threadRegister", registers.getThreadRegister());
911 /*
912 * We use Kind.Illegal as a marker value instead of null because constant snippet
913 * parameters cannot be null.
914 */
915 args.addConst("knownElementKind", newArrayNode.getKnownElementKind() == null ? JavaKind.Illegal : newArrayNode.getKnownElementKind());
916 if (newArrayNode.getKnownElementKind() != null) {
917 args.addConst("knownLayoutHelper", lookupArrayClass(tool, newArrayNode.getKnownElementKind()).layoutHelper());
918 } else {
919 args.addConst("knownLayoutHelper", 0);
920 }
921 args.addConst("useBulkZeroing", tool.getLowerer().supportBulkZeroing());
922 args.add("prototypeMarkWord", lookupArrayClass(tool, JavaKind.Object).prototypeMarkWord());
923 args.addConst("counters", counters);
924 SnippetTemplate template = template(newArrayNode, args);
925 template.instantiate(providers.getMetaAccess(), newArrayNode, DEFAULT_REPLACER, args);
926 }
927
928 private static HotSpotResolvedObjectType lookupArrayClass(LoweringTool tool, JavaKind kind) {
929 return (HotSpotResolvedObjectType) tool.getMetaAccess().lookupJavaType(kind == JavaKind.Object ? Object.class : kind.toJavaClass()).getArrayClass();
930 }
931
932 public void lower(NewMultiArrayNode newmultiarrayNode, LoweringTool tool) {
933 StructuredGraph graph = newmultiarrayNode.graph();
934 OptionValues localOptions = graph.getOptions();
935 int rank = newmultiarrayNode.dimensionCount();
936 ValueNode[] dims = new ValueNode[rank];
937 for (int i = 0; i < newmultiarrayNode.dimensionCount(); i++) {
938 dims[i] = newmultiarrayNode.dimension(i);
939 }
940 HotSpotResolvedObjectType type = (HotSpotResolvedObjectType) newmultiarrayNode.type();
941 ConstantNode hub = ConstantNode.forConstant(KlassPointerStamp.klassNonNull(), type.klass(), providers.getMetaAccess(), graph);
942
943 SnippetInfo snippet = GeneratePIC.getValue(localOptions) ? newmultiarrayPIC : newmultiarray;
944 Arguments args = new Arguments(snippet, graph.getGuardsStage(), tool.getLoweringStage());
945 args.add("hub", hub);
946 args.addConst("rank", rank);
947 args.addVarargs("dimensions", int.class, StampFactory.forKind(JavaKind.Int), dims);
948 template(newmultiarrayNode, args).instantiate(providers.getMetaAccess(), newmultiarrayNode, DEFAULT_REPLACER, args);
949 }
950
951 private static long instanceSize(HotSpotResolvedObjectType type) {
952 long size = type.instanceSize();
953 assert size >= 0;
954 return size;
955 }
956
957 public void lower(VerifyHeapNode verifyHeapNode, HotSpotRegistersProvider registers, LoweringTool tool) {
958 if (config.cAssertions) {
959 Arguments args = new Arguments(verifyHeap, verifyHeapNode.graph().getGuardsStage(), tool.getLoweringStage());
960 args.addConst("threadRegister", registers.getThreadRegister());
961
962 SnippetTemplate template = template(verifyHeapNode, args);
963 template.instantiate(providers.getMetaAccess(), verifyHeapNode, DEFAULT_REPLACER, args);
964 } else {
965 GraphUtil.removeFixedWithUnusedInputs(verifyHeapNode);
966 }
967 }
968 }
969 }