1 /*
2 * Copyright (c) 2012, 2019, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23
24
25 package org.graalvm.compiler.hotspot.replacements;
26
27 import static jdk.vm.ci.meta.DeoptimizationAction.InvalidateRecompile;
28 import static jdk.vm.ci.meta.DeoptimizationAction.None;
29 import static jdk.vm.ci.meta.DeoptimizationReason.RuntimeConstraint;
30 import static org.graalvm.compiler.core.common.GraalOptions.GeneratePIC;
31 import static org.graalvm.compiler.core.common.GraalOptions.MinimalBulkZeroingSize;
32 import static org.graalvm.compiler.core.common.calc.UnsignedMath.belowThan;
33 import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfigBase.INJECTED_OPTIONVALUES;
34 import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfigBase.INJECTED_VMCONFIG;
35 import static org.graalvm.compiler.hotspot.HotSpotBackend.NEW_ARRAY;
36 import static org.graalvm.compiler.hotspot.HotSpotBackend.NEW_ARRAY_OR_NULL;
37 import static org.graalvm.compiler.hotspot.HotSpotBackend.NEW_INSTANCE;
38 import static org.graalvm.compiler.hotspot.HotSpotBackend.NEW_INSTANCE_OR_NULL;
39 import static org.graalvm.compiler.hotspot.HotSpotBackend.NEW_MULTI_ARRAY;
40 import static org.graalvm.compiler.hotspot.HotSpotBackend.NEW_MULTI_ARRAY_OR_NULL;
41 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.CLASS_ARRAY_KLASS_LOCATION;
42 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.CLASS_INIT_STATE_LOCATION;
43 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.HUB_WRITE_LOCATION;
44 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.MARK_WORD_LOCATION;
45 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.PROTOTYPE_MARK_WORD_LOCATION;
46 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_END_LOCATION;
47 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_TOP_LOCATION;
48 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.allocateInstancePrefetchLines;
49 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.allocatePrefetchDistance;
50 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.allocatePrefetchLines;
51 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.allocatePrefetchStepSize;
52 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.allocatePrefetchStyle;
53 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayAllocationSize;
54 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayKlassOffset;
55 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayLengthOffset;
56 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.initializeObjectHeader;
57 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.instanceHeaderSize;
58 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.instanceKlassStateBeingInitialized;
59 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.isInstanceKlassFullyInitialized;
60 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperHeaderSizeMask;
61 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperHeaderSizeShift;
62 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperLog2ElementSizeMask;
63 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperLog2ElementSizeShift;
64 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.loadKlassFromObject;
65 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.prototypeMarkWordOffset;
66 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readInstanceKlassInitState;
67 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readInstanceKlassInitThread;
68 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readLayoutHelper;
69 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readTlabEnd;
70 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readTlabTop;
71 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.registerAsWord;
72 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.useBiasedLocking;
73 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.useTLAB;
74 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.verifyOop;
75 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.writeTlabTop;
76 import static org.graalvm.compiler.hotspot.replacements.HotspotSnippetsOptions.ProfileAllocations;
77 import static org.graalvm.compiler.hotspot.replacements.HotspotSnippetsOptions.ProfileAllocationsContext;
78 import static org.graalvm.compiler.nodes.PiArrayNode.piArrayCastToSnippetReplaceeStamp;
79 import static org.graalvm.compiler.nodes.PiNode.piCastToSnippetReplaceeStamp;
80 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.DEOPT_PROBABILITY;
81 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.FAST_PATH_PROBABILITY;
82 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.FREQUENT_PROBABILITY;
83 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.SLOW_PATH_PROBABILITY;
84 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.VERY_FAST_PATH_PROBABILITY;
85 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.probability;
86 import static org.graalvm.compiler.replacements.ReplacementsUtil.REPLACEMENTS_ASSERTIONS_ENABLED;
87 import static org.graalvm.compiler.replacements.ReplacementsUtil.runtimeAssert;
88 import static org.graalvm.compiler.replacements.ReplacementsUtil.staticAssert;
89 import static org.graalvm.compiler.replacements.SnippetTemplate.DEFAULT_REPLACER;
90 import static org.graalvm.compiler.replacements.nodes.CStringConstant.cstring;
91 import static org.graalvm.compiler.replacements.nodes.ExplodeLoopNode.explodeLoop;
92
93 import org.graalvm.compiler.api.replacements.Fold;
94 import org.graalvm.compiler.api.replacements.Fold.InjectedParameter;
95 import org.graalvm.compiler.api.replacements.Snippet;
96 import org.graalvm.compiler.api.replacements.Snippet.ConstantParameter;
97 import org.graalvm.compiler.api.replacements.Snippet.VarargsParameter;
98 import org.graalvm.compiler.core.common.spi.ForeignCallDescriptor;
99 import org.graalvm.compiler.core.common.type.StampFactory;
100 import org.graalvm.compiler.debug.DebugHandlersFactory;
101 import org.graalvm.compiler.debug.GraalError;
102 import org.graalvm.compiler.graph.Node.ConstantNodeParameter;
103 import org.graalvm.compiler.graph.Node.NodeIntrinsic;
104 import org.graalvm.compiler.hotspot.GraalHotSpotVMConfig;
105 import org.graalvm.compiler.hotspot.meta.HotSpotProviders;
106 import org.graalvm.compiler.hotspot.meta.HotSpotRegistersProvider;
107 import org.graalvm.compiler.hotspot.nodes.DimensionsNode;
108 import org.graalvm.compiler.hotspot.nodes.KlassBeingInitializedCheckNode;
109 import org.graalvm.compiler.hotspot.nodes.aot.LoadConstantIndirectlyFixedNode;
110 import org.graalvm.compiler.hotspot.nodes.aot.LoadConstantIndirectlyNode;
111 import org.graalvm.compiler.hotspot.nodes.type.KlassPointerStamp;
112 import org.graalvm.compiler.hotspot.word.KlassPointer;
113 import org.graalvm.compiler.nodes.ConstantNode;
114 import org.graalvm.compiler.nodes.DeoptimizeNode;
115 import org.graalvm.compiler.nodes.PiNode;
116 import org.graalvm.compiler.nodes.PrefetchAllocateNode;
117 import org.graalvm.compiler.nodes.SnippetAnchorNode;
118 import org.graalvm.compiler.nodes.StructuredGraph;
119 import org.graalvm.compiler.nodes.ValueNode;
120 import org.graalvm.compiler.nodes.debug.DynamicCounterNode;
121 import org.graalvm.compiler.nodes.debug.VerifyHeapNode;
122 import org.graalvm.compiler.nodes.extended.BranchProbabilityNode;
123 import org.graalvm.compiler.nodes.extended.ForeignCallNode;
124 import org.graalvm.compiler.nodes.extended.MembarNode;
125 import org.graalvm.compiler.nodes.java.DynamicNewArrayNode;
126 import org.graalvm.compiler.nodes.java.DynamicNewInstanceNode;
127 import org.graalvm.compiler.nodes.java.NewArrayNode;
128 import org.graalvm.compiler.nodes.java.NewInstanceNode;
129 import org.graalvm.compiler.nodes.java.NewMultiArrayNode;
130 import org.graalvm.compiler.nodes.memory.address.OffsetAddressNode;
131 import org.graalvm.compiler.nodes.spi.LoweringTool;
132 import org.graalvm.compiler.nodes.util.GraphUtil;
133 import org.graalvm.compiler.options.OptionValues;
134 import org.graalvm.compiler.replacements.ReplacementsUtil;
135 import org.graalvm.compiler.replacements.SnippetCounter;
136 import org.graalvm.compiler.replacements.SnippetCounter.Group;
137 import org.graalvm.compiler.replacements.SnippetTemplate;
138 import org.graalvm.compiler.replacements.SnippetTemplate.AbstractTemplates;
139 import org.graalvm.compiler.replacements.SnippetTemplate.Arguments;
140 import org.graalvm.compiler.replacements.SnippetTemplate.SnippetInfo;
141 import org.graalvm.compiler.replacements.Snippets;
142 import org.graalvm.compiler.replacements.nodes.ExplodeLoopNode;
143 import org.graalvm.compiler.replacements.nodes.ZeroMemoryNode;
144 import org.graalvm.compiler.word.Word;
145 import jdk.internal.vm.compiler.word.LocationIdentity;
146 import jdk.internal.vm.compiler.word.WordFactory;
147
148 import jdk.vm.ci.code.CodeUtil;
149 import jdk.vm.ci.code.MemoryBarriers;
150 import jdk.vm.ci.code.Register;
151 import jdk.vm.ci.code.TargetDescription;
152 import jdk.vm.ci.hotspot.HotSpotResolvedObjectType;
153 import jdk.vm.ci.meta.JavaKind;
154 import jdk.vm.ci.meta.ResolvedJavaType;
155
156 /**
157 * Snippets used for implementing NEW, ANEWARRAY and NEWARRAY.
158 */
159 public class NewObjectSnippets implements Snippets {
160
161 enum ProfileContext {
162 AllocatingMethod,
163 InstanceOrArray,
164 AllocatedType,
165 AllocatedTypesInMethod,
166 Total
167 }
168
169 @Fold
170 static String createName(@Fold.InjectedParameter OptionValues options, String path, String typeContext) {
171 switch (ProfileAllocationsContext.getValue(options)) {
172 case AllocatingMethod:
173 return "";
174 case InstanceOrArray:
175 return path;
176 case AllocatedType:
177 case AllocatedTypesInMethod:
178 return typeContext;
179 case Total:
180 return "bytes";
181 default:
182 throw GraalError.shouldNotReachHere();
183 }
184 }
185
186 @Fold
187 static boolean doProfile(@Fold.InjectedParameter OptionValues options) {
188 return ProfileAllocations.getValue(options);
189 }
190
191 @Fold
192 static boolean withContext(@Fold.InjectedParameter OptionValues options) {
193 ProfileContext context = ProfileAllocationsContext.getValue(options);
194 return context == ProfileContext.AllocatingMethod || context == ProfileContext.AllocatedTypesInMethod;
195 }
196
197 protected static void profileAllocation(String path, long size, String typeContext) {
198 if (doProfile(INJECTED_OPTIONVALUES)) {
199 String name = createName(INJECTED_OPTIONVALUES, path, typeContext);
200
201 boolean context = withContext(INJECTED_OPTIONVALUES);
202 DynamicCounterNode.counter("number of bytes allocated", name, size, context);
203 DynamicCounterNode.counter("number of allocations", name, 1, context);
204 }
205 }
206
207 public static void emitPrefetchAllocate(Word address, boolean isArray) {
208 if (allocatePrefetchStyle(INJECTED_VMCONFIG) > 0) {
209 // Insert a prefetch for each allocation only on the fast-path
210 // Generate several prefetch instructions.
211 int lines = isArray ? allocatePrefetchLines(INJECTED_VMCONFIG) : allocateInstancePrefetchLines(INJECTED_VMCONFIG);
212 int stepSize = allocatePrefetchStepSize(INJECTED_VMCONFIG);
213 int distance = allocatePrefetchDistance(INJECTED_VMCONFIG);
214 ExplodeLoopNode.explodeLoop();
215 for (int i = 0; i < lines; i++) {
216 PrefetchAllocateNode.prefetch(OffsetAddressNode.address(address, distance));
217 distance += stepSize;
218 }
219 }
220 }
221
222 @Snippet
223 public static Object allocateInstance(@ConstantParameter long size,
224 KlassPointer hub,
225 Word prototypeMarkWord,
226 @ConstantParameter boolean fillContents,
227 @ConstantParameter boolean emitMemoryBarrier,
228 @ConstantParameter Register threadRegister,
229 @ConstantParameter boolean constantSize,
230 @ConstantParameter String typeContext,
231 @ConstantParameter Counters counters) {
232 return piCastToSnippetReplaceeStamp(allocateInstanceHelper(size, hub, prototypeMarkWord, fillContents, emitMemoryBarrier, threadRegister, constantSize, typeContext, counters));
233 }
234
235 public static Object allocateInstanceHelper(long size,
236 KlassPointer hub,
237 Word prototypeMarkWord,
238 boolean fillContents,
239 boolean emitMemoryBarrier,
240 Register threadRegister,
241 boolean constantSize,
242 String typeContext,
243 Counters counters) {
244 Object result;
245 Word thread = registerAsWord(threadRegister);
246 Word top = readTlabTop(thread);
247 Word end = readTlabEnd(thread);
248 Word newTop = top.add(WordFactory.unsigned(size));
249 if (useTLAB(INJECTED_VMCONFIG) && probability(FAST_PATH_PROBABILITY, newTop.belowOrEqual(end))) {
250 writeTlabTop(thread, newTop);
251 emitPrefetchAllocate(newTop, false);
252 result = formatObject(hub, size, top, prototypeMarkWord, fillContents, emitMemoryBarrier, constantSize, counters);
253 } else {
254 Counters theCounters = counters;
255 if (theCounters != null && theCounters.stub != null) {
256 theCounters.stub.inc();
257 }
258 result = newInstanceStub(hub);
259 }
260 profileAllocation("instance", size, typeContext);
261 return verifyOop(result);
262 }
263
264 public static Object newInstanceStub(KlassPointer hub) {
265 if (useNullAllocationStubs(INJECTED_VMCONFIG)) {
266 return nonNullOrDeopt(newInstanceOrNull(NEW_INSTANCE_OR_NULL, hub));
267 } else {
268 return newInstance(NEW_INSTANCE, hub);
269 }
270 }
271
272 @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = true)
273 private static native Object newInstance(@ConstantNodeParameter ForeignCallDescriptor descriptor, KlassPointer hub);
274
275 @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = false)
276 private static native Object newInstanceOrNull(@ConstantNodeParameter ForeignCallDescriptor descriptor, KlassPointer hub);
277
278 @Snippet
279 public static Object allocateInstancePIC(@ConstantParameter long size,
280 KlassPointer hub,
281 Word prototypeMarkWord,
282 @ConstantParameter boolean fillContents,
283 @ConstantParameter boolean emitMemoryBarrier,
284 @ConstantParameter Register threadRegister,
285 @ConstantParameter boolean constantSize,
286 @ConstantParameter String typeContext,
287 @ConstantParameter Counters counters) {
288 // Klass must be initialized by the time the first instance is allocated, therefore we can
289 // just load it from the corresponding cell and avoid the resolution check. We have to use a
290 // fixed load though, to prevent it from floating above the initialization.
291 KlassPointer picHub = LoadConstantIndirectlyFixedNode.loadKlass(hub);
292 return piCastToSnippetReplaceeStamp(allocateInstanceHelper(size, picHub, prototypeMarkWord, fillContents, emitMemoryBarrier, threadRegister, constantSize, typeContext, counters));
293 }
294
295 @Snippet
296 public static Object allocateInstanceDynamic(Class<?> type, Class<?> classClass,
297 @ConstantParameter boolean fillContents,
298 @ConstantParameter boolean emitMemoryBarrier,
299 @ConstantParameter Register threadRegister,
300 @ConstantParameter Counters counters) {
301 if (probability(DEOPT_PROBABILITY, type == null)) {
302 DeoptimizeNode.deopt(None, RuntimeConstraint);
303 }
304 Class<?> nonNullType = PiNode.piCastNonNullClass(type, SnippetAnchorNode.anchor());
305
306 if (probability(DEOPT_PROBABILITY, DynamicNewInstanceNode.throwsInstantiationException(type, classClass))) {
307 DeoptimizeNode.deopt(None, RuntimeConstraint);
308 }
309
310 return PiNode.piCastToSnippetReplaceeStamp(allocateInstanceDynamicHelper(type, fillContents, emitMemoryBarrier, threadRegister, counters, nonNullType));
311 }
312
313 private static Object allocateInstanceDynamicHelper(Class<?> type,
314 boolean fillContents,
315 boolean emitMemoryBarrier,
316 Register threadRegister,
317 Counters counters,
318 Class<?> nonNullType) {
319 KlassPointer hub = ClassGetHubNode.readClass(nonNullType);
320 if (probability(FAST_PATH_PROBABILITY, !hub.isNull())) {
321 KlassPointer nonNullHub = ClassGetHubNode.piCastNonNull(hub, SnippetAnchorNode.anchor());
322
323 if (probability(VERY_FAST_PATH_PROBABILITY, isInstanceKlassFullyInitialized(nonNullHub))) {
324 int layoutHelper = readLayoutHelper(nonNullHub);
325 /*
326 * src/share/vm/oops/klass.hpp: For instances, layout helper is a positive number,
327 * the instance size. This size is already passed through align_object_size and
328 * scaled to bytes. The low order bit is set if instances of this class cannot be
329 * allocated using the fastpath.
330 */
331 if (probability(FAST_PATH_PROBABILITY, (layoutHelper & 1) == 0)) {
332 Word prototypeMarkWord = nonNullHub.readWord(prototypeMarkWordOffset(INJECTED_VMCONFIG), PROTOTYPE_MARK_WORD_LOCATION);
333 /*
334 * FIXME(je,ds): we should actually pass typeContext instead of "" but late
335 * binding of parameters is not yet supported by the GraphBuilderPlugin system.
336 */
337 return allocateInstanceHelper(layoutHelper, nonNullHub, prototypeMarkWord, fillContents, emitMemoryBarrier, threadRegister, false, "", counters);
338 }
339 } else {
340 DeoptimizeNode.deopt(None, RuntimeConstraint);
341 }
342 }
343 return dynamicNewInstanceStub(type);
344 }
345
346 /**
347 * Maximum array length for which fast path allocation is used.
348 */
349 public static final int MAX_ARRAY_FAST_PATH_ALLOCATION_LENGTH = 0x00FFFFFF;
350
351 @Snippet
352 public static Object allocatePrimitiveArrayPIC(KlassPointer hub,
353 int length,
354 Word prototypeMarkWord,
355 @ConstantParameter int headerSize,
356 @ConstantParameter int log2ElementSize,
357 @ConstantParameter boolean fillContents,
358 @ConstantParameter boolean emitMemoryBarrier,
359 @ConstantParameter Register threadRegister,
360 @ConstantParameter boolean maybeUnroll,
361 @ConstantParameter String typeContext,
362 @ConstantParameter int bulkZeroingStride,
363 @ConstantParameter Counters counters) {
364 // Primitive array types are eagerly pre-resolved. We can use a floating load.
365 KlassPointer picHub = LoadConstantIndirectlyNode.loadKlass(hub);
366 return allocateArrayImpl(picHub, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents,
367 emitMemoryBarrier, threadRegister, maybeUnroll, typeContext, bulkZeroingStride, counters);
368 }
369
370 @Snippet
371 public static Object allocateArrayPIC(KlassPointer hub,
372 int length,
373 Word prototypeMarkWord,
374 @ConstantParameter int headerSize,
375 @ConstantParameter int log2ElementSize,
376 @ConstantParameter boolean fillContents,
377 @ConstantParameter boolean emitMemoryBarrier,
378 @ConstantParameter Register threadRegister,
379 @ConstantParameter boolean maybeUnroll,
380 @ConstantParameter String typeContext,
381 @ConstantParameter int bulkZeroingStride,
382 @ConstantParameter Counters counters) {
383 // Array type would be resolved by dominating resolution.
384 KlassPointer picHub = LoadConstantIndirectlyFixedNode.loadKlass(hub);
385 return allocateArrayImpl(picHub, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents,
386 emitMemoryBarrier, threadRegister, maybeUnroll, typeContext, bulkZeroingStride, counters);
387 }
388
389 @Snippet
390 public static Object allocateArray(KlassPointer hub,
391 int length,
392 Word prototypeMarkWord,
393 @ConstantParameter int headerSize,
394 @ConstantParameter int log2ElementSize,
395 @ConstantParameter boolean fillContents,
396 @ConstantParameter boolean emitMemoryBarrier,
397 @ConstantParameter Register threadRegister,
398 @ConstantParameter boolean maybeUnroll,
399 @ConstantParameter String typeContext,
400 @ConstantParameter int bulkZeroingStride,
401 @ConstantParameter Counters counters) {
402 Object result = allocateArrayImpl(hub,
403 length,
404 prototypeMarkWord,
405 headerSize,
406 log2ElementSize,
407 fillContents,
408 emitMemoryBarrier, threadRegister,
409 maybeUnroll,
410 typeContext,
411 bulkZeroingStride,
412 counters);
413 return piArrayCastToSnippetReplaceeStamp(verifyOop(result), length);
414 }
415
416 /**
417 * When allocating on the slow path, determines whether to use a version of the runtime call
418 * that returns {@code null} on a failed allocation instead of raising an OutOfMemoryError.
419 */
420 @Fold
421 static boolean useNullAllocationStubs(@InjectedParameter GraalHotSpotVMConfig config) {
422 return config.areNullAllocationStubsAvailable();
423 }
424
425 private static Object allocateArrayImpl(KlassPointer hub,
426 int length,
427 Word prototypeMarkWord,
428 int headerSize,
429 int log2ElementSize,
430 boolean fillContents,
431 boolean emitMemoryBarrier,
432 Register threadRegister,
433 boolean maybeUnroll,
434 String typeContext,
435 int bulkZeroingStride,
436 Counters counters) {
437 Object result;
438 long allocationSize = arrayAllocationSize(length, headerSize, log2ElementSize);
439 Word thread = registerAsWord(threadRegister);
440 Word top = readTlabTop(thread);
441 Word end = readTlabEnd(thread);
442 Word newTop = top.add(WordFactory.unsigned(allocationSize));
443 if (probability(FREQUENT_PROBABILITY, belowThan(length, MAX_ARRAY_FAST_PATH_ALLOCATION_LENGTH)) && useTLAB(INJECTED_VMCONFIG) &&
444 probability(FAST_PATH_PROBABILITY, newTop.belowOrEqual(end))) {
445 writeTlabTop(thread, newTop);
446 emitPrefetchAllocate(newTop, true);
447 Counters theCounters = counters;
448 if (theCounters != null && theCounters.arrayLoopInit != null) {
449 theCounters.arrayLoopInit.inc();
450 }
451 result = formatArray(hub, allocationSize, length, headerSize, top, prototypeMarkWord, fillContents, emitMemoryBarrier, maybeUnroll, bulkZeroingStride, counters);
452 } else {
453 result = newArrayStub(hub, length);
454 }
455 profileAllocation("array", allocationSize, typeContext);
456 return result;
457 }
458
459 public static Object newArrayStub(KlassPointer hub, int length) {
460 if (useNullAllocationStubs(INJECTED_VMCONFIG)) {
461 return nonNullOrDeopt(newArrayOrNull(NEW_ARRAY_OR_NULL, hub, length));
462 } else {
463 return newArray(NEW_ARRAY, hub, length);
464 }
465 }
466
467 @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = true)
468 private static native Object newArray(@ConstantNodeParameter ForeignCallDescriptor descriptor, KlassPointer hub, int length);
469
470 @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = false)
471 private static native Object newArrayOrNull(@ConstantNodeParameter ForeignCallDescriptor descriptor, KlassPointer hub, int length);
472
473 /**
474 * New dynamic array stub that throws an {@link OutOfMemoryError} on allocation failure.
475 */
476 public static final ForeignCallDescriptor DYNAMIC_NEW_INSTANCE = new ForeignCallDescriptor("dynamic_new_instance", Object.class, Class.class);
477
478 /**
479 * New dynamic array stub that returns null on allocation failure.
480 */
481 public static final ForeignCallDescriptor DYNAMIC_NEW_INSTANCE_OR_NULL = new ForeignCallDescriptor("dynamic_new_instance_or_null", Object.class, Class.class);
482
483 public static Object dynamicNewInstanceStub(Class<?> elementType) {
484 if (useNullAllocationStubs(INJECTED_VMCONFIG)) {
485 return nonNullOrDeopt(dynamicNewInstanceOrNull(DYNAMIC_NEW_INSTANCE_OR_NULL, elementType));
486 } else {
487 return dynamicNewInstance(DYNAMIC_NEW_INSTANCE, elementType);
488 }
489 }
490
491 /**
492 * Deoptimizes if {@code obj == null} otherwise returns {@code obj}.
493 */
494 private static Object nonNullOrDeopt(Object obj) {
495 if (BranchProbabilityNode.probability(BranchProbabilityNode.DEOPT_PROBABILITY, obj == null)) {
496 DeoptimizeNode.deopt(None, RuntimeConstraint);
497 }
498 return obj;
499 }
500
501 @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = true)
502 public static native Object dynamicNewInstance(@ConstantNodeParameter ForeignCallDescriptor descriptor, Class<?> elementType);
503
504 @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = false)
505 public static native Object dynamicNewInstanceOrNull(@ConstantNodeParameter ForeignCallDescriptor descriptor, Class<?> elementType);
506
507 @Snippet
508 public static Object allocateArrayDynamic(Class<?> elementType,
509 Class<?> voidClass,
510 int length,
511 @ConstantParameter boolean fillContents,
512 @ConstantParameter boolean emitMemoryBarrier,
513 @ConstantParameter Register threadRegister,
514 @ConstantParameter JavaKind knownElementKind,
515 @ConstantParameter int knownLayoutHelper,
516 @ConstantParameter int bulkZeroingStride,
517 Word prototypeMarkWord,
518 @ConstantParameter Counters counters) {
519 Object result = allocateArrayDynamicImpl(elementType, voidClass, length, fillContents, emitMemoryBarrier, threadRegister, knownElementKind,
520 knownLayoutHelper, bulkZeroingStride, prototypeMarkWord, counters);
521 return result;
522 }
523
524 private static Object allocateArrayDynamicImpl(Class<?> elementType,
525 Class<?> voidClass,
526 int length,
527 boolean fillContents,
528 boolean emitMemoryBarrier,
529 Register threadRegister,
530 JavaKind knownElementKind,
531 int knownLayoutHelper,
532 int bulkZeroingStride,
533 Word prototypeMarkWord,
534 Counters counters) {
535 /*
536 * We only need the dynamic check for void when we have no static information from
537 * knownElementKind.
538 */
539 staticAssert(knownElementKind != JavaKind.Void, "unsupported knownElementKind");
540 if (knownElementKind == JavaKind.Illegal && probability(SLOW_PATH_PROBABILITY, elementType == null || DynamicNewArrayNode.throwsIllegalArgumentException(elementType, voidClass))) {
541 DeoptimizeNode.deopt(None, RuntimeConstraint);
542 }
543
544 KlassPointer klass = loadKlassFromObject(elementType, arrayKlassOffset(INJECTED_VMCONFIG), CLASS_ARRAY_KLASS_LOCATION);
545 if (probability(DEOPT_PROBABILITY, klass.isNull())) {
546 DeoptimizeNode.deopt(None, RuntimeConstraint);
547 }
548 KlassPointer nonNullKlass = ClassGetHubNode.piCastNonNull(klass, SnippetAnchorNode.anchor());
549
550 if (probability(DEOPT_PROBABILITY, length < 0)) {
551 DeoptimizeNode.deopt(None, RuntimeConstraint);
552 }
553 int layoutHelper;
554 if (knownElementKind == JavaKind.Illegal) {
555 layoutHelper = readLayoutHelper(nonNullKlass);
556 } else {
557 runtimeAssert(knownLayoutHelper == readLayoutHelper(nonNullKlass), "layout mismatch");
558 layoutHelper = knownLayoutHelper;
559 }
560 //@formatter:off
561 // from src/share/vm/oops/klass.hpp:
562 //
563 // For arrays, layout helper is a negative number, containing four
564 // distinct bytes, as follows:
565 // MSB:[tag, hsz, ebt, log2(esz)]:LSB
566 // where:
567 // tag is 0x80 if the elements are oops, 0xC0 if non-oops
568 // hsz is array header size in bytes (i.e., offset of first element)
569 // ebt is the BasicType of the elements
570 // esz is the element size in bytes
571 //@formatter:on
572
573 int headerSize = (layoutHelper >> layoutHelperHeaderSizeShift(INJECTED_VMCONFIG)) & layoutHelperHeaderSizeMask(INJECTED_VMCONFIG);
574 int log2ElementSize = (layoutHelper >> layoutHelperLog2ElementSizeShift(INJECTED_VMCONFIG)) & layoutHelperLog2ElementSizeMask(INJECTED_VMCONFIG);
575
576 Object result = allocateArrayImpl(nonNullKlass, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents,
577 emitMemoryBarrier, threadRegister, false, "dynamic type", bulkZeroingStride, counters);
578 return piArrayCastToSnippetReplaceeStamp(verifyOop(result), length);
579 }
580
581 /**
582 * Calls the runtime stub for implementing MULTIANEWARRAY.
583 */
584 @Snippet
585 private static Object newmultiarray(KlassPointer hub, @ConstantParameter int rank, @VarargsParameter int[] dimensions) {
586 Word dims = DimensionsNode.allocaDimsArray(rank);
587 ExplodeLoopNode.explodeLoop();
588 for (int i = 0; i < rank; i++) {
589 dims.writeInt(i * 4, dimensions[i], LocationIdentity.init());
590 }
591 return newMultiArrayStub(hub, rank, dims);
592 }
593
594 private static Object newMultiArrayStub(KlassPointer hub, int rank, Word dims) {
595 if (useNullAllocationStubs(INJECTED_VMCONFIG)) {
596 return nonNullOrDeopt(newMultiArrayOrNull(NEW_MULTI_ARRAY_OR_NULL, hub, rank, dims));
597 } else {
598 return newMultiArray(NEW_MULTI_ARRAY, hub, rank, dims);
599 }
600 }
601
602 @Snippet
603 private static Object newmultiarrayPIC(KlassPointer hub, @ConstantParameter int rank, @VarargsParameter int[] dimensions) {
604 // Array type would be resolved by dominating resolution.
605 KlassPointer picHub = LoadConstantIndirectlyFixedNode.loadKlass(hub);
606 return newmultiarray(picHub, rank, dimensions);
607 }
608
609 @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = true)
610 private static native Object newMultiArray(@ConstantNodeParameter ForeignCallDescriptor descriptor, KlassPointer hub, int rank, Word dims);
611
612 @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = false)
613 private static native Object newMultiArrayOrNull(@ConstantNodeParameter ForeignCallDescriptor descriptor, KlassPointer hub, int rank, Word dims);
614
615 /**
616 * Maximum number of long stores to emit when zeroing an object with a constant size. Larger
617 * objects have their bodies initialized in a loop.
618 */
619 private static final int MAX_UNROLLED_OBJECT_ZEROING_STORES = 8;
620
621 /**
622 * Zero uninitialized memory in a newly allocated object, unrolling as necessary and ensuring
623 * that stores are aligned.
624 *
625 * @param memory beginning of object which is being zeroed
626 * @param startOffset offset to begin zeroing (inclusive). May not be word aligned.
627 * @param endOffset offset to stop zeroing (exclusive). May not be word aligned.
628 * @param isEndOffsetConstant is {@code endOffset} known to be constant in the snippet
629 * @param manualUnroll maximally unroll zeroing
630 * @param bulkZeroingStride stride of bulk zeroing supported by the backend
631 */
632 private static void zeroMemory(Word memory, int startOffset, long endOffset, boolean isEndOffsetConstant, boolean manualUnroll,
633 int bulkZeroingStride, Counters counters) {
634 fillMemory(0, memory, startOffset, endOffset, isEndOffsetConstant, manualUnroll, bulkZeroingStride, counters);
635 }
636
637 private static void fillMemory(long value, Word memory, int startOffset, long offsetLimit, boolean constantOffsetLimit, boolean manualUnroll,
638 int bulkZeroingStride, Counters counters) {
639 ReplacementsUtil.runtimeAssert((offsetLimit & 0x7) == 0, "unaligned object size");
640 int offset = startOffset;
641 if ((offset & 0x7) != 0) {
642 memory.writeInt(offset, (int) value, LocationIdentity.init());
643 offset += 4;
644 }
645 ReplacementsUtil.runtimeAssert((offset & 0x7) == 0, "unaligned offset");
646 Counters theCounters = counters;
647 if (manualUnroll && ((offsetLimit - offset) / 8) <= MAX_UNROLLED_OBJECT_ZEROING_STORES) {
648 ReplacementsUtil.staticAssert(!constantOffsetLimit, "size shouldn't be constant at instantiation time");
649 // This case handles arrays of constant length. Instead of having a snippet variant for
650 // each length, generate a chain of stores of maximum length. Once it's inlined the
651 // break statement will trim excess stores.
652 if (theCounters != null && theCounters.instanceSeqInit != null) {
653 theCounters.instanceSeqInit.inc();
654 }
655
656 explodeLoop();
657 for (int i = 0; i < MAX_UNROLLED_OBJECT_ZEROING_STORES; i++, offset += 8) {
658 if (offset == offsetLimit) {
659 break;
660 }
661 memory.initializeLong(offset, value, LocationIdentity.init());
662 }
663 } else {
664 // Use Word instead of int to avoid extension to long in generated code
665 Word off = WordFactory.signed(offset);
666 if (bulkZeroingStride > 0 && value == 0 && probability(SLOW_PATH_PROBABILITY, (offsetLimit - offset) >= getMinimalBulkZeroingSize(INJECTED_OPTIONVALUES))) {
667 if (theCounters != null && theCounters.instanceBulkInit != null) {
668 theCounters.instanceBulkInit.inc();
669 }
670 ZeroMemoryNode.zero(memory.add(off), offsetLimit - offset, LocationIdentity.init());
671 } else {
672 if (constantOffsetLimit && ((offsetLimit - offset) / 8) <= MAX_UNROLLED_OBJECT_ZEROING_STORES) {
673 if (theCounters != null && theCounters.instanceSeqInit != null) {
674 theCounters.instanceSeqInit.inc();
675 }
676 explodeLoop();
677 } else {
678 if (theCounters != null && theCounters.instanceLoopInit != null) {
679 theCounters.instanceLoopInit.inc();
680 }
681 }
682 for (; off.rawValue() < offsetLimit; off = off.add(8)) {
683 memory.initializeLong(off, value, LocationIdentity.init());
684 }
685 }
686 }
687 }
688
689 @Fold
690 static int getMinimalBulkZeroingSize(@InjectedParameter OptionValues optionValues) {
691 return MinimalBulkZeroingSize.getValue(optionValues);
692 }
693
694 /**
695 * Fill uninitialized memory with garbage value in a newly allocated object, unrolling as
696 * necessary and ensuring that stores are aligned.
697 *
698 * @param memory beginning of object which is being zeroed
699 * @param startOffset offset to begin filling garbage value (inclusive). May not be word
700 * aligned.
701 * @param endOffset offset to stop filling garbage value (exclusive). May not be word aligned.
702 * @param isEndOffsetConstant is {@code endOffset} known to be constant in the snippet
703 * @param manualUnroll maximally unroll zeroing
704 */
705 private static void fillWithGarbage(Word memory, int startOffset, long endOffset, boolean isEndOffsetConstant, boolean manualUnroll, Counters counters) {
706 fillMemory(0xfefefefefefefefeL, memory, startOffset, endOffset, isEndOffsetConstant, manualUnroll, 0, counters);
707 }
708
709 /**
710 * Formats some allocated memory with an object header and zeroes out the rest.
711 */
712 private static Object formatObject(KlassPointer hub,
713 long size,
714 Word memory,
715 Word compileTimePrototypeMarkWord,
716 boolean fillContents,
717 boolean emitMemoryBarrier,
718 boolean constantSize,
719 Counters counters) {
720 Word prototypeMarkWord = useBiasedLocking(INJECTED_VMCONFIG) ? hub.readWord(prototypeMarkWordOffset(INJECTED_VMCONFIG), PROTOTYPE_MARK_WORD_LOCATION) : compileTimePrototypeMarkWord;
721 initializeObjectHeader(memory, prototypeMarkWord, hub);
722 if (fillContents) {
723 zeroMemory(memory, instanceHeaderSize(INJECTED_VMCONFIG), size, constantSize, false, 0, counters);
724 } else if (REPLACEMENTS_ASSERTIONS_ENABLED) {
725 fillWithGarbage(memory, instanceHeaderSize(INJECTED_VMCONFIG), size, constantSize, false, counters);
726 }
727 if (emitMemoryBarrier) {
728 MembarNode.memoryBarrier(MemoryBarriers.STORE_STORE, LocationIdentity.init());
729 }
730 return memory.toObjectNonNull();
731 }
732
733 @Snippet
734 private static void verifyHeap(@ConstantParameter Register threadRegister) {
735 Word thread = registerAsWord(threadRegister);
736 Word topValue = readTlabTop(thread);
737 if (!topValue.equal(WordFactory.zero())) {
738 Word topValueContents = topValue.readWord(0, MARK_WORD_LOCATION);
739 if (topValueContents.equal(WordFactory.zero())) {
740 AssertionSnippets.vmMessageC(AssertionSnippets.ASSERTION_VM_MESSAGE_C, true, cstring("overzeroing of TLAB detected"), 0L, 0L, 0L);
741 }
742 }
743 }
744
745 @Snippet
746 private static void threadBeingInitializedCheck(@ConstantParameter Register threadRegister, KlassPointer klass) {
747 int state = readInstanceKlassInitState(klass);
748 if (state != instanceKlassStateBeingInitialized(INJECTED_VMCONFIG)) {
749 // The klass is no longer being initialized so force recompilation
750 DeoptimizeNode.deopt(InvalidateRecompile, RuntimeConstraint);
751 } else if (registerAsWord(threadRegister) != readInstanceKlassInitThread(klass)) {
752 // The klass is being initialized but this isn't the initializing thread so
753 // so deopt and allow execution to resume in the interpreter where it should block.
754 DeoptimizeNode.deopt(None, RuntimeConstraint);
755 }
756 }
757
758 /**
759 * Formats some allocated memory with an object header and zeroes out the rest.
760 */
761 private static Object formatArray(KlassPointer hub,
762 long allocationSize,
763 int length,
764 int headerSize,
765 Word memory,
766 Word prototypeMarkWord,
767 boolean fillContents,
768 boolean emitMemoryBarrier,
769 boolean maybeUnroll,
770 int bulkZeroingStride,
771 Counters counters) {
772 memory.writeInt(arrayLengthOffset(INJECTED_VMCONFIG), length, LocationIdentity.init());
773 /*
774 * store hub last as the concurrent garbage collectors assume length is valid if hub field
775 * is not null
776 */
777 initializeObjectHeader(memory, prototypeMarkWord, hub);
778 if (fillContents) {
779 zeroMemory(memory, headerSize, allocationSize, false, maybeUnroll, bulkZeroingStride, counters);
780 } else if (REPLACEMENTS_ASSERTIONS_ENABLED) {
781 fillWithGarbage(memory, headerSize, allocationSize, false, maybeUnroll, counters);
782 }
783 if (emitMemoryBarrier) {
784 MembarNode.memoryBarrier(MemoryBarriers.STORE_STORE, LocationIdentity.init());
785 }
786 return memory.toObjectNonNull();
787 }
788
789 static class Counters {
790 Counters(SnippetCounter.Group.Factory factory) {
791 Group newInstance = factory.createSnippetCounterGroup("NewInstance");
792 Group newArray = factory.createSnippetCounterGroup("NewArray");
793 instanceSeqInit = new SnippetCounter(newInstance, "tlabSeqInit", "TLAB alloc with unrolled zeroing");
794 instanceLoopInit = new SnippetCounter(newInstance, "tlabLoopInit", "TLAB alloc with zeroing in a loop");
795 instanceBulkInit = new SnippetCounter(newArray, "tlabBulkInit", "TLAB alloc with bulk zeroing");
796 arrayLoopInit = new SnippetCounter(newArray, "tlabLoopInit", "TLAB alloc with zeroing in a loop");
797 stub = new SnippetCounter(newInstance, "stub", "alloc and zeroing via stub");
798 }
799
800 final SnippetCounter instanceSeqInit;
801 final SnippetCounter instanceLoopInit;
802 final SnippetCounter instanceBulkInit;
803 final SnippetCounter arrayLoopInit;
804 final SnippetCounter stub;
805 }
806
807 public static class Templates extends AbstractTemplates {
808
809 private final SnippetInfo allocateInstance = snippet(NewObjectSnippets.class, "allocateInstance", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION, TLAB_END_LOCATION,
810 PROTOTYPE_MARK_WORD_LOCATION);
811 private final SnippetInfo allocateInstancePIC = snippet(NewObjectSnippets.class, "allocateInstancePIC", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION,
812 TLAB_END_LOCATION, PROTOTYPE_MARK_WORD_LOCATION);
813 private final SnippetInfo allocateArray = snippet(NewObjectSnippets.class, "allocateArray", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION, TLAB_END_LOCATION);
814 private final SnippetInfo allocateArrayPIC = snippet(NewObjectSnippets.class, "allocateArrayPIC", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION, TLAB_END_LOCATION);
815 private final SnippetInfo allocatePrimitiveArrayPIC = snippet(NewObjectSnippets.class, "allocatePrimitiveArrayPIC", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION,
816 TLAB_END_LOCATION);
817 private final SnippetInfo allocateArrayDynamic = snippet(NewObjectSnippets.class, "allocateArrayDynamic", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION,
818 TLAB_END_LOCATION);
819 private final SnippetInfo allocateInstanceDynamic = snippet(NewObjectSnippets.class, "allocateInstanceDynamic", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION,
820 TLAB_END_LOCATION, PROTOTYPE_MARK_WORD_LOCATION, CLASS_INIT_STATE_LOCATION);
821 private final SnippetInfo newmultiarray = snippet(NewObjectSnippets.class, "newmultiarray", TLAB_TOP_LOCATION, TLAB_END_LOCATION);
822 private final SnippetInfo newmultiarrayPIC = snippet(NewObjectSnippets.class, "newmultiarrayPIC", TLAB_TOP_LOCATION, TLAB_END_LOCATION);
823 private final SnippetInfo verifyHeap = snippet(NewObjectSnippets.class, "verifyHeap");
824 private final SnippetInfo threadBeingInitializedCheck = snippet(NewObjectSnippets.class, "threadBeingInitializedCheck");
825 private final GraalHotSpotVMConfig config;
826 private final Counters counters;
827
828 public Templates(OptionValues options, Iterable<DebugHandlersFactory> factories, SnippetCounter.Group.Factory factory, HotSpotProviders providers, TargetDescription target,
829 GraalHotSpotVMConfig config) {
830 super(options, factories, providers, providers.getSnippetReflection(), target);
831 this.config = config;
832 counters = new Counters(factory);
833 }
834
835 /**
836 * Lowers a {@link NewInstanceNode}.
837 */
838 public void lower(NewInstanceNode newInstanceNode, HotSpotRegistersProvider registers, LoweringTool tool) {
839 StructuredGraph graph = newInstanceNode.graph();
840 HotSpotResolvedObjectType type = (HotSpotResolvedObjectType) newInstanceNode.instanceClass();
841 assert !type.isArray();
842 ConstantNode hub = ConstantNode.forConstant(KlassPointerStamp.klassNonNull(), type.klass(), providers.getMetaAccess(), graph);
843 long size = instanceSize(type);
844
845 OptionValues localOptions = graph.getOptions();
846 SnippetInfo snippet = GeneratePIC.getValue(localOptions) ? allocateInstancePIC : allocateInstance;
847 Arguments args = new Arguments(snippet, graph.getGuardsStage(), tool.getLoweringStage());
848 args.addConst("size", size);
849 args.add("hub", hub);
850 args.add("prototypeMarkWord", type.prototypeMarkWord());
851 args.addConst("fillContents", newInstanceNode.fillContents());
852 args.addConst("emitMemoryBarrier", newInstanceNode.emitMemoryBarrier());
853 args.addConst("threadRegister", registers.getThreadRegister());
854 args.addConst("constantSize", true);
855 args.addConst("typeContext", ProfileAllocations.getValue(localOptions) ? type.toJavaName(false) : "");
856 args.addConst("counters", counters);
857
858 SnippetTemplate template = template(newInstanceNode, args);
859 graph.getDebug().log("Lowering allocateInstance in %s: node=%s, template=%s, arguments=%s", graph, newInstanceNode, template, args);
860 template.instantiate(providers.getMetaAccess(), newInstanceNode, DEFAULT_REPLACER, args);
861 }
862
863 /**
864 * Lowers a {@link NewArrayNode}.
865 */
866 public void lower(NewArrayNode newArrayNode, HotSpotRegistersProvider registers, LoweringTool tool) {
867 StructuredGraph graph = newArrayNode.graph();
868 ResolvedJavaType elementType = newArrayNode.elementType();
869 HotSpotResolvedObjectType arrayType = (HotSpotResolvedObjectType) elementType.getArrayClass();
870 JavaKind elementKind = elementType.getJavaKind();
871 ConstantNode hub = ConstantNode.forConstant(KlassPointerStamp.klassNonNull(), arrayType.klass(), providers.getMetaAccess(), graph);
872 final int headerSize = tool.getMetaAccess().getArrayBaseOffset(elementKind);
873 int log2ElementSize = CodeUtil.log2(tool.getMetaAccess().getArrayIndexScale(elementKind));
874
875 OptionValues localOptions = graph.getOptions();
876 SnippetInfo snippet;
877 if (GeneratePIC.getValue(localOptions)) {
878 if (elementType.isPrimitive()) {
879 snippet = allocatePrimitiveArrayPIC;
880 } else {
881 snippet = allocateArrayPIC;
882 }
883 } else {
884 snippet = allocateArray;
885 }
886
887 Arguments args = new Arguments(snippet, graph.getGuardsStage(), tool.getLoweringStage());
888 args.add("hub", hub);
889 ValueNode length = newArrayNode.length();
890 args.add("length", length.isAlive() ? length : graph.addOrUniqueWithInputs(length));
891 assert arrayType.prototypeMarkWord() == lookupArrayClass(tool, JavaKind.Object).prototypeMarkWord() : "all array types are assumed to have the same prototypeMarkWord";
892 args.add("prototypeMarkWord", arrayType.prototypeMarkWord());
893 args.addConst("headerSize", headerSize);
894 args.addConst("log2ElementSize", log2ElementSize);
895 args.addConst("fillContents", newArrayNode.fillContents());
896 args.addConst("emitMemoryBarrier", newArrayNode.emitMemoryBarrier());
897 args.addConst("threadRegister", registers.getThreadRegister());
898 args.addConst("maybeUnroll", length.isConstant());
899 args.addConst("typeContext", ProfileAllocations.getValue(localOptions) ? arrayType.toJavaName(false) : "");
900 args.addConst("bulkZeroingStride", tool.getLowerer().bulkZeroingStride());
901 args.addConst("counters", counters);
902 SnippetTemplate template = template(newArrayNode, args);
903 graph.getDebug().log("Lowering allocateArray in %s: node=%s, template=%s, arguments=%s", graph, newArrayNode, template, args);
904 template.instantiate(providers.getMetaAccess(), newArrayNode, DEFAULT_REPLACER, args);
905 }
906
907 public void lower(DynamicNewInstanceNode newInstanceNode, HotSpotRegistersProvider registers, LoweringTool tool) {
908 Arguments args = new Arguments(allocateInstanceDynamic, newInstanceNode.graph().getGuardsStage(), tool.getLoweringStage());
909 args.add("type", newInstanceNode.getInstanceType());
910 ValueNode classClass = newInstanceNode.getClassClass();
911 assert classClass != null;
912 args.add("classClass", classClass);
913 args.addConst("fillContents", newInstanceNode.fillContents());
914 args.addConst("emitMemoryBarrier", newInstanceNode.emitMemoryBarrier());
915 args.addConst("threadRegister", registers.getThreadRegister());
916 args.addConst("counters", counters);
917
918 SnippetTemplate template = template(newInstanceNode, args);
919 template.instantiate(providers.getMetaAccess(), newInstanceNode, DEFAULT_REPLACER, args);
920 }
921
922 public void lower(DynamicNewArrayNode newArrayNode, HotSpotRegistersProvider registers, LoweringTool tool) {
923 StructuredGraph graph = newArrayNode.graph();
924 Arguments args = new Arguments(allocateArrayDynamic, graph.getGuardsStage(), tool.getLoweringStage());
925 args.add("elementType", newArrayNode.getElementType());
926 ValueNode voidClass = newArrayNode.getVoidClass();
927 assert voidClass != null;
928 args.add("voidClass", voidClass);
929 ValueNode length = newArrayNode.length();
930 args.add("length", length.isAlive() ? length : graph.addOrUniqueWithInputs(length));
931 args.addConst("fillContents", newArrayNode.fillContents());
932 args.addConst("emitMemoryBarrier", newArrayNode.emitMemoryBarrier());
933 args.addConst("threadRegister", registers.getThreadRegister());
934 /*
935 * We use Kind.Illegal as a marker value instead of null because constant snippet
936 * parameters cannot be null.
937 */
938 args.addConst("knownElementKind", newArrayNode.getKnownElementKind() == null ? JavaKind.Illegal : newArrayNode.getKnownElementKind());
939 if (newArrayNode.getKnownElementKind() != null) {
940 args.addConst("knownLayoutHelper", lookupArrayClass(tool, newArrayNode.getKnownElementKind()).layoutHelper());
941 } else {
942 args.addConst("knownLayoutHelper", 0);
943 }
944 args.addConst("bulkZeroingStride", tool.getLowerer().bulkZeroingStride());
945 args.add("prototypeMarkWord", lookupArrayClass(tool, JavaKind.Object).prototypeMarkWord());
946 args.addConst("counters", counters);
947 SnippetTemplate template = template(newArrayNode, args);
948 template.instantiate(providers.getMetaAccess(), newArrayNode, DEFAULT_REPLACER, args);
949 }
950
951 private static HotSpotResolvedObjectType lookupArrayClass(LoweringTool tool, JavaKind kind) {
952 return (HotSpotResolvedObjectType) tool.getMetaAccess().lookupJavaType(kind == JavaKind.Object ? Object.class : kind.toJavaClass()).getArrayClass();
953 }
954
955 public void lower(NewMultiArrayNode newmultiarrayNode, LoweringTool tool) {
956 StructuredGraph graph = newmultiarrayNode.graph();
957 OptionValues localOptions = graph.getOptions();
958 int rank = newmultiarrayNode.dimensionCount();
959 ValueNode[] dims = new ValueNode[rank];
960 for (int i = 0; i < newmultiarrayNode.dimensionCount(); i++) {
961 dims[i] = newmultiarrayNode.dimension(i);
962 }
963 HotSpotResolvedObjectType type = (HotSpotResolvedObjectType) newmultiarrayNode.type();
964 ConstantNode hub = ConstantNode.forConstant(KlassPointerStamp.klassNonNull(), type.klass(), providers.getMetaAccess(), graph);
965
966 SnippetInfo snippet = GeneratePIC.getValue(localOptions) ? newmultiarrayPIC : newmultiarray;
967 Arguments args = new Arguments(snippet, graph.getGuardsStage(), tool.getLoweringStage());
968 args.add("hub", hub);
969 args.addConst("rank", rank);
970 args.addVarargs("dimensions", int.class, StampFactory.forKind(JavaKind.Int), dims);
971 template(newmultiarrayNode, args).instantiate(providers.getMetaAccess(), newmultiarrayNode, DEFAULT_REPLACER, args);
972 }
973
974 private static long instanceSize(HotSpotResolvedObjectType type) {
975 long size = type.instanceSize();
976 assert size >= 0;
977 return size;
978 }
979
980 public void lower(VerifyHeapNode verifyHeapNode, HotSpotRegistersProvider registers, LoweringTool tool) {
981 if (config.cAssertions) {
982 Arguments args = new Arguments(verifyHeap, verifyHeapNode.graph().getGuardsStage(), tool.getLoweringStage());
983 args.addConst("threadRegister", registers.getThreadRegister());
984
985 SnippetTemplate template = template(verifyHeapNode, args);
986 template.instantiate(providers.getMetaAccess(), verifyHeapNode, DEFAULT_REPLACER, args);
987 } else {
988 GraphUtil.removeFixedWithUnusedInputs(verifyHeapNode);
989 }
990 }
991
992 public void lower(KlassBeingInitializedCheckNode verifyHeapNode, HotSpotRegistersProvider registers, LoweringTool tool) {
993 Arguments args = new Arguments(threadBeingInitializedCheck, verifyHeapNode.graph().getGuardsStage(), tool.getLoweringStage());
994 args.addConst("threadRegister", registers.getThreadRegister());
995 args.add("klass", verifyHeapNode.getKlass());
996
997 SnippetTemplate template = template(verifyHeapNode, args);
998 template.instantiate(providers.getMetaAccess(), verifyHeapNode, DEFAULT_REPLACER, args);
999 }
1000 }
1001 }