1 /*
2 * Copyright (c) 2012, 2016, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23 package org.graalvm.compiler.hotspot.replacements;
24
25 import static org.graalvm.compiler.core.common.GraalOptions.GeneratePIC;
26 import static org.graalvm.compiler.core.common.GraalOptions.SnippetCounters;
27 import static org.graalvm.compiler.core.common.calc.UnsignedMath.belowThan;
28 import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfig.INJECTED_VMCONFIG;
29 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.CLASS_ARRAY_KLASS_LOCATION;
30 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.HUB_WRITE_LOCATION;
31 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.MARK_WORD_LOCATION;
32 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.PROTOTYPE_MARK_WORD_LOCATION;
33 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_END_LOCATION;
34 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_TOP_LOCATION;
35 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayKlassOffset;
36 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayLengthOffset;
37 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.config;
38 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.initializeObjectHeader;
39 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.instanceHeaderSize;
40 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.isInstanceKlassFullyInitialized;
41 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperHeaderSizeMask;
42 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperHeaderSizeShift;
43 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperLog2ElementSizeMask;
44 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperLog2ElementSizeShift;
45 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.loadKlassFromObject;
46 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.prototypeMarkWordOffset;
47 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readLayoutHelper;
48 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readTlabEnd;
49 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readTlabTop;
50 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.registerAsWord;
51 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.useBiasedLocking;
52 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.useTLAB;
53 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.verifyOop;
54 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.wordSize;
55 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.writeTlabTop;
56 import static org.graalvm.compiler.nodes.PiArrayNode.piArrayCast;
57 import static org.graalvm.compiler.nodes.PiNode.piCast;
58 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.FAST_PATH_PROBABILITY;
59 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.FREQUENT_PROBABILITY;
60 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.SLOW_PATH_PROBABILITY;
61 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.probability;
62 import static org.graalvm.compiler.replacements.ReplacementsUtil.REPLACEMENTS_ASSERTIONS_ENABLED;
63 import static org.graalvm.compiler.replacements.ReplacementsUtil.staticAssert;
64 import static org.graalvm.compiler.replacements.SnippetTemplate.DEFAULT_REPLACER;
65 import static org.graalvm.compiler.replacements.nodes.CStringConstant.cstring;
66 import static org.graalvm.compiler.replacements.nodes.ExplodeLoopNode.explodeLoop;
67 import static jdk.vm.ci.hotspot.HotSpotJVMCIRuntimeProvider.getArrayBaseOffset;
68 import static jdk.vm.ci.hotspot.HotSpotMetaAccessProvider.computeArrayAllocationSize;
69
70 import org.graalvm.compiler.api.replacements.Fold;
71 import org.graalvm.compiler.api.replacements.Snippet;
72 import org.graalvm.compiler.api.replacements.Snippet.ConstantParameter;
73 import org.graalvm.compiler.api.replacements.Snippet.VarargsParameter;
74 import org.graalvm.compiler.core.common.LocationIdentity;
75 import org.graalvm.compiler.core.common.spi.ForeignCallDescriptor;
76 import org.graalvm.compiler.core.common.type.StampFactory;
77 import org.graalvm.compiler.debug.Debug;
78 import org.graalvm.compiler.debug.GraalError;
79 import org.graalvm.compiler.graph.Node.ConstantNodeParameter;
80 import org.graalvm.compiler.graph.Node.NodeIntrinsic;
81 import org.graalvm.compiler.hotspot.GraalHotSpotVMConfig;
82 import org.graalvm.compiler.hotspot.HotSpotBackend;
83 import org.graalvm.compiler.hotspot.meta.HotSpotProviders;
84 import org.graalvm.compiler.hotspot.meta.HotSpotRegistersProvider;
85 import org.graalvm.compiler.hotspot.nodes.DimensionsNode;
86 import org.graalvm.compiler.hotspot.nodes.aot.LoadConstantIndirectlyFixedNode;
87 import org.graalvm.compiler.hotspot.nodes.aot.LoadConstantIndirectlyNode;
88 import org.graalvm.compiler.hotspot.nodes.type.KlassPointerStamp;
89 import org.graalvm.compiler.hotspot.replacements.aot.ResolveConstantSnippets;
90 import org.graalvm.compiler.hotspot.word.KlassPointer;
91 import org.graalvm.compiler.nodes.ConstantNode;
92 import org.graalvm.compiler.nodes.DeoptimizeNode;
93 import org.graalvm.compiler.nodes.NamedLocationIdentity;
94 import org.graalvm.compiler.nodes.PrefetchAllocateNode;
95 import org.graalvm.compiler.nodes.StructuredGraph;
96 import org.graalvm.compiler.nodes.ValueNode;
97 import org.graalvm.compiler.nodes.debug.DynamicCounterNode;
98 import org.graalvm.compiler.nodes.debug.VerifyHeapNode;
99 import org.graalvm.compiler.nodes.extended.BranchProbabilityNode;
100 import org.graalvm.compiler.nodes.extended.ForeignCallNode;
101 import org.graalvm.compiler.nodes.extended.MembarNode;
102 import org.graalvm.compiler.nodes.java.DynamicNewArrayNode;
103 import org.graalvm.compiler.nodes.java.DynamicNewInstanceNode;
104 import org.graalvm.compiler.nodes.java.NewArrayNode;
105 import org.graalvm.compiler.nodes.java.NewInstanceNode;
106 import org.graalvm.compiler.nodes.java.NewMultiArrayNode;
107 import org.graalvm.compiler.nodes.memory.address.OffsetAddressNode;
108 import org.graalvm.compiler.nodes.spi.LoweringTool;
109 import org.graalvm.compiler.nodes.util.GraphUtil;
110 import org.graalvm.compiler.replacements.ReplacementsUtil;
111 import org.graalvm.compiler.replacements.SnippetCounter;
112 import org.graalvm.compiler.replacements.SnippetTemplate;
113 import org.graalvm.compiler.replacements.SnippetTemplate.AbstractTemplates;
114 import org.graalvm.compiler.replacements.SnippetTemplate.Arguments;
115 import org.graalvm.compiler.replacements.SnippetTemplate.SnippetInfo;
116 import org.graalvm.compiler.replacements.Snippets;
117 import org.graalvm.compiler.replacements.nodes.ExplodeLoopNode;
118 import org.graalvm.compiler.word.Word;
119
120 import jdk.vm.ci.code.CodeUtil;
121 import jdk.vm.ci.code.MemoryBarriers;
122 import jdk.vm.ci.code.Register;
123 import jdk.vm.ci.code.TargetDescription;
124 import jdk.vm.ci.hotspot.HotSpotJVMCIRuntimeProvider;
125 import jdk.vm.ci.hotspot.HotSpotResolvedObjectType;
126 import jdk.vm.ci.meta.DeoptimizationAction;
127 import jdk.vm.ci.meta.DeoptimizationReason;
128 import jdk.vm.ci.meta.JavaKind;
129 import jdk.vm.ci.meta.ResolvedJavaType;
130
131 /**
132 * Snippets used for implementing NEW, ANEWARRAY and NEWARRAY.
133 */
134 public class NewObjectSnippets implements Snippets {
135
136 public static final LocationIdentity INIT_LOCATION = NamedLocationIdentity.mutable("Initialization");
137
138 enum ProfileContext {
139 AllocatingMethod,
140 InstanceOrArray,
141 AllocatedType,
142 AllocatedTypesInMethod,
143 Total
144 }
145
146 @Fold
147 static String createName(String path, String typeContext) {
148 switch (HotspotSnippetsOptions.ProfileAllocationsContext.getValue()) {
149 case AllocatingMethod:
150 return "";
151 case InstanceOrArray:
152 return path;
153 case AllocatedType:
154 case AllocatedTypesInMethod:
155 return typeContext;
156 case Total:
157 return "bytes";
158 default:
159 throw GraalError.shouldNotReachHere();
160 }
161 }
162
163 @Fold
164 static boolean doProfile() {
165 return HotspotSnippetsOptions.ProfileAllocations.getValue();
166 }
167
168 @Fold
169 static boolean withContext() {
170 ProfileContext context = HotspotSnippetsOptions.ProfileAllocationsContext.getValue();
171 return context == ProfileContext.AllocatingMethod || context == ProfileContext.AllocatedTypesInMethod;
172 }
173
174 protected static void profileAllocation(String path, long size, String typeContext) {
175 if (doProfile()) {
176 String name = createName(path, typeContext);
177
178 boolean context = withContext();
179 DynamicCounterNode.counter(name, "number of bytes allocated", size, context);
180 DynamicCounterNode.counter(name, "number of allocations", 1, context);
181 }
182 }
183
184 public static void emitPrefetchAllocate(Word address, boolean isArray) {
185 GraalHotSpotVMConfig config = config(INJECTED_VMCONFIG);
186 if (config.allocatePrefetchStyle > 0) {
187 // Insert a prefetch for each allocation only on the fast-path
188 // Generate several prefetch instructions.
189 int lines = isArray ? config.allocatePrefetchLines : config.allocateInstancePrefetchLines;
190 int stepSize = config.allocatePrefetchStepSize;
191 int distance = config.allocatePrefetchDistance;
192 ExplodeLoopNode.explodeLoop();
193 for (int i = 0; i < lines; i++) {
194 PrefetchAllocateNode.prefetch(OffsetAddressNode.address(address, distance));
195 distance += stepSize;
196 }
197 }
198 }
199
200 @Snippet
201 public static Object allocateInstance(@ConstantParameter int size, KlassPointer hub, Word prototypeMarkWord, @ConstantParameter boolean fillContents, @ConstantParameter Register threadRegister,
202 @ConstantParameter boolean constantSize, @ConstantParameter String typeContext) {
203 Object result;
204 Word thread = registerAsWord(threadRegister);
205 Word top = readTlabTop(thread);
206 Word end = readTlabEnd(thread);
207 Word newTop = top.add(size);
208 if (useTLAB(INJECTED_VMCONFIG) && probability(FAST_PATH_PROBABILITY, newTop.belowOrEqual(end))) {
209 writeTlabTop(thread, newTop);
210 emitPrefetchAllocate(newTop, false);
211 result = formatObject(hub, size, top, prototypeMarkWord, fillContents, constantSize, true);
212 } else {
213 new_stub.inc();
214 result = newInstance(HotSpotBackend.NEW_INSTANCE, hub);
215 }
216 profileAllocation("instance", size, typeContext);
217 return piCast(verifyOop(result), StampFactory.forNodeIntrinsic());
218 }
219
220 @NodeIntrinsic(value = ForeignCallNode.class, returnStampIsNonNull = true)
221 public static native Object newInstance(@ConstantNodeParameter ForeignCallDescriptor descriptor, KlassPointer hub);
222
223 @Snippet
224 public static Object allocateInstancePIC(@ConstantParameter int size, KlassPointer hub, Word prototypeMarkWord, @ConstantParameter boolean fillContents,
225 @ConstantParameter Register threadRegister,
226 @ConstantParameter boolean constantSize, @ConstantParameter String typeContext) {
227 // Klass must be initialized by the time the first instance is allocated, therefore we can
228 // just load it from the corresponding cell and avoid the resolution check. We have to use a
229 // fixed load though, to prevent it from floating above the initialization.
230 KlassPointer picHub = LoadConstantIndirectlyFixedNode.loadKlass(hub);
231 return allocateInstance(size, picHub, prototypeMarkWord, fillContents, threadRegister, constantSize, typeContext);
232 }
233
234 @Snippet
235 public static Object allocateInstanceDynamic(Class<?> type, Class<?> classClass, @ConstantParameter boolean fillContents, @ConstantParameter Register threadRegister) {
236 if (probability(SLOW_PATH_PROBABILITY, type == null || DynamicNewInstanceNode.throwsInstantiationException(type, classClass))) {
237 DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
238 }
239
240 KlassPointer hub = ClassGetHubNode.readClass(type);
241 if (probability(FAST_PATH_PROBABILITY, !hub.isNull())) {
242 if (probability(FAST_PATH_PROBABILITY, isInstanceKlassFullyInitialized(hub))) {
243 int layoutHelper = readLayoutHelper(hub);
244 /*
245 * src/share/vm/oops/klass.hpp: For instances, layout helper is a positive number,
246 * the instance size. This size is already passed through align_object_size and
247 * scaled to bytes. The low order bit is set if instances of this class cannot be
248 * allocated using the fastpath.
249 */
250 if (probability(FAST_PATH_PROBABILITY, (layoutHelper & 1) == 0)) {
251 Word prototypeMarkWord = hub.readWord(prototypeMarkWordOffset(INJECTED_VMCONFIG), PROTOTYPE_MARK_WORD_LOCATION);
252 /*
253 * FIXME(je,ds): we should actually pass typeContext instead of "" but late
254 * binding of parameters is not yet supported by the GraphBuilderPlugin system.
255 */
256 return allocateInstance(layoutHelper, hub, prototypeMarkWord, fillContents, threadRegister, false, "");
257 }
258 }
259 }
260 return dynamicNewInstanceStub(type);
261 }
262
263 /**
264 * Maximum array length for which fast path allocation is used.
265 */
266 public static final int MAX_ARRAY_FAST_PATH_ALLOCATION_LENGTH = 0x00FFFFFF;
267
268 @Snippet
269 public static Object allocatePrimitiveArrayPIC(KlassPointer hub, int length, Word prototypeMarkWord, @ConstantParameter int headerSize, @ConstantParameter int log2ElementSize,
270 @ConstantParameter boolean fillContents, @ConstantParameter Register threadRegister, @ConstantParameter boolean maybeUnroll, @ConstantParameter String typeContext) {
271 KlassPointer picHub = LoadConstantIndirectlyNode.loadKlass(hub);
272 return allocateArrayImpl(picHub, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents, threadRegister, maybeUnroll, typeContext, false);
273 }
274
275 @Snippet
276 public static Object allocateArrayPIC(KlassPointer hub, int length, Word prototypeMarkWord, @ConstantParameter int headerSize, @ConstantParameter int log2ElementSize,
277 @ConstantParameter boolean fillContents, @ConstantParameter Register threadRegister, @ConstantParameter boolean maybeUnroll, @ConstantParameter String typeContext) {
278 KlassPointer picHub = ResolveConstantSnippets.resolveKlassConstant(hub);
279 return allocateArrayImpl(picHub, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents, threadRegister, maybeUnroll, typeContext, false);
280 }
281
282 @Snippet
283 public static Object allocateArray(KlassPointer hub, int length, Word prototypeMarkWord, @ConstantParameter int headerSize, @ConstantParameter int log2ElementSize,
284 @ConstantParameter boolean fillContents, @ConstantParameter Register threadRegister, @ConstantParameter boolean maybeUnroll, @ConstantParameter String typeContext) {
285 Object result = allocateArrayImpl(hub, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents, threadRegister, maybeUnroll, typeContext, false);
286 return piArrayCast(verifyOop(result), length, StampFactory.forNodeIntrinsic());
287 }
288
289 private static Object allocateArrayImpl(KlassPointer hub, int length, Word prototypeMarkWord, int headerSize, int log2ElementSize, boolean fillContents,
290 @ConstantParameter Register threadRegister,
291 @ConstantParameter boolean maybeUnroll, String typeContext, boolean skipNegativeCheck) {
292 Object result;
293 int alignment = wordSize();
294 int allocationSize = computeArrayAllocationSize(length, alignment, headerSize, log2ElementSize);
295 Word thread = registerAsWord(threadRegister);
296 Word top = readTlabTop(thread);
297 Word end = readTlabEnd(thread);
298 Word newTop = top.add(allocationSize);
299 if (probability(FREQUENT_PROBABILITY, skipNegativeCheck || belowThan(length, MAX_ARRAY_FAST_PATH_ALLOCATION_LENGTH)) && useTLAB(INJECTED_VMCONFIG) &&
300 probability(FAST_PATH_PROBABILITY, newTop.belowOrEqual(end))) {
301 writeTlabTop(thread, newTop);
302 emitPrefetchAllocate(newTop, true);
303 newarray_loopInit.inc();
304 result = formatArray(hub, allocationSize, length, headerSize, top, prototypeMarkWord, fillContents, maybeUnroll, true);
305 } else {
306 result = newArray(HotSpotBackend.NEW_ARRAY, hub, length, fillContents);
307 }
308 profileAllocation("array", allocationSize, typeContext);
309 return result;
310 }
311
312 @NodeIntrinsic(value = ForeignCallNode.class, returnStampIsNonNull = true)
313 public static native Object newArray(@ConstantNodeParameter ForeignCallDescriptor descriptor, KlassPointer hub, int length, boolean fillContents);
314
315 public static final ForeignCallDescriptor DYNAMIC_NEW_ARRAY = new ForeignCallDescriptor("dynamic_new_array", Object.class, Class.class, int.class);
316 public static final ForeignCallDescriptor DYNAMIC_NEW_INSTANCE = new ForeignCallDescriptor("dynamic_new_instance", Object.class, Class.class);
317
318 @NodeIntrinsic(value = ForeignCallNode.class, returnStampIsNonNull = true)
319 public static native Object dynamicNewArrayStub(@ConstantNodeParameter ForeignCallDescriptor descriptor, Class<?> elementType, int length);
320
321 public static Object dynamicNewInstanceStub(Class<?> elementType) {
322 return dynamicNewInstanceStubCall(DYNAMIC_NEW_INSTANCE, elementType);
323 }
324
325 @NodeIntrinsic(value = ForeignCallNode.class, returnStampIsNonNull = true)
326 public static native Object dynamicNewInstanceStubCall(@ConstantNodeParameter ForeignCallDescriptor descriptor, Class<?> elementType);
327
328 @Snippet
329 public static Object allocateArrayDynamic(Class<?> elementType, Class<?> voidClass, int length, @ConstantParameter boolean fillContents, @ConstantParameter Register threadRegister,
330 @ConstantParameter JavaKind knownElementKind, @ConstantParameter int knownLayoutHelper, Word prototypeMarkWord) {
331 Object result = allocateArrayDynamicImpl(elementType, voidClass, length, fillContents, threadRegister, knownElementKind, knownLayoutHelper, prototypeMarkWord);
332 return result;
333 }
334
335 private static Object allocateArrayDynamicImpl(Class<?> elementType, Class<?> voidClass, int length, boolean fillContents, Register threadRegister, JavaKind knownElementKind,
336 int knownLayoutHelper, Word prototypeMarkWord) {
337 /*
338 * We only need the dynamic check for void when we have no static information from
339 * knownElementKind.
340 */
341 staticAssert(knownElementKind != JavaKind.Void, "unsupported knownElementKind");
342 if (knownElementKind == JavaKind.Illegal && probability(SLOW_PATH_PROBABILITY, elementType == null || DynamicNewArrayNode.throwsIllegalArgumentException(elementType, voidClass))) {
343 DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
344 }
345
346 KlassPointer klass = loadKlassFromObject(elementType, arrayKlassOffset(INJECTED_VMCONFIG), CLASS_ARRAY_KLASS_LOCATION);
347 if (probability(BranchProbabilityNode.NOT_FREQUENT_PROBABILITY, klass.isNull() || length < 0)) {
348 DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
349 }
350 int layoutHelper = knownElementKind != JavaKind.Illegal ? knownLayoutHelper : readLayoutHelper(klass);
351 //@formatter:off
352 // from src/share/vm/oops/klass.hpp:
353 //
354 // For arrays, layout helper is a negative number, containing four
355 // distinct bytes, as follows:
356 // MSB:[tag, hsz, ebt, log2(esz)]:LSB
357 // where:
358 // tag is 0x80 if the elements are oops, 0xC0 if non-oops
359 // hsz is array header size in bytes (i.e., offset of first element)
360 // ebt is the BasicType of the elements
361 // esz is the element size in bytes
362 //@formatter:on
363
364 int headerSize = (layoutHelper >> layoutHelperHeaderSizeShift(INJECTED_VMCONFIG)) & layoutHelperHeaderSizeMask(INJECTED_VMCONFIG);
365 int log2ElementSize = (layoutHelper >> layoutHelperLog2ElementSizeShift(INJECTED_VMCONFIG)) & layoutHelperLog2ElementSizeMask(INJECTED_VMCONFIG);
366
367 Object result = allocateArrayImpl(klass, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents, threadRegister, false, "dynamic type", true);
368 return piArrayCast(verifyOop(result), length, StampFactory.forNodeIntrinsic());
369 }
370
371 /**
372 * Calls the runtime stub for implementing MULTIANEWARRAY.
373 */
374 @Snippet
375 public static Object newmultiarray(KlassPointer hub, @ConstantParameter int rank, @VarargsParameter int[] dimensions) {
376 Word dims = DimensionsNode.allocaDimsArray(rank);
377 ExplodeLoopNode.explodeLoop();
378 for (int i = 0; i < rank; i++) {
379 dims.writeInt(i * 4, dimensions[i], INIT_LOCATION);
380 }
381 return newArrayCall(HotSpotBackend.NEW_MULTI_ARRAY, hub, rank, dims);
382 }
383
384 @Snippet
385 public static Object newmultiarrayPIC(KlassPointer hub, @ConstantParameter int rank, @VarargsParameter int[] dimensions) {
386 KlassPointer hubPIC = ResolveConstantSnippets.resolveKlassConstant(hub);
387 return newmultiarray(hubPIC, rank, dimensions);
388 }
389
390 @NodeIntrinsic(value = ForeignCallNode.class, returnStampIsNonNull = true)
391 public static native Object newArrayCall(@ConstantNodeParameter ForeignCallDescriptor descriptor, KlassPointer hub, int rank, Word dims);
392
393 /**
394 * Maximum number of long stores to emit when zeroing an object with a constant size. Larger
395 * objects have their bodies initialized in a loop.
396 */
397 private static final int MAX_UNROLLED_OBJECT_ZEROING_STORES = 8;
398
399 /**
400 * Zero uninitialized memory in a newly allocated object, unrolling as necessary and ensuring
401 * that stores are aligned.
402 *
403 * @param size number of bytes to zero
404 * @param memory beginning of object which is being zeroed
405 * @param constantSize is {@code size} known to be constant in the snippet
406 * @param startOffset offset to begin zeroing. May not be word aligned.
407 * @param manualUnroll maximally unroll zeroing
408 */
409 private static void zeroMemory(int size, Word memory, boolean constantSize, int startOffset, boolean manualUnroll, boolean useSnippetCounters) {
410 fillMemory(0, size, memory, constantSize, startOffset, manualUnroll, useSnippetCounters);
411 }
412
413 private static void fillMemory(long value, int size, Word memory, boolean constantSize, int startOffset, boolean manualUnroll, boolean useSnippetCounters) {
414 ReplacementsUtil.runtimeAssert((size & 0x7) == 0, "unaligned object size");
415 int offset = startOffset;
416 if ((offset & 0x7) != 0) {
417 memory.writeInt(offset, (int) value, INIT_LOCATION);
418 offset += 4;
419 }
420 ReplacementsUtil.runtimeAssert((offset & 0x7) == 0, "unaligned offset");
421 if (manualUnroll && ((size - offset) / 8) <= MAX_UNROLLED_OBJECT_ZEROING_STORES) {
422 ReplacementsUtil.staticAssert(!constantSize, "size shouldn't be constant at instantiation time");
423 // This case handles arrays of constant length. Instead of having a snippet variant for
424 // each length, generate a chain of stores of maximum length. Once it's inlined the
425 // break statement will trim excess stores.
426 if (useSnippetCounters) {
427 new_seqInit.inc();
428 }
429 explodeLoop();
430 for (int i = 0; i < MAX_UNROLLED_OBJECT_ZEROING_STORES; i++, offset += 8) {
431 if (offset == size) {
432 break;
433 }
434 memory.initializeLong(offset, value, INIT_LOCATION);
435 }
436 } else {
437 // Use Word instead of int to avoid extension to long in generated code
438 Word off = Word.signed(offset);
439 if (constantSize && ((size - offset) / 8) <= MAX_UNROLLED_OBJECT_ZEROING_STORES) {
440 if (useSnippetCounters) {
441 new_seqInit.inc();
442 }
443 explodeLoop();
444 } else {
445 if (useSnippetCounters) {
446 new_loopInit.inc();
447 }
448 }
449 for (; off.rawValue() < size; off = off.add(8)) {
450 memory.initializeLong(off, value, INIT_LOCATION);
451 }
452 }
453 }
454
455 /**
456 * Fill uninitialized memory with garbage value in a newly allocated object, unrolling as
457 * necessary and ensuring that stores are aligned.
458 *
459 * @param size number of bytes to zero
460 * @param memory beginning of object which is being zeroed
461 * @param constantSize is {@code size} known to be constant in the snippet
462 * @param startOffset offset to begin zeroing. May not be word aligned.
463 * @param manualUnroll maximally unroll zeroing
464 */
465 private static void fillWithGarbage(int size, Word memory, boolean constantSize, int startOffset, boolean manualUnroll, boolean useSnippetCounters) {
466 fillMemory(0xfefefefefefefefeL, size, memory, constantSize, startOffset, manualUnroll, useSnippetCounters);
467 }
468
469 /**
470 * Formats some allocated memory with an object header and zeroes out the rest. Disables asserts
471 * since they can't be compiled in stubs.
472 */
473 public static Object formatObjectForStub(KlassPointer hub, int size, Word memory, Word compileTimePrototypeMarkWord) {
474 return formatObject(hub, size, memory, compileTimePrototypeMarkWord, true, false, false);
475 }
476
477 /**
478 * Formats some allocated memory with an object header and zeroes out the rest.
479 */
480 protected static Object formatObject(KlassPointer hub, int size, Word memory, Word compileTimePrototypeMarkWord, boolean fillContents, boolean constantSize, boolean useSnippetCounters) {
481 Word prototypeMarkWord = useBiasedLocking(INJECTED_VMCONFIG) ? hub.readWord(prototypeMarkWordOffset(INJECTED_VMCONFIG), PROTOTYPE_MARK_WORD_LOCATION) : compileTimePrototypeMarkWord;
482 initializeObjectHeader(memory, prototypeMarkWord, hub);
483 if (fillContents) {
484 zeroMemory(size, memory, constantSize, instanceHeaderSize(INJECTED_VMCONFIG), false, useSnippetCounters);
485 } else if (REPLACEMENTS_ASSERTIONS_ENABLED) {
486 fillWithGarbage(size, memory, constantSize, instanceHeaderSize(INJECTED_VMCONFIG), false, useSnippetCounters);
487 }
488 MembarNode.memoryBarrier(MemoryBarriers.STORE_STORE, INIT_LOCATION);
489 return memory.toObject();
490 }
491
492 @Snippet
493 protected static void verifyHeap(@ConstantParameter Register threadRegister) {
494 Word thread = registerAsWord(threadRegister);
495 Word topValue = readTlabTop(thread);
496 if (!topValue.equal(Word.zero())) {
497 Word topValueContents = topValue.readWord(0, MARK_WORD_LOCATION);
498 if (topValueContents.equal(Word.zero())) {
499 AssertionSnippets.vmMessageC(AssertionSnippets.ASSERTION_VM_MESSAGE_C, true, cstring("overzeroing of TLAB detected"), 0L, 0L, 0L);
500 }
501 }
502 }
503
504 /**
505 * Formats some allocated memory with an object header and zeroes out the rest.
506 */
507 public static Object formatArray(KlassPointer hub, int allocationSize, int length, int headerSize, Word memory, Word prototypeMarkWord, boolean fillContents, boolean maybeUnroll,
508 boolean useSnippetCounters) {
509 memory.writeInt(arrayLengthOffset(INJECTED_VMCONFIG), length, INIT_LOCATION);
510 /*
511 * store hub last as the concurrent garbage collectors assume length is valid if hub field
512 * is not null
513 */
514 initializeObjectHeader(memory, prototypeMarkWord, hub);
515 if (fillContents) {
516 zeroMemory(allocationSize, memory, false, headerSize, maybeUnroll, useSnippetCounters);
517 } else if (REPLACEMENTS_ASSERTIONS_ENABLED) {
518 fillWithGarbage(allocationSize, memory, false, headerSize, maybeUnroll, useSnippetCounters);
519 }
520 MembarNode.memoryBarrier(MemoryBarriers.STORE_STORE, INIT_LOCATION);
521 return memory.toObject();
522 }
523
524 public static class Templates extends AbstractTemplates {
525
526 private final SnippetInfo allocateInstance = snippet(NewObjectSnippets.class, "allocateInstance", INIT_LOCATION, MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION, TLAB_END_LOCATION);
527 private final SnippetInfo allocateInstancePIC = snippet(NewObjectSnippets.class, "allocateInstancePIC", INIT_LOCATION, MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION,
528 TLAB_END_LOCATION);
529 private final SnippetInfo allocateArray = snippet(NewObjectSnippets.class, "allocateArray", INIT_LOCATION, MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION, TLAB_END_LOCATION);
530 private final SnippetInfo allocateArrayPIC = snippet(NewObjectSnippets.class, "allocateArrayPIC", INIT_LOCATION, MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION, TLAB_END_LOCATION);
531 private final SnippetInfo allocatePrimitiveArrayPIC = snippet(NewObjectSnippets.class, "allocatePrimitiveArrayPIC", INIT_LOCATION, MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION,
532 TLAB_END_LOCATION);
533 private final SnippetInfo allocateArrayDynamic = snippet(NewObjectSnippets.class, "allocateArrayDynamic", INIT_LOCATION, MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION,
534 TLAB_END_LOCATION);
535 private final SnippetInfo allocateInstanceDynamic = snippet(NewObjectSnippets.class, "allocateInstanceDynamic", INIT_LOCATION, MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION,
536 TLAB_END_LOCATION);
537 private final SnippetInfo newmultiarray = snippet(NewObjectSnippets.class, "newmultiarray", INIT_LOCATION, TLAB_TOP_LOCATION, TLAB_END_LOCATION);
538 private final SnippetInfo newmultiarrayPIC = snippet(NewObjectSnippets.class, "newmultiarrayPIC", INIT_LOCATION, TLAB_TOP_LOCATION, TLAB_END_LOCATION);
539 private final SnippetInfo verifyHeap = snippet(NewObjectSnippets.class, "verifyHeap");
540 private final GraalHotSpotVMConfig config;
541
542 public Templates(HotSpotProviders providers, TargetDescription target, GraalHotSpotVMConfig config) {
543 super(providers, providers.getSnippetReflection(), target);
544 this.config = config;
545 }
546
547 /**
548 * Lowers a {@link NewInstanceNode}.
549 */
550 public void lower(NewInstanceNode newInstanceNode, HotSpotRegistersProvider registers, LoweringTool tool) {
551 StructuredGraph graph = newInstanceNode.graph();
552 HotSpotResolvedObjectType type = (HotSpotResolvedObjectType) newInstanceNode.instanceClass();
553 assert !type.isArray();
554 ConstantNode hub = ConstantNode.forConstant(KlassPointerStamp.klassNonNull(), type.klass(), providers.getMetaAccess(), graph);
555 int size = instanceSize(type);
556
557 SnippetInfo snippet = GeneratePIC.getValue() ? allocateInstancePIC : allocateInstance;
558 Arguments args = new Arguments(snippet, graph.getGuardsStage(), tool.getLoweringStage());
559 args.addConst("size", size);
560 args.add("hub", hub);
561 args.add("prototypeMarkWord", type.prototypeMarkWord());
562 args.addConst("fillContents", newInstanceNode.fillContents());
563 args.addConst("threadRegister", registers.getThreadRegister());
564 args.addConst("constantSize", true);
565 args.addConst("typeContext", HotspotSnippetsOptions.ProfileAllocations.getValue() ? type.toJavaName(false) : "");
566
567 SnippetTemplate template = template(args);
568 Debug.log("Lowering allocateInstance in %s: node=%s, template=%s, arguments=%s", graph, newInstanceNode, template, args);
569 template.instantiate(providers.getMetaAccess(), newInstanceNode, DEFAULT_REPLACER, args);
570 }
571
572 /**
573 * Lowers a {@link NewArrayNode}.
574 */
575 public void lower(NewArrayNode newArrayNode, HotSpotRegistersProvider registers, LoweringTool tool) {
576 StructuredGraph graph = newArrayNode.graph();
577 ResolvedJavaType elementType = newArrayNode.elementType();
578 HotSpotResolvedObjectType arrayType = (HotSpotResolvedObjectType) elementType.getArrayClass();
579 JavaKind elementKind = elementType.getJavaKind();
580 ConstantNode hub = ConstantNode.forConstant(KlassPointerStamp.klassNonNull(), arrayType.klass(), providers.getMetaAccess(), graph);
581 final int headerSize = getArrayBaseOffset(elementKind);
582 int log2ElementSize = CodeUtil.log2(HotSpotJVMCIRuntimeProvider.getArrayIndexScale(elementKind));
583
584 SnippetInfo snippet;
585 if (GeneratePIC.getValue()) {
586 if (elementType.isPrimitive()) {
587 snippet = allocatePrimitiveArrayPIC;
588 } else {
589 snippet = allocateArrayPIC;
590 }
591 } else {
592 snippet = allocateArray;
593 }
594
595 Arguments args = new Arguments(snippet, graph.getGuardsStage(), tool.getLoweringStage());
596 args.add("hub", hub);
597 ValueNode length = newArrayNode.length();
598 args.add("length", length.isAlive() ? length : graph.addOrUniqueWithInputs(length));
599 assert arrayType.prototypeMarkWord() == lookupArrayClass(tool, JavaKind.Object).prototypeMarkWord() : "all array types are assumed to have the same prototypeMarkWord";
600 args.add("prototypeMarkWord", arrayType.prototypeMarkWord());
601 args.addConst("headerSize", headerSize);
602 args.addConst("log2ElementSize", log2ElementSize);
603 args.addConst("fillContents", newArrayNode.fillContents());
604 args.addConst("threadRegister", registers.getThreadRegister());
605 args.addConst("maybeUnroll", length.isConstant());
606 args.addConst("typeContext", HotspotSnippetsOptions.ProfileAllocations.getValue() ? arrayType.toJavaName(false) : "");
607 SnippetTemplate template = template(args);
608 Debug.log("Lowering allocateArray in %s: node=%s, template=%s, arguments=%s", graph, newArrayNode, template, args);
609 template.instantiate(providers.getMetaAccess(), newArrayNode, DEFAULT_REPLACER, args);
610 }
611
612 public void lower(DynamicNewInstanceNode newInstanceNode, HotSpotRegistersProvider registers, LoweringTool tool) {
613 Arguments args = new Arguments(allocateInstanceDynamic, newInstanceNode.graph().getGuardsStage(), tool.getLoweringStage());
614 args.add("type", newInstanceNode.getInstanceType());
615 ValueNode classClass = newInstanceNode.getClassClass();
616 assert classClass != null;
617 args.add("classClass", classClass);
618 args.addConst("fillContents", newInstanceNode.fillContents());
619 args.addConst("threadRegister", registers.getThreadRegister());
620
621 SnippetTemplate template = template(args);
622 template.instantiate(providers.getMetaAccess(), newInstanceNode, DEFAULT_REPLACER, args);
623 }
624
625 public void lower(DynamicNewArrayNode newArrayNode, HotSpotRegistersProvider registers, LoweringTool tool) {
626 StructuredGraph graph = newArrayNode.graph();
627 Arguments args = new Arguments(allocateArrayDynamic, newArrayNode.graph().getGuardsStage(), tool.getLoweringStage());
628 args.add("elementType", newArrayNode.getElementType());
629 ValueNode voidClass = newArrayNode.getVoidClass();
630 assert voidClass != null;
631 args.add("voidClass", voidClass);
632 ValueNode length = newArrayNode.length();
633 args.add("length", length.isAlive() ? length : graph.addOrUniqueWithInputs(length));
634 args.addConst("fillContents", newArrayNode.fillContents());
635 args.addConst("threadRegister", registers.getThreadRegister());
636 /*
637 * We use Kind.Illegal as a marker value instead of null because constant snippet
638 * parameters cannot be null.
639 */
640 args.addConst("knownElementKind", newArrayNode.getKnownElementKind() == null ? JavaKind.Illegal : newArrayNode.getKnownElementKind());
641 if (newArrayNode.getKnownElementKind() != null) {
642 args.addConst("knownLayoutHelper", lookupArrayClass(tool, newArrayNode.getKnownElementKind()).layoutHelper());
643 } else {
644 args.addConst("knownLayoutHelper", 0);
645 }
646 args.add("prototypeMarkWord", lookupArrayClass(tool, JavaKind.Object).prototypeMarkWord());
647 SnippetTemplate template = template(args);
648 template.instantiate(providers.getMetaAccess(), newArrayNode, DEFAULT_REPLACER, args);
649 }
650
651 private static HotSpotResolvedObjectType lookupArrayClass(LoweringTool tool, JavaKind kind) {
652 return (HotSpotResolvedObjectType) tool.getMetaAccess().lookupJavaType(kind == JavaKind.Object ? Object.class : kind.toJavaClass()).getArrayClass();
653 }
654
655 public void lower(NewMultiArrayNode newmultiarrayNode, LoweringTool tool) {
656 StructuredGraph graph = newmultiarrayNode.graph();
657 int rank = newmultiarrayNode.dimensionCount();
658 ValueNode[] dims = new ValueNode[rank];
659 for (int i = 0; i < newmultiarrayNode.dimensionCount(); i++) {
660 dims[i] = newmultiarrayNode.dimension(i);
661 }
662 HotSpotResolvedObjectType type = (HotSpotResolvedObjectType) newmultiarrayNode.type();
663 ConstantNode hub = ConstantNode.forConstant(KlassPointerStamp.klassNonNull(), type.klass(), providers.getMetaAccess(), graph);
664
665 SnippetInfo snippet = GeneratePIC.getValue() ? newmultiarrayPIC : newmultiarray;
666 Arguments args = new Arguments(snippet, graph.getGuardsStage(), tool.getLoweringStage());
667 args.add("hub", hub);
668 args.addConst("rank", rank);
669 args.addVarargs("dimensions", int.class, StampFactory.forKind(JavaKind.Int), dims);
670 template(args).instantiate(providers.getMetaAccess(), newmultiarrayNode, DEFAULT_REPLACER, args);
671 }
672
673 private static int instanceSize(HotSpotResolvedObjectType type) {
674 int size = type.instanceSize();
675 assert size >= 0;
676 return size;
677 }
678
679 public void lower(VerifyHeapNode verifyHeapNode, HotSpotRegistersProvider registers, LoweringTool tool) {
680 if (config.cAssertions) {
681 Arguments args = new Arguments(verifyHeap, verifyHeapNode.graph().getGuardsStage(), tool.getLoweringStage());
682 args.addConst("threadRegister", registers.getThreadRegister());
683
684 SnippetTemplate template = template(args);
685 template.instantiate(providers.getMetaAccess(), verifyHeapNode, DEFAULT_REPLACER, args);
686 } else {
687 GraphUtil.removeFixedWithUnusedInputs(verifyHeapNode);
688 }
689 }
690 }
691
692 private static final SnippetCounter.Group countersNew = SnippetCounters.getValue() ? new SnippetCounter.Group("NewInstance") : null;
693 private static final SnippetCounter new_seqInit = new SnippetCounter(countersNew, "tlabSeqInit", "TLAB alloc with unrolled zeroing");
694 private static final SnippetCounter new_loopInit = new SnippetCounter(countersNew, "tlabLoopInit", "TLAB alloc with zeroing in a loop");
695 private static final SnippetCounter new_stub = new SnippetCounter(countersNew, "stub", "alloc and zeroing via stub");
696
697 private static final SnippetCounter.Group countersNewArray = SnippetCounters.getValue() ? new SnippetCounter.Group("NewArray") : null;
698 private static final SnippetCounter newarray_loopInit = new SnippetCounter(countersNewArray, "tlabLoopInit", "TLAB alloc with zeroing in a loop");
699 private static final SnippetCounter newarray_stub = new SnippetCounter(countersNewArray, "stub", "alloc and zeroing via stub");
700 }