1 /*
2 * Copyright (c) 2012, 2016, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23 package org.graalvm.compiler.hotspot.replacements;
24
25 import static jdk.vm.ci.hotspot.HotSpotJVMCIRuntimeProvider.getArrayBaseOffset;
26 import static org.graalvm.compiler.core.common.GraalOptions.GeneratePIC;
27 import static org.graalvm.compiler.core.common.calc.UnsignedMath.belowThan;
28 import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfig.INJECTED_VMCONFIG;
29 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.CLASS_ARRAY_KLASS_LOCATION;
30 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.HUB_WRITE_LOCATION;
31 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.MARK_WORD_LOCATION;
32 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.PROTOTYPE_MARK_WORD_LOCATION;
33 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_END_LOCATION;
34 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_TOP_LOCATION;
35 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayAllocationSize;
36 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayKlassOffset;
37 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayLengthOffset;
38 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.config;
39 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.initializeObjectHeader;
40 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.instanceHeaderSize;
41 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.isInstanceKlassFullyInitialized;
42 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperHeaderSizeMask;
43 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperHeaderSizeShift;
44 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperLog2ElementSizeMask;
45 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperLog2ElementSizeShift;
46 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.loadKlassFromObject;
47 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.prototypeMarkWordOffset;
48 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readLayoutHelper;
49 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readTlabEnd;
50 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readTlabTop;
51 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.registerAsWord;
52 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.useBiasedLocking;
53 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.useTLAB;
54 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.verifyOop;
55 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.writeTlabTop;
56 import static org.graalvm.compiler.hotspot.replacements.HotspotSnippetsOptions.ProfileAllocations;
57 import static org.graalvm.compiler.hotspot.replacements.HotspotSnippetsOptions.ProfileAllocationsContext;
58 import static org.graalvm.compiler.nodes.PiArrayNode.piArrayCastToSnippetReplaceeStamp;
59 import static org.graalvm.compiler.nodes.PiNode.piCastToSnippetReplaceeStamp;
60 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.FAST_PATH_PROBABILITY;
61 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.FREQUENT_PROBABILITY;
62 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.SLOW_PATH_PROBABILITY;
63 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.probability;
64 import static org.graalvm.compiler.replacements.ReplacementsUtil.REPLACEMENTS_ASSERTIONS_ENABLED;
65 import static org.graalvm.compiler.replacements.ReplacementsUtil.runtimeAssert;
66 import static org.graalvm.compiler.replacements.ReplacementsUtil.staticAssert;
67 import static org.graalvm.compiler.replacements.SnippetTemplate.DEFAULT_REPLACER;
68 import static org.graalvm.compiler.replacements.nodes.CStringConstant.cstring;
69 import static org.graalvm.compiler.replacements.nodes.ExplodeLoopNode.explodeLoop;
70
71 import org.graalvm.compiler.api.replacements.Fold;
72 import org.graalvm.compiler.api.replacements.Snippet;
73 import org.graalvm.compiler.api.replacements.Snippet.ConstantParameter;
74 import org.graalvm.compiler.api.replacements.Snippet.VarargsParameter;
75 import org.graalvm.compiler.core.common.spi.ForeignCallDescriptor;
76 import org.graalvm.compiler.core.common.type.StampFactory;
77 import org.graalvm.compiler.debug.DebugHandlersFactory;
78 import org.graalvm.compiler.debug.GraalError;
79 import org.graalvm.compiler.graph.Node.ConstantNodeParameter;
80 import org.graalvm.compiler.graph.Node.NodeIntrinsic;
81 import org.graalvm.compiler.hotspot.GraalHotSpotVMConfig;
82 import org.graalvm.compiler.hotspot.HotSpotBackend;
83 import org.graalvm.compiler.hotspot.meta.HotSpotProviders;
84 import org.graalvm.compiler.hotspot.meta.HotSpotRegistersProvider;
85 import org.graalvm.compiler.hotspot.nodes.DimensionsNode;
86 import org.graalvm.compiler.hotspot.nodes.aot.LoadConstantIndirectlyFixedNode;
87 import org.graalvm.compiler.hotspot.nodes.aot.LoadConstantIndirectlyNode;
88 import org.graalvm.compiler.hotspot.nodes.type.KlassPointerStamp;
89 import org.graalvm.compiler.hotspot.word.KlassPointer;
90 import org.graalvm.compiler.nodes.ConstantNode;
91 import org.graalvm.compiler.nodes.DeoptimizeNode;
92 import org.graalvm.compiler.nodes.PiNode;
93 import org.graalvm.compiler.nodes.PrefetchAllocateNode;
94 import org.graalvm.compiler.nodes.SnippetAnchorNode;
95 import org.graalvm.compiler.nodes.StructuredGraph;
96 import org.graalvm.compiler.nodes.ValueNode;
97 import org.graalvm.compiler.nodes.debug.DynamicCounterNode;
98 import org.graalvm.compiler.nodes.debug.VerifyHeapNode;
99 import org.graalvm.compiler.nodes.extended.ForeignCallNode;
100 import org.graalvm.compiler.nodes.extended.MembarNode;
101 import org.graalvm.compiler.nodes.java.DynamicNewArrayNode;
102 import org.graalvm.compiler.nodes.java.DynamicNewInstanceNode;
103 import org.graalvm.compiler.nodes.java.NewArrayNode;
104 import org.graalvm.compiler.nodes.java.NewInstanceNode;
105 import org.graalvm.compiler.nodes.java.NewMultiArrayNode;
106 import org.graalvm.compiler.nodes.memory.address.OffsetAddressNode;
107 import org.graalvm.compiler.nodes.spi.LoweringTool;
108 import org.graalvm.compiler.nodes.util.GraphUtil;
109 import org.graalvm.compiler.options.OptionValues;
110 import org.graalvm.compiler.replacements.ReplacementsUtil;
111 import org.graalvm.compiler.replacements.SnippetCounter;
112 import org.graalvm.compiler.replacements.SnippetCounter.Group;
113 import org.graalvm.compiler.replacements.SnippetTemplate;
114 import org.graalvm.compiler.replacements.SnippetTemplate.AbstractTemplates;
115 import org.graalvm.compiler.replacements.SnippetTemplate.Arguments;
116 import org.graalvm.compiler.replacements.SnippetTemplate.SnippetInfo;
117 import org.graalvm.compiler.replacements.Snippets;
118 import org.graalvm.compiler.replacements.nodes.ExplodeLoopNode;
119 import org.graalvm.compiler.word.Word;
120 import org.graalvm.word.LocationIdentity;
121 import org.graalvm.word.WordFactory;
122
123 import jdk.vm.ci.code.CodeUtil;
124 import jdk.vm.ci.code.MemoryBarriers;
125 import jdk.vm.ci.code.Register;
126 import jdk.vm.ci.code.TargetDescription;
127 import jdk.vm.ci.hotspot.HotSpotJVMCIRuntimeProvider;
128 import jdk.vm.ci.hotspot.HotSpotResolvedObjectType;
129 import jdk.vm.ci.meta.DeoptimizationAction;
130 import jdk.vm.ci.meta.DeoptimizationReason;
131 import jdk.vm.ci.meta.JavaKind;
132 import jdk.vm.ci.meta.ResolvedJavaType;
133
134 /**
135 * Snippets used for implementing NEW, ANEWARRAY and NEWARRAY.
136 */
137 public class NewObjectSnippets implements Snippets {
138
139 enum ProfileContext {
140 AllocatingMethod,
141 InstanceOrArray,
142 AllocatedType,
143 AllocatedTypesInMethod,
144 Total
145 }
146
147 @Fold
148 static String createName(String path, String typeContext, OptionValues options) {
149 switch (ProfileAllocationsContext.getValue(options)) {
150 case AllocatingMethod:
151 return "";
152 case InstanceOrArray:
153 return path;
154 case AllocatedType:
155 case AllocatedTypesInMethod:
156 return typeContext;
157 case Total:
158 return "bytes";
159 default:
160 throw GraalError.shouldNotReachHere();
161 }
162 }
163
164 @Fold
165 static boolean doProfile(OptionValues options) {
166 return ProfileAllocations.getValue(options);
167 }
168
169 @Fold
170 static boolean withContext(OptionValues options) {
171 ProfileContext context = ProfileAllocationsContext.getValue(options);
172 return context == ProfileContext.AllocatingMethod || context == ProfileContext.AllocatedTypesInMethod;
173 }
174
175 protected static void profileAllocation(String path, long size, String typeContext, OptionValues options) {
176 if (doProfile(options)) {
177 String name = createName(path, typeContext, options);
178
179 boolean context = withContext(options);
180 DynamicCounterNode.counter(name, "number of bytes allocated", size, context);
181 DynamicCounterNode.counter(name, "number of allocations", 1, context);
182 }
183 }
184
185 public static void emitPrefetchAllocate(Word address, boolean isArray) {
186 GraalHotSpotVMConfig config = config(INJECTED_VMCONFIG);
187 if (config.allocatePrefetchStyle > 0) {
188 // Insert a prefetch for each allocation only on the fast-path
189 // Generate several prefetch instructions.
190 int lines = isArray ? config.allocatePrefetchLines : config.allocateInstancePrefetchLines;
191 int stepSize = config.allocatePrefetchStepSize;
192 int distance = config.allocatePrefetchDistance;
193 ExplodeLoopNode.explodeLoop();
194 for (int i = 0; i < lines; i++) {
195 PrefetchAllocateNode.prefetch(OffsetAddressNode.address(address, distance));
196 distance += stepSize;
197 }
198 }
199 }
200
201 @Snippet
202 public static Object allocateInstance(@ConstantParameter int size, KlassPointer hub, Word prototypeMarkWord, @ConstantParameter boolean fillContents,
203 @ConstantParameter Register threadRegister, @ConstantParameter boolean constantSize, @ConstantParameter String typeContext, @ConstantParameter OptionValues options,
204 @ConstantParameter Counters counters) {
205 return piCastToSnippetReplaceeStamp(allocateInstanceHelper(size, hub, prototypeMarkWord, fillContents, threadRegister, constantSize, typeContext, options, counters));
206 }
207
208 public static Object allocateInstanceHelper(int size, KlassPointer hub, Word prototypeMarkWord, boolean fillContents,
209 Register threadRegister, boolean constantSize, String typeContext, OptionValues options, Counters counters) {
210 Object result;
211 Word thread = registerAsWord(threadRegister);
212 Word top = readTlabTop(thread);
213 Word end = readTlabEnd(thread);
214 Word newTop = top.add(size);
215 if (useTLAB(INJECTED_VMCONFIG) && probability(FAST_PATH_PROBABILITY, newTop.belowOrEqual(end))) {
216 writeTlabTop(thread, newTop);
217 emitPrefetchAllocate(newTop, false);
218 result = formatObject(hub, size, top, prototypeMarkWord, fillContents, constantSize, counters);
219 } else {
220 if (counters != null && counters.stub != null) {
221 counters.stub.inc();
222 }
223 result = newInstance(HotSpotBackend.NEW_INSTANCE, hub);
224 }
225 profileAllocation("instance", size, typeContext, options);
226 return verifyOop(result);
227 }
228
229 @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = true)
230 public static native Object newInstance(@ConstantNodeParameter ForeignCallDescriptor descriptor, KlassPointer hub);
231
232 @Snippet
233 public static Object allocateInstancePIC(@ConstantParameter int size, KlassPointer hub, Word prototypeMarkWord, @ConstantParameter boolean fillContents,
234 @ConstantParameter Register threadRegister, @ConstantParameter boolean constantSize, @ConstantParameter String typeContext, @ConstantParameter OptionValues options,
235 @ConstantParameter Counters counters) {
236 // Klass must be initialized by the time the first instance is allocated, therefore we can
237 // just load it from the corresponding cell and avoid the resolution check. We have to use a
238 // fixed load though, to prevent it from floating above the initialization.
239 KlassPointer picHub = LoadConstantIndirectlyFixedNode.loadKlass(hub);
240 return piCastToSnippetReplaceeStamp(allocateInstanceHelper(size, picHub, prototypeMarkWord, fillContents, threadRegister, constantSize, typeContext, options, counters));
241 }
242
243 @Snippet
244 public static Object allocateInstanceDynamic(Class<?> type, Class<?> classClass, @ConstantParameter boolean fillContents, @ConstantParameter Register threadRegister,
245 @ConstantParameter OptionValues options, @ConstantParameter Counters counters) {
246 if (probability(SLOW_PATH_PROBABILITY, type == null)) {
247 DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
248 }
249 Class<?> nonNullType = PiNode.piCastNonNullClass(type, SnippetAnchorNode.anchor());
250
251 if (probability(SLOW_PATH_PROBABILITY, DynamicNewInstanceNode.throwsInstantiationException(type, classClass))) {
252 DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
253 }
254
255 return PiNode.piCastToSnippetReplaceeStamp(allocateInstanceDynamicHelper(type, fillContents, threadRegister, options, counters, nonNullType));
256 }
257
258 private static Object allocateInstanceDynamicHelper(Class<?> type, boolean fillContents, Register threadRegister, OptionValues options, Counters counters, Class<?> nonNullType) {
259 KlassPointer hub = ClassGetHubNode.readClass(nonNullType);
260 if (probability(FAST_PATH_PROBABILITY, !hub.isNull())) {
261 KlassPointer nonNullHub = ClassGetHubNode.piCastNonNull(hub, SnippetAnchorNode.anchor());
262
263 if (probability(FAST_PATH_PROBABILITY, isInstanceKlassFullyInitialized(nonNullHub))) {
264 int layoutHelper = readLayoutHelper(nonNullHub);
265 /*
266 * src/share/vm/oops/klass.hpp: For instances, layout helper is a positive number,
267 * the instance size. This size is already passed through align_object_size and
268 * scaled to bytes. The low order bit is set if instances of this class cannot be
269 * allocated using the fastpath.
270 */
271 if (probability(FAST_PATH_PROBABILITY, (layoutHelper & 1) == 0)) {
272 Word prototypeMarkWord = nonNullHub.readWord(prototypeMarkWordOffset(INJECTED_VMCONFIG), PROTOTYPE_MARK_WORD_LOCATION);
273 /*
274 * FIXME(je,ds): we should actually pass typeContext instead of "" but late
275 * binding of parameters is not yet supported by the GraphBuilderPlugin system.
276 */
277 return allocateInstanceHelper(layoutHelper, nonNullHub, prototypeMarkWord, fillContents, threadRegister, false, "", options, counters);
278 }
279 }
280 }
281 return dynamicNewInstanceStub(type);
282 }
283
284 /**
285 * Maximum array length for which fast path allocation is used.
286 */
287 public static final int MAX_ARRAY_FAST_PATH_ALLOCATION_LENGTH = 0x00FFFFFF;
288
289 @Snippet
290 public static Object allocatePrimitiveArrayPIC(KlassPointer hub, int length, Word prototypeMarkWord, @ConstantParameter int headerSize, @ConstantParameter int log2ElementSize,
291 @ConstantParameter boolean fillContents, @ConstantParameter Register threadRegister, @ConstantParameter boolean maybeUnroll, @ConstantParameter String typeContext,
292 @ConstantParameter OptionValues options, @ConstantParameter Counters counters) {
293 // Primitive array types are eagerly pre-resolved. We can use a floating load.
294 KlassPointer picHub = LoadConstantIndirectlyNode.loadKlass(hub);
295 return allocateArrayImpl(picHub, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents, threadRegister, maybeUnroll, typeContext, false, options, counters);
296 }
297
298 @Snippet
299 public static Object allocateArrayPIC(KlassPointer hub, int length, Word prototypeMarkWord, @ConstantParameter int headerSize, @ConstantParameter int log2ElementSize,
300 @ConstantParameter boolean fillContents, @ConstantParameter Register threadRegister, @ConstantParameter boolean maybeUnroll, @ConstantParameter String typeContext,
301 @ConstantParameter OptionValues options, @ConstantParameter Counters counters) {
302 // Array type would be resolved by dominating resolution.
303 KlassPointer picHub = LoadConstantIndirectlyFixedNode.loadKlass(hub);
304 return allocateArrayImpl(picHub, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents, threadRegister, maybeUnroll, typeContext, false, options, counters);
305 }
306
307 @Snippet
308 public static Object allocateArray(KlassPointer hub, int length, Word prototypeMarkWord, @ConstantParameter int headerSize, @ConstantParameter int log2ElementSize,
309 @ConstantParameter boolean fillContents, @ConstantParameter Register threadRegister, @ConstantParameter boolean maybeUnroll, @ConstantParameter String typeContext,
310 @ConstantParameter OptionValues options, @ConstantParameter Counters counters) {
311 Object result = allocateArrayImpl(hub, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents, threadRegister, maybeUnroll, typeContext, false, options, counters);
312 return piArrayCastToSnippetReplaceeStamp(verifyOop(result), length);
313 }
314
315 private static Object allocateArrayImpl(KlassPointer hub, int length, Word prototypeMarkWord, int headerSize, int log2ElementSize, boolean fillContents, Register threadRegister,
316 boolean maybeUnroll, String typeContext, boolean skipNegativeCheck, OptionValues options, Counters counters) {
317 Object result;
318 int allocationSize = arrayAllocationSize(length, headerSize, log2ElementSize);
319 Word thread = registerAsWord(threadRegister);
320 Word top = readTlabTop(thread);
321 Word end = readTlabEnd(thread);
322 Word newTop = top.add(allocationSize);
323 if (probability(FREQUENT_PROBABILITY, skipNegativeCheck || belowThan(length, MAX_ARRAY_FAST_PATH_ALLOCATION_LENGTH)) && useTLAB(INJECTED_VMCONFIG) &&
324 probability(FAST_PATH_PROBABILITY, newTop.belowOrEqual(end))) {
325 writeTlabTop(thread, newTop);
326 emitPrefetchAllocate(newTop, true);
327 if (counters != null && counters.arrayLoopInit != null) {
328 counters.arrayLoopInit.inc();
329 }
330 result = formatArray(hub, allocationSize, length, headerSize, top, prototypeMarkWord, fillContents, maybeUnroll, counters);
331 } else {
332 result = newArray(HotSpotBackend.NEW_ARRAY, hub, length, fillContents);
333 }
334 profileAllocation("array", allocationSize, typeContext, options);
335 return result;
336 }
337
338 @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = true)
339 public static native Object newArray(@ConstantNodeParameter ForeignCallDescriptor descriptor, KlassPointer hub, int length, boolean fillContents);
340
341 public static final ForeignCallDescriptor DYNAMIC_NEW_ARRAY = new ForeignCallDescriptor("dynamic_new_array", Object.class, Class.class, int.class);
342 public static final ForeignCallDescriptor DYNAMIC_NEW_INSTANCE = new ForeignCallDescriptor("dynamic_new_instance", Object.class, Class.class);
343
344 @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = true)
345 public static native Object dynamicNewArrayStub(@ConstantNodeParameter ForeignCallDescriptor descriptor, Class<?> elementType, int length);
346
347 public static Object dynamicNewInstanceStub(Class<?> elementType) {
348 return dynamicNewInstanceStubCall(DYNAMIC_NEW_INSTANCE, elementType);
349 }
350
351 @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = true)
352 public static native Object dynamicNewInstanceStubCall(@ConstantNodeParameter ForeignCallDescriptor descriptor, Class<?> elementType);
353
354 @Snippet
355 public static Object allocateArrayDynamic(Class<?> elementType, Class<?> voidClass, int length, @ConstantParameter boolean fillContents, @ConstantParameter Register threadRegister,
356 @ConstantParameter JavaKind knownElementKind, @ConstantParameter int knownLayoutHelper, Word prototypeMarkWord, @ConstantParameter OptionValues options,
357 @ConstantParameter Counters counters) {
358 Object result = allocateArrayDynamicImpl(elementType, voidClass, length, fillContents, threadRegister, knownElementKind, knownLayoutHelper, prototypeMarkWord, options, counters);
359 return result;
360 }
361
362 private static Object allocateArrayDynamicImpl(Class<?> elementType, Class<?> voidClass, int length, boolean fillContents, Register threadRegister, JavaKind knownElementKind,
363 int knownLayoutHelper, Word prototypeMarkWord, OptionValues options, Counters counters) {
364 /*
365 * We only need the dynamic check for void when we have no static information from
366 * knownElementKind.
367 */
368 staticAssert(knownElementKind != JavaKind.Void, "unsupported knownElementKind");
369 if (knownElementKind == JavaKind.Illegal && probability(SLOW_PATH_PROBABILITY, elementType == null || DynamicNewArrayNode.throwsIllegalArgumentException(elementType, voidClass))) {
370 DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
371 }
372
373 KlassPointer klass = loadKlassFromObject(elementType, arrayKlassOffset(INJECTED_VMCONFIG), CLASS_ARRAY_KLASS_LOCATION);
374 if (klass.isNull()) {
375 DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
376 }
377 KlassPointer nonNullKlass = ClassGetHubNode.piCastNonNull(klass, SnippetAnchorNode.anchor());
378
379 if (length < 0) {
380 DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
381 }
382 int layoutHelper;
383 if (knownElementKind == JavaKind.Illegal) {
384 layoutHelper = readLayoutHelper(nonNullKlass);
385 } else {
386 runtimeAssert(knownLayoutHelper == readLayoutHelper(nonNullKlass), "layout mismatch");
387 layoutHelper = knownLayoutHelper;
388 }
389 //@formatter:off
390 // from src/share/vm/oops/klass.hpp:
391 //
392 // For arrays, layout helper is a negative number, containing four
393 // distinct bytes, as follows:
394 // MSB:[tag, hsz, ebt, log2(esz)]:LSB
395 // where:
396 // tag is 0x80 if the elements are oops, 0xC0 if non-oops
397 // hsz is array header size in bytes (i.e., offset of first element)
398 // ebt is the BasicType of the elements
399 // esz is the element size in bytes
400 //@formatter:on
401
402 int headerSize = (layoutHelper >> layoutHelperHeaderSizeShift(INJECTED_VMCONFIG)) & layoutHelperHeaderSizeMask(INJECTED_VMCONFIG);
403 int log2ElementSize = (layoutHelper >> layoutHelperLog2ElementSizeShift(INJECTED_VMCONFIG)) & layoutHelperLog2ElementSizeMask(INJECTED_VMCONFIG);
404
405 Object result = allocateArrayImpl(nonNullKlass, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents, threadRegister, false, "dynamic type", true, options, counters);
406 return piArrayCastToSnippetReplaceeStamp(verifyOop(result), length);
407 }
408
409 /**
410 * Calls the runtime stub for implementing MULTIANEWARRAY.
411 */
412 @Snippet
413 public static Object newmultiarray(KlassPointer hub, @ConstantParameter int rank, @VarargsParameter int[] dimensions) {
414 Word dims = DimensionsNode.allocaDimsArray(rank);
415 ExplodeLoopNode.explodeLoop();
416 for (int i = 0; i < rank; i++) {
417 dims.writeInt(i * 4, dimensions[i], LocationIdentity.init());
418 }
419 return newArrayCall(HotSpotBackend.NEW_MULTI_ARRAY, hub, rank, dims);
420 }
421
422 @Snippet
423 public static Object newmultiarrayPIC(KlassPointer hub, @ConstantParameter int rank, @VarargsParameter int[] dimensions) {
424 // Array type would be resolved by dominating resolution.
425 KlassPointer picHub = LoadConstantIndirectlyFixedNode.loadKlass(hub);
426 return newmultiarray(picHub, rank, dimensions);
427 }
428
429 @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = true)
430 public static native Object newArrayCall(@ConstantNodeParameter ForeignCallDescriptor descriptor, KlassPointer hub, int rank, Word dims);
431
432 /**
433 * Maximum number of long stores to emit when zeroing an object with a constant size. Larger
434 * objects have their bodies initialized in a loop.
435 */
436 private static final int MAX_UNROLLED_OBJECT_ZEROING_STORES = 8;
437
438 /**
439 * Zero uninitialized memory in a newly allocated object, unrolling as necessary and ensuring
440 * that stores are aligned.
441 *
442 * @param size number of bytes to zero
443 * @param memory beginning of object which is being zeroed
444 * @param constantSize is {@code size} known to be constant in the snippet
445 * @param startOffset offset to begin zeroing. May not be word aligned.
446 * @param manualUnroll maximally unroll zeroing
447 */
448 private static void zeroMemory(int size, Word memory, boolean constantSize, int startOffset, boolean manualUnroll, Counters counters) {
449 fillMemory(0, size, memory, constantSize, startOffset, manualUnroll, counters);
450 }
451
452 private static void fillMemory(long value, int size, Word memory, boolean constantSize, int startOffset, boolean manualUnroll, Counters counters) {
453 ReplacementsUtil.runtimeAssert((size & 0x7) == 0, "unaligned object size");
454 int offset = startOffset;
455 if ((offset & 0x7) != 0) {
456 memory.writeInt(offset, (int) value, LocationIdentity.init());
457 offset += 4;
458 }
459 ReplacementsUtil.runtimeAssert((offset & 0x7) == 0, "unaligned offset");
460 if (manualUnroll && ((size - offset) / 8) <= MAX_UNROLLED_OBJECT_ZEROING_STORES) {
461 ReplacementsUtil.staticAssert(!constantSize, "size shouldn't be constant at instantiation time");
462 // This case handles arrays of constant length. Instead of having a snippet variant for
463 // each length, generate a chain of stores of maximum length. Once it's inlined the
464 // break statement will trim excess stores.
465 if (counters != null && counters.instanceSeqInit != null) {
466 counters.instanceSeqInit.inc();
467 }
468
469 explodeLoop();
470 for (int i = 0; i < MAX_UNROLLED_OBJECT_ZEROING_STORES; i++, offset += 8) {
471 if (offset == size) {
472 break;
473 }
474 memory.initializeLong(offset, value, LocationIdentity.init());
475 }
476 } else {
477 // Use Word instead of int to avoid extension to long in generated code
478 Word off = WordFactory.signed(offset);
479 if (constantSize && ((size - offset) / 8) <= MAX_UNROLLED_OBJECT_ZEROING_STORES) {
480 if (counters != null && counters.instanceSeqInit != null) {
481 counters.instanceSeqInit.inc();
482 }
483 explodeLoop();
484 } else {
485 if (counters != null && counters.instanceLoopInit != null) {
486 counters.instanceLoopInit.inc();
487 }
488 }
489 for (; off.rawValue() < size; off = off.add(8)) {
490 memory.initializeLong(off, value, LocationIdentity.init());
491 }
492 }
493 }
494
495 /**
496 * Fill uninitialized memory with garbage value in a newly allocated object, unrolling as
497 * necessary and ensuring that stores are aligned.
498 *
499 * @param size number of bytes to zero
500 * @param memory beginning of object which is being zeroed
501 * @param constantSize is {@code size} known to be constant in the snippet
502 * @param startOffset offset to begin zeroing. May not be word aligned.
503 * @param manualUnroll maximally unroll zeroing
504 */
505 private static void fillWithGarbage(int size, Word memory, boolean constantSize, int startOffset, boolean manualUnroll, Counters counters) {
506 fillMemory(0xfefefefefefefefeL, size, memory, constantSize, startOffset, manualUnroll, counters);
507 }
508
509 /**
510 * Formats some allocated memory with an object header and zeroes out the rest. Disables asserts
511 * since they can't be compiled in stubs.
512 */
513 public static Object formatObjectForStub(KlassPointer hub, int size, Word memory, Word compileTimePrototypeMarkWord) {
514 return formatObject(hub, size, memory, compileTimePrototypeMarkWord, true, false, null);
515 }
516
517 /**
518 * Formats some allocated memory with an object header and zeroes out the rest.
519 */
520 protected static Object formatObject(KlassPointer hub, int size, Word memory, Word compileTimePrototypeMarkWord, boolean fillContents, boolean constantSize, Counters counters) {
521 Word prototypeMarkWord = useBiasedLocking(INJECTED_VMCONFIG) ? hub.readWord(prototypeMarkWordOffset(INJECTED_VMCONFIG), PROTOTYPE_MARK_WORD_LOCATION) : compileTimePrototypeMarkWord;
522 initializeObjectHeader(memory, prototypeMarkWord, hub);
523 if (fillContents) {
524 zeroMemory(size, memory, constantSize, instanceHeaderSize(INJECTED_VMCONFIG), false, counters);
525 } else if (REPLACEMENTS_ASSERTIONS_ENABLED) {
526 fillWithGarbage(size, memory, constantSize, instanceHeaderSize(INJECTED_VMCONFIG), false, counters);
527 }
528 MembarNode.memoryBarrier(MemoryBarriers.STORE_STORE, LocationIdentity.init());
529 return memory.toObjectNonNull();
530 }
531
532 @Snippet
533 protected static void verifyHeap(@ConstantParameter Register threadRegister) {
534 Word thread = registerAsWord(threadRegister);
535 Word topValue = readTlabTop(thread);
536 if (!topValue.equal(WordFactory.zero())) {
537 Word topValueContents = topValue.readWord(0, MARK_WORD_LOCATION);
538 if (topValueContents.equal(WordFactory.zero())) {
539 AssertionSnippets.vmMessageC(AssertionSnippets.ASSERTION_VM_MESSAGE_C, true, cstring("overzeroing of TLAB detected"), 0L, 0L, 0L);
540 }
541 }
542 }
543
544 /**
545 * Formats some allocated memory with an object header and zeroes out the rest.
546 */
547 public static Object formatArray(KlassPointer hub, int allocationSize, int length, int headerSize, Word memory, Word prototypeMarkWord, boolean fillContents, boolean maybeUnroll,
548 Counters counters) {
549 memory.writeInt(arrayLengthOffset(INJECTED_VMCONFIG), length, LocationIdentity.init());
550 /*
551 * store hub last as the concurrent garbage collectors assume length is valid if hub field
552 * is not null
553 */
554 initializeObjectHeader(memory, prototypeMarkWord, hub);
555 if (fillContents) {
556 zeroMemory(allocationSize, memory, false, headerSize, maybeUnroll, counters);
557 } else if (REPLACEMENTS_ASSERTIONS_ENABLED) {
558 fillWithGarbage(allocationSize, memory, false, headerSize, maybeUnroll, counters);
559 }
560 MembarNode.memoryBarrier(MemoryBarriers.STORE_STORE, LocationIdentity.init());
561 return memory.toObjectNonNull();
562 }
563
564 static class Counters {
565 Counters(SnippetCounter.Group.Factory factory) {
566 Group newInstance = factory.createSnippetCounterGroup("NewInstance");
567 Group newArray = factory.createSnippetCounterGroup("NewArray");
568 instanceSeqInit = new SnippetCounter(newInstance, "tlabSeqInit", "TLAB alloc with unrolled zeroing");
569 instanceLoopInit = new SnippetCounter(newInstance, "tlabLoopInit", "TLAB alloc with zeroing in a loop");
570 arrayLoopInit = new SnippetCounter(newArray, "tlabLoopInit", "TLAB alloc with zeroing in a loop");
571 stub = new SnippetCounter(newInstance, "stub", "alloc and zeroing via stub");
572 }
573
574 final SnippetCounter instanceSeqInit;
575 final SnippetCounter instanceLoopInit;
576 final SnippetCounter arrayLoopInit;
577 final SnippetCounter stub;
578 }
579
580 public static class Templates extends AbstractTemplates {
581
582 private final SnippetInfo allocateInstance = snippet(NewObjectSnippets.class, "allocateInstance", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION, TLAB_END_LOCATION);
583 private final SnippetInfo allocateInstancePIC = snippet(NewObjectSnippets.class, "allocateInstancePIC", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION,
584 TLAB_END_LOCATION);
585 private final SnippetInfo allocateArray = snippet(NewObjectSnippets.class, "allocateArray", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION, TLAB_END_LOCATION);
586 private final SnippetInfo allocateArrayPIC = snippet(NewObjectSnippets.class, "allocateArrayPIC", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION, TLAB_END_LOCATION);
587 private final SnippetInfo allocatePrimitiveArrayPIC = snippet(NewObjectSnippets.class, "allocatePrimitiveArrayPIC", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION,
588 TLAB_END_LOCATION);
589 private final SnippetInfo allocateArrayDynamic = snippet(NewObjectSnippets.class, "allocateArrayDynamic", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION,
590 TLAB_END_LOCATION);
591 private final SnippetInfo allocateInstanceDynamic = snippet(NewObjectSnippets.class, "allocateInstanceDynamic", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION,
592 TLAB_END_LOCATION);
593 private final SnippetInfo newmultiarray = snippet(NewObjectSnippets.class, "newmultiarray", TLAB_TOP_LOCATION, TLAB_END_LOCATION);
594 private final SnippetInfo newmultiarrayPIC = snippet(NewObjectSnippets.class, "newmultiarrayPIC", TLAB_TOP_LOCATION, TLAB_END_LOCATION);
595 private final SnippetInfo verifyHeap = snippet(NewObjectSnippets.class, "verifyHeap");
596 private final GraalHotSpotVMConfig config;
597 private final Counters counters;
598
599 public Templates(OptionValues options, Iterable<DebugHandlersFactory> factories, SnippetCounter.Group.Factory factory, HotSpotProviders providers, TargetDescription target,
600 GraalHotSpotVMConfig config) {
601 super(options, factories, providers, providers.getSnippetReflection(), target);
602 this.config = config;
603 counters = new Counters(factory);
604 }
605
606 /**
607 * Lowers a {@link NewInstanceNode}.
608 */
609 public void lower(NewInstanceNode newInstanceNode, HotSpotRegistersProvider registers, LoweringTool tool) {
610 StructuredGraph graph = newInstanceNode.graph();
611 HotSpotResolvedObjectType type = (HotSpotResolvedObjectType) newInstanceNode.instanceClass();
612 assert !type.isArray();
613 ConstantNode hub = ConstantNode.forConstant(KlassPointerStamp.klassNonNull(), type.klass(), providers.getMetaAccess(), graph);
614 int size = instanceSize(type);
615
616 OptionValues localOptions = graph.getOptions();
617 SnippetInfo snippet = GeneratePIC.getValue(localOptions) ? allocateInstancePIC : allocateInstance;
618 Arguments args = new Arguments(snippet, graph.getGuardsStage(), tool.getLoweringStage());
619 args.addConst("size", size);
620 args.add("hub", hub);
621 args.add("prototypeMarkWord", type.prototypeMarkWord());
622 args.addConst("fillContents", newInstanceNode.fillContents());
623 args.addConst("threadRegister", registers.getThreadRegister());
624 args.addConst("constantSize", true);
625 args.addConst("typeContext", ProfileAllocations.getValue(localOptions) ? type.toJavaName(false) : "");
626 args.addConst("options", localOptions);
627 args.addConst("counters", counters);
628
629 SnippetTemplate template = template(newInstanceNode, args);
630 graph.getDebug().log("Lowering allocateInstance in %s: node=%s, template=%s, arguments=%s", graph, newInstanceNode, template, args);
631 template.instantiate(providers.getMetaAccess(), newInstanceNode, DEFAULT_REPLACER, args);
632 }
633
634 /**
635 * Lowers a {@link NewArrayNode}.
636 */
637 public void lower(NewArrayNode newArrayNode, HotSpotRegistersProvider registers, LoweringTool tool) {
638 StructuredGraph graph = newArrayNode.graph();
639 ResolvedJavaType elementType = newArrayNode.elementType();
640 HotSpotResolvedObjectType arrayType = (HotSpotResolvedObjectType) elementType.getArrayClass();
641 JavaKind elementKind = elementType.getJavaKind();
642 ConstantNode hub = ConstantNode.forConstant(KlassPointerStamp.klassNonNull(), arrayType.klass(), providers.getMetaAccess(), graph);
643 final int headerSize = getArrayBaseOffset(elementKind);
644 int log2ElementSize = CodeUtil.log2(HotSpotJVMCIRuntimeProvider.getArrayIndexScale(elementKind));
645
646 OptionValues localOptions = graph.getOptions();
647 SnippetInfo snippet;
648 if (GeneratePIC.getValue(localOptions)) {
649 if (elementType.isPrimitive()) {
650 snippet = allocatePrimitiveArrayPIC;
651 } else {
652 snippet = allocateArrayPIC;
653 }
654 } else {
655 snippet = allocateArray;
656 }
657
658 Arguments args = new Arguments(snippet, graph.getGuardsStage(), tool.getLoweringStage());
659 args.add("hub", hub);
660 ValueNode length = newArrayNode.length();
661 args.add("length", length.isAlive() ? length : graph.addOrUniqueWithInputs(length));
662 assert arrayType.prototypeMarkWord() == lookupArrayClass(tool, JavaKind.Object).prototypeMarkWord() : "all array types are assumed to have the same prototypeMarkWord";
663 args.add("prototypeMarkWord", arrayType.prototypeMarkWord());
664 args.addConst("headerSize", headerSize);
665 args.addConst("log2ElementSize", log2ElementSize);
666 args.addConst("fillContents", newArrayNode.fillContents());
667 args.addConst("threadRegister", registers.getThreadRegister());
668 args.addConst("maybeUnroll", length.isConstant());
669 args.addConst("typeContext", ProfileAllocations.getValue(localOptions) ? arrayType.toJavaName(false) : "");
670 args.addConst("options", localOptions);
671 args.addConst("counters", counters);
672 SnippetTemplate template = template(newArrayNode, args);
673 graph.getDebug().log("Lowering allocateArray in %s: node=%s, template=%s, arguments=%s", graph, newArrayNode, template, args);
674 template.instantiate(providers.getMetaAccess(), newArrayNode, DEFAULT_REPLACER, args);
675 }
676
677 public void lower(DynamicNewInstanceNode newInstanceNode, HotSpotRegistersProvider registers, LoweringTool tool) {
678 Arguments args = new Arguments(allocateInstanceDynamic, newInstanceNode.graph().getGuardsStage(), tool.getLoweringStage());
679 OptionValues localOptions = newInstanceNode.getOptions();
680 args.add("type", newInstanceNode.getInstanceType());
681 ValueNode classClass = newInstanceNode.getClassClass();
682 assert classClass != null;
683 args.add("classClass", classClass);
684 args.addConst("fillContents", newInstanceNode.fillContents());
685 args.addConst("threadRegister", registers.getThreadRegister());
686 args.addConst("options", localOptions);
687 args.addConst("counters", counters);
688
689 SnippetTemplate template = template(newInstanceNode, args);
690 template.instantiate(providers.getMetaAccess(), newInstanceNode, DEFAULT_REPLACER, args);
691 }
692
693 public void lower(DynamicNewArrayNode newArrayNode, HotSpotRegistersProvider registers, LoweringTool tool) {
694 StructuredGraph graph = newArrayNode.graph();
695 OptionValues localOptions = graph.getOptions();
696 Arguments args = new Arguments(allocateArrayDynamic, newArrayNode.graph().getGuardsStage(), tool.getLoweringStage());
697 args.add("elementType", newArrayNode.getElementType());
698 ValueNode voidClass = newArrayNode.getVoidClass();
699 assert voidClass != null;
700 args.add("voidClass", voidClass);
701 ValueNode length = newArrayNode.length();
702 args.add("length", length.isAlive() ? length : graph.addOrUniqueWithInputs(length));
703 args.addConst("fillContents", newArrayNode.fillContents());
704 args.addConst("threadRegister", registers.getThreadRegister());
705 /*
706 * We use Kind.Illegal as a marker value instead of null because constant snippet
707 * parameters cannot be null.
708 */
709 args.addConst("knownElementKind", newArrayNode.getKnownElementKind() == null ? JavaKind.Illegal : newArrayNode.getKnownElementKind());
710 if (newArrayNode.getKnownElementKind() != null) {
711 args.addConst("knownLayoutHelper", lookupArrayClass(tool, newArrayNode.getKnownElementKind()).layoutHelper());
712 } else {
713 args.addConst("knownLayoutHelper", 0);
714 }
715 args.add("prototypeMarkWord", lookupArrayClass(tool, JavaKind.Object).prototypeMarkWord());
716 args.addConst("options", localOptions);
717 args.addConst("counters", counters);
718 SnippetTemplate template = template(newArrayNode, args);
719 template.instantiate(providers.getMetaAccess(), newArrayNode, DEFAULT_REPLACER, args);
720 }
721
722 private static HotSpotResolvedObjectType lookupArrayClass(LoweringTool tool, JavaKind kind) {
723 return (HotSpotResolvedObjectType) tool.getMetaAccess().lookupJavaType(kind == JavaKind.Object ? Object.class : kind.toJavaClass()).getArrayClass();
724 }
725
726 public void lower(NewMultiArrayNode newmultiarrayNode, LoweringTool tool) {
727 StructuredGraph graph = newmultiarrayNode.graph();
728 OptionValues localOptions = graph.getOptions();
729 int rank = newmultiarrayNode.dimensionCount();
730 ValueNode[] dims = new ValueNode[rank];
731 for (int i = 0; i < newmultiarrayNode.dimensionCount(); i++) {
732 dims[i] = newmultiarrayNode.dimension(i);
733 }
734 HotSpotResolvedObjectType type = (HotSpotResolvedObjectType) newmultiarrayNode.type();
735 ConstantNode hub = ConstantNode.forConstant(KlassPointerStamp.klassNonNull(), type.klass(), providers.getMetaAccess(), graph);
736
737 SnippetInfo snippet = GeneratePIC.getValue(localOptions) ? newmultiarrayPIC : newmultiarray;
738 Arguments args = new Arguments(snippet, graph.getGuardsStage(), tool.getLoweringStage());
739 args.add("hub", hub);
740 args.addConst("rank", rank);
741 args.addVarargs("dimensions", int.class, StampFactory.forKind(JavaKind.Int), dims);
742 template(newmultiarrayNode, args).instantiate(providers.getMetaAccess(), newmultiarrayNode, DEFAULT_REPLACER, args);
743 }
744
745 private static int instanceSize(HotSpotResolvedObjectType type) {
746 int size = type.instanceSize();
747 assert size >= 0;
748 return size;
749 }
750
751 public void lower(VerifyHeapNode verifyHeapNode, HotSpotRegistersProvider registers, LoweringTool tool) {
752 if (config.cAssertions) {
753 Arguments args = new Arguments(verifyHeap, verifyHeapNode.graph().getGuardsStage(), tool.getLoweringStage());
754 args.addConst("threadRegister", registers.getThreadRegister());
755
756 SnippetTemplate template = template(verifyHeapNode, args);
757 template.instantiate(providers.getMetaAccess(), verifyHeapNode, DEFAULT_REPLACER, args);
758 } else {
759 GraphUtil.removeFixedWithUnusedInputs(verifyHeapNode);
760 }
761 }
762 }
763 }