11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23
24
25 package org.graalvm.compiler.hotspot.replacements;
26
27 import static jdk.vm.ci.meta.DeoptimizationAction.None;
28 import static jdk.vm.ci.meta.DeoptimizationReason.RuntimeConstraint;
29 import static org.graalvm.compiler.core.common.GraalOptions.GeneratePIC;
30 import static org.graalvm.compiler.core.common.calc.UnsignedMath.belowThan;
31 import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfigBase.INJECTED_VMCONFIG;
32 import static org.graalvm.compiler.hotspot.HotSpotBackend.NEW_ARRAY;
33 import static org.graalvm.compiler.hotspot.HotSpotBackend.NEW_ARRAY_OR_NULL;
34 import static org.graalvm.compiler.hotspot.HotSpotBackend.NEW_INSTANCE;
35 import static org.graalvm.compiler.hotspot.HotSpotBackend.NEW_INSTANCE_OR_NULL;
36 import static org.graalvm.compiler.hotspot.HotSpotBackend.NEW_MULTI_ARRAY;
37 import static org.graalvm.compiler.hotspot.HotSpotBackend.NEW_MULTI_ARRAY_OR_NULL;
38 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.CLASS_ARRAY_KLASS_LOCATION;
39 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.HUB_WRITE_LOCATION;
40 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.MARK_WORD_LOCATION;
41 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.PROTOTYPE_MARK_WORD_LOCATION;
42 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_END_LOCATION;
43 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_TOP_LOCATION;
44 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayAllocationSize;
45 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayKlassOffset;
46 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayLengthOffset;
47 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.config;
48 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.initializeObjectHeader;
49 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.instanceHeaderSize;
50 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.isInstanceKlassFullyInitialized;
51 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperHeaderSizeMask;
52 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperHeaderSizeShift;
53 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperLog2ElementSizeMask;
54 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperLog2ElementSizeShift;
55 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.loadKlassFromObject;
56 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.prototypeMarkWordOffset;
57 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readLayoutHelper;
58 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readTlabEnd;
59 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readTlabTop;
60 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.registerAsWord;
61 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.useBiasedLocking;
62 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.useTLAB;
63 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.verifyOop;
64 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.writeTlabTop;
65 import static org.graalvm.compiler.hotspot.replacements.HotspotSnippetsOptions.ProfileAllocations;
66 import static org.graalvm.compiler.hotspot.replacements.HotspotSnippetsOptions.ProfileAllocationsContext;
67 import static org.graalvm.compiler.nodes.PiArrayNode.piArrayCastToSnippetReplaceeStamp;
134 import jdk.vm.ci.code.Register;
135 import jdk.vm.ci.code.TargetDescription;
136 import jdk.vm.ci.hotspot.HotSpotResolvedObjectType;
137 import jdk.vm.ci.meta.JavaKind;
138 import jdk.vm.ci.meta.ResolvedJavaType;
139
140 /**
141 * Snippets used for implementing NEW, ANEWARRAY and NEWARRAY.
142 */
143 public class NewObjectSnippets implements Snippets {
144
145 enum ProfileContext {
146 AllocatingMethod,
147 InstanceOrArray,
148 AllocatedType,
149 AllocatedTypesInMethod,
150 Total
151 }
152
153 @Fold
154 static String createName(String path, String typeContext, OptionValues options) {
155 switch (ProfileAllocationsContext.getValue(options)) {
156 case AllocatingMethod:
157 return "";
158 case InstanceOrArray:
159 return path;
160 case AllocatedType:
161 case AllocatedTypesInMethod:
162 return typeContext;
163 case Total:
164 return "bytes";
165 default:
166 throw GraalError.shouldNotReachHere();
167 }
168 }
169
170 @Fold
171 static boolean doProfile(OptionValues options) {
172 return ProfileAllocations.getValue(options);
173 }
174
175 @Fold
176 static boolean withContext(OptionValues options) {
177 ProfileContext context = ProfileAllocationsContext.getValue(options);
178 return context == ProfileContext.AllocatingMethod || context == ProfileContext.AllocatedTypesInMethod;
179 }
180
181 protected static void profileAllocation(String path, long size, String typeContext, OptionValues options) {
182 if (doProfile(options)) {
183 String name = createName(path, typeContext, options);
184
185 boolean context = withContext(options);
186 DynamicCounterNode.counter("number of bytes allocated", name, size, context);
187 DynamicCounterNode.counter("number of allocations", name, 1, context);
188 }
189 }
190
191 public static void emitPrefetchAllocate(Word address, boolean isArray) {
192 GraalHotSpotVMConfig config = config(INJECTED_VMCONFIG);
193 if (config.allocatePrefetchStyle > 0) {
194 // Insert a prefetch for each allocation only on the fast-path
195 // Generate several prefetch instructions.
196 int lines = isArray ? config.allocatePrefetchLines : config.allocateInstancePrefetchLines;
197 int stepSize = config.allocatePrefetchStepSize;
198 int distance = config.allocatePrefetchDistance;
199 ExplodeLoopNode.explodeLoop();
200 for (int i = 0; i < lines; i++) {
201 PrefetchAllocateNode.prefetch(OffsetAddressNode.address(address, distance));
202 distance += stepSize;
203 }
204 }
205 }
206
207 @Snippet
208 public static Object allocateInstance(@ConstantParameter int size, KlassPointer hub, Word prototypeMarkWord, @ConstantParameter boolean fillContents,
209 @ConstantParameter Register threadRegister, @ConstantParameter boolean constantSize, @ConstantParameter String typeContext, @ConstantParameter OptionValues options,
210 @ConstantParameter Counters counters) {
211 return piCastToSnippetReplaceeStamp(allocateInstanceHelper(size, hub, prototypeMarkWord, fillContents, threadRegister, constantSize, typeContext, options, counters));
212 }
213
214 public static Object allocateInstanceHelper(int size, KlassPointer hub, Word prototypeMarkWord, boolean fillContents,
215 Register threadRegister, boolean constantSize, String typeContext, OptionValues options, Counters counters) {
216 Object result;
217 Word thread = registerAsWord(threadRegister);
218 Word top = readTlabTop(thread);
219 Word end = readTlabEnd(thread);
220 Word newTop = top.add(size);
221 if (useTLAB(INJECTED_VMCONFIG) && probability(FAST_PATH_PROBABILITY, newTop.belowOrEqual(end))) {
222 writeTlabTop(thread, newTop);
223 emitPrefetchAllocate(newTop, false);
224 result = formatObject(hub, size, top, prototypeMarkWord, fillContents, constantSize, counters);
225 } else {
226 if (counters != null && counters.stub != null) {
227 counters.stub.inc();
228 }
229 result = newInstanceStub(hub);
230 }
231 profileAllocation("instance", size, typeContext, options);
232 return verifyOop(result);
233 }
234
235 public static Object newInstanceStub(KlassPointer hub) {
236 if (useNullAllocationStubs(INJECTED_VMCONFIG)) {
237 return nonNullOrDeopt(newInstanceOrNull(NEW_INSTANCE_OR_NULL, hub));
238 } else {
239 return newInstance(NEW_INSTANCE, hub);
240 }
241 }
242
243 @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = true)
244 private static native Object newInstance(@ConstantNodeParameter ForeignCallDescriptor descriptor, KlassPointer hub);
245
246 @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = false)
247 private static native Object newInstanceOrNull(@ConstantNodeParameter ForeignCallDescriptor descriptor, KlassPointer hub);
248
249 @Snippet
250 public static Object allocateInstancePIC(@ConstantParameter int size, KlassPointer hub, Word prototypeMarkWord, @ConstantParameter boolean fillContents,
251 @ConstantParameter Register threadRegister, @ConstantParameter boolean constantSize, @ConstantParameter String typeContext, @ConstantParameter OptionValues options,
252 @ConstantParameter Counters counters) {
253 // Klass must be initialized by the time the first instance is allocated, therefore we can
254 // just load it from the corresponding cell and avoid the resolution check. We have to use a
255 // fixed load though, to prevent it from floating above the initialization.
256 KlassPointer picHub = LoadConstantIndirectlyFixedNode.loadKlass(hub);
257 return piCastToSnippetReplaceeStamp(allocateInstanceHelper(size, picHub, prototypeMarkWord, fillContents, threadRegister, constantSize, typeContext, options, counters));
258 }
259
260 @Snippet
261 public static Object allocateInstanceDynamic(Class<?> type, Class<?> classClass, @ConstantParameter boolean fillContents, @ConstantParameter Register threadRegister,
262 @ConstantParameter OptionValues options, @ConstantParameter Counters counters) {
263 if (probability(SLOW_PATH_PROBABILITY, type == null)) {
264 DeoptimizeNode.deopt(None, RuntimeConstraint);
265 }
266 Class<?> nonNullType = PiNode.piCastNonNullClass(type, SnippetAnchorNode.anchor());
267
268 if (probability(SLOW_PATH_PROBABILITY, DynamicNewInstanceNode.throwsInstantiationException(type, classClass))) {
269 DeoptimizeNode.deopt(None, RuntimeConstraint);
270 }
271
272 return PiNode.piCastToSnippetReplaceeStamp(allocateInstanceDynamicHelper(type, fillContents, threadRegister, options, counters, nonNullType));
273 }
274
275 private static Object allocateInstanceDynamicHelper(Class<?> type, boolean fillContents, Register threadRegister, OptionValues options, Counters counters, Class<?> nonNullType) {
276 KlassPointer hub = ClassGetHubNode.readClass(nonNullType);
277 if (probability(FAST_PATH_PROBABILITY, !hub.isNull())) {
278 KlassPointer nonNullHub = ClassGetHubNode.piCastNonNull(hub, SnippetAnchorNode.anchor());
279
280 if (probability(FAST_PATH_PROBABILITY, isInstanceKlassFullyInitialized(nonNullHub))) {
281 int layoutHelper = readLayoutHelper(nonNullHub);
282 /*
283 * src/share/vm/oops/klass.hpp: For instances, layout helper is a positive number,
284 * the instance size. This size is already passed through align_object_size and
285 * scaled to bytes. The low order bit is set if instances of this class cannot be
286 * allocated using the fastpath.
287 */
288 if (probability(FAST_PATH_PROBABILITY, (layoutHelper & 1) == 0)) {
289 Word prototypeMarkWord = nonNullHub.readWord(prototypeMarkWordOffset(INJECTED_VMCONFIG), PROTOTYPE_MARK_WORD_LOCATION);
290 /*
291 * FIXME(je,ds): we should actually pass typeContext instead of "" but late
292 * binding of parameters is not yet supported by the GraphBuilderPlugin system.
293 */
294 return allocateInstanceHelper(layoutHelper, nonNullHub, prototypeMarkWord, fillContents, threadRegister, false, "", options, counters);
295 }
296 } else {
297 DeoptimizeNode.deopt(None, RuntimeConstraint);
298 }
299 }
300 return dynamicNewInstanceStub(type);
301 }
302
303 /**
304 * Maximum array length for which fast path allocation is used.
305 */
306 public static final int MAX_ARRAY_FAST_PATH_ALLOCATION_LENGTH = 0x00FFFFFF;
307
308 @Snippet
309 public static Object allocatePrimitiveArrayPIC(KlassPointer hub, int length, Word prototypeMarkWord, @ConstantParameter int headerSize, @ConstantParameter int log2ElementSize,
310 @ConstantParameter boolean fillContents, @ConstantParameter Register threadRegister, @ConstantParameter boolean maybeUnroll, @ConstantParameter String typeContext,
311 @ConstantParameter OptionValues options, @ConstantParameter Counters counters) {
312 // Primitive array types are eagerly pre-resolved. We can use a floating load.
313 KlassPointer picHub = LoadConstantIndirectlyNode.loadKlass(hub);
314 return allocateArrayImpl(picHub, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents, threadRegister, maybeUnroll, typeContext, false, options, counters);
315 }
316
317 @Snippet
318 public static Object allocateArrayPIC(KlassPointer hub, int length, Word prototypeMarkWord, @ConstantParameter int headerSize, @ConstantParameter int log2ElementSize,
319 @ConstantParameter boolean fillContents, @ConstantParameter Register threadRegister, @ConstantParameter boolean maybeUnroll, @ConstantParameter String typeContext,
320 @ConstantParameter OptionValues options, @ConstantParameter Counters counters) {
321 // Array type would be resolved by dominating resolution.
322 KlassPointer picHub = LoadConstantIndirectlyFixedNode.loadKlass(hub);
323 return allocateArrayImpl(picHub, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents, threadRegister, maybeUnroll, typeContext, false, options, counters);
324 }
325
326 @Snippet
327 public static Object allocateArray(KlassPointer hub,
328 int length,
329 Word prototypeMarkWord,
330 @ConstantParameter int headerSize,
331 @ConstantParameter int log2ElementSize,
332 @ConstantParameter boolean fillContents,
333 @ConstantParameter Register threadRegister,
334 @ConstantParameter boolean maybeUnroll,
335 @ConstantParameter String typeContext,
336 @ConstantParameter OptionValues options,
337 @ConstantParameter Counters counters) {
338 Object result = allocateArrayImpl(hub, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents, threadRegister, maybeUnroll, typeContext, false, options, counters);
339 return piArrayCastToSnippetReplaceeStamp(verifyOop(result), length);
340 }
341
342 /**
343 * When allocating on the slow path, determines whether to use a version of the runtime call
344 * that returns {@code null} on a failed allocation instead of raising an OutOfMemoryError.
345 */
346 @Fold
347 static boolean useNullAllocationStubs(@InjectedParameter GraalHotSpotVMConfig config) {
348 return config.areNullAllocationStubsAvailable();
349 }
350
351 private static Object allocateArrayImpl(KlassPointer hub, int length, Word prototypeMarkWord, int headerSize, int log2ElementSize, boolean fillContents, Register threadRegister,
352 boolean maybeUnroll, String typeContext, boolean skipNegativeCheck, OptionValues options, Counters counters) {
353 Object result;
354 int allocationSize = arrayAllocationSize(length, headerSize, log2ElementSize);
355 Word thread = registerAsWord(threadRegister);
356 Word top = readTlabTop(thread);
357 Word end = readTlabEnd(thread);
358 Word newTop = top.add(allocationSize);
359 if (probability(FREQUENT_PROBABILITY, skipNegativeCheck || belowThan(length, MAX_ARRAY_FAST_PATH_ALLOCATION_LENGTH)) && useTLAB(INJECTED_VMCONFIG) &&
360 probability(FAST_PATH_PROBABILITY, newTop.belowOrEqual(end))) {
361 writeTlabTop(thread, newTop);
362 emitPrefetchAllocate(newTop, true);
363 if (counters != null && counters.arrayLoopInit != null) {
364 counters.arrayLoopInit.inc();
365 }
366 result = formatArray(hub, allocationSize, length, headerSize, top, prototypeMarkWord, fillContents, maybeUnroll, counters);
367 } else {
368 result = newArrayStub(hub, length);
369 }
370 profileAllocation("array", allocationSize, typeContext, options);
371 return result;
372 }
373
374 public static Object newArrayStub(KlassPointer hub, int length) {
375 if (useNullAllocationStubs(INJECTED_VMCONFIG)) {
376 return nonNullOrDeopt(newArrayOrNull(NEW_ARRAY_OR_NULL, hub, length));
377 } else {
378 return newArray(NEW_ARRAY, hub, length);
379 }
380 }
381
382 @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = true)
383 private static native Object newArray(@ConstantNodeParameter ForeignCallDescriptor descriptor, KlassPointer hub, int length);
384
385 @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = false)
386 private static native Object newArrayOrNull(@ConstantNodeParameter ForeignCallDescriptor descriptor, KlassPointer hub, int length);
387
388 /**
389 * New dynamic array stub that throws an {@link OutOfMemoryError} on allocation failure.
390 */
404 }
405
406 /**
407 * Deoptimizes if {@code obj == null} otherwise returns {@code obj}.
408 */
409 private static Object nonNullOrDeopt(Object obj) {
410 if (obj == null) {
411 DeoptimizeNode.deopt(None, RuntimeConstraint);
412 }
413 return obj;
414 }
415
416 @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = true)
417 public static native Object dynamicNewInstance(@ConstantNodeParameter ForeignCallDescriptor descriptor, Class<?> elementType);
418
419 @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = false)
420 public static native Object dynamicNewInstanceOrNull(@ConstantNodeParameter ForeignCallDescriptor descriptor, Class<?> elementType);
421
422 @Snippet
423 public static Object allocateArrayDynamic(Class<?> elementType, Class<?> voidClass, int length, @ConstantParameter boolean fillContents, @ConstantParameter Register threadRegister,
424 @ConstantParameter JavaKind knownElementKind, @ConstantParameter int knownLayoutHelper, Word prototypeMarkWord, @ConstantParameter OptionValues options,
425 @ConstantParameter Counters counters) {
426 Object result = allocateArrayDynamicImpl(elementType, voidClass, length, fillContents, threadRegister, knownElementKind, knownLayoutHelper, prototypeMarkWord, options, counters);
427 return result;
428 }
429
430 private static Object allocateArrayDynamicImpl(Class<?> elementType, Class<?> voidClass, int length, boolean fillContents, Register threadRegister, JavaKind knownElementKind,
431 int knownLayoutHelper, Word prototypeMarkWord, OptionValues options, Counters counters) {
432 /*
433 * We only need the dynamic check for void when we have no static information from
434 * knownElementKind.
435 */
436 staticAssert(knownElementKind != JavaKind.Void, "unsupported knownElementKind");
437 if (knownElementKind == JavaKind.Illegal && probability(SLOW_PATH_PROBABILITY, elementType == null || DynamicNewArrayNode.throwsIllegalArgumentException(elementType, voidClass))) {
438 DeoptimizeNode.deopt(None, RuntimeConstraint);
439 }
440
441 KlassPointer klass = loadKlassFromObject(elementType, arrayKlassOffset(INJECTED_VMCONFIG), CLASS_ARRAY_KLASS_LOCATION);
442 if (klass.isNull()) {
443 DeoptimizeNode.deopt(None, RuntimeConstraint);
444 }
445 KlassPointer nonNullKlass = ClassGetHubNode.piCastNonNull(klass, SnippetAnchorNode.anchor());
446
447 if (length < 0) {
448 DeoptimizeNode.deopt(None, RuntimeConstraint);
449 }
450 int layoutHelper;
451 if (knownElementKind == JavaKind.Illegal) {
453 } else {
454 runtimeAssert(knownLayoutHelper == readLayoutHelper(nonNullKlass), "layout mismatch");
455 layoutHelper = knownLayoutHelper;
456 }
457 //@formatter:off
458 // from src/share/vm/oops/klass.hpp:
459 //
460 // For arrays, layout helper is a negative number, containing four
461 // distinct bytes, as follows:
462 // MSB:[tag, hsz, ebt, log2(esz)]:LSB
463 // where:
464 // tag is 0x80 if the elements are oops, 0xC0 if non-oops
465 // hsz is array header size in bytes (i.e., offset of first element)
466 // ebt is the BasicType of the elements
467 // esz is the element size in bytes
468 //@formatter:on
469
470 int headerSize = (layoutHelper >> layoutHelperHeaderSizeShift(INJECTED_VMCONFIG)) & layoutHelperHeaderSizeMask(INJECTED_VMCONFIG);
471 int log2ElementSize = (layoutHelper >> layoutHelperLog2ElementSizeShift(INJECTED_VMCONFIG)) & layoutHelperLog2ElementSizeMask(INJECTED_VMCONFIG);
472
473 Object result = allocateArrayImpl(nonNullKlass, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents, threadRegister, false, "dynamic type", true, options, counters);
474 return piArrayCastToSnippetReplaceeStamp(verifyOop(result), length);
475 }
476
477 /**
478 * Calls the runtime stub for implementing MULTIANEWARRAY.
479 */
480 @Snippet
481 private static Object newmultiarray(KlassPointer hub, @ConstantParameter int rank, @VarargsParameter int[] dimensions) {
482 Word dims = DimensionsNode.allocaDimsArray(rank);
483 ExplodeLoopNode.explodeLoop();
484 for (int i = 0; i < rank; i++) {
485 dims.writeInt(i * 4, dimensions[i], LocationIdentity.init());
486 }
487 return newMultiArrayStub(hub, rank, dims);
488 }
489
490 private static Object newMultiArrayStub(KlassPointer hub, int rank, Word dims) {
491 if (useNullAllocationStubs(INJECTED_VMCONFIG)) {
492 return nonNullOrDeopt(newMultiArrayOrNull(NEW_MULTI_ARRAY_OR_NULL, hub, rank, dims));
493 } else {
519 * that stores are aligned.
520 *
521 * @param size number of bytes to zero
522 * @param memory beginning of object which is being zeroed
523 * @param constantSize is {@code size} known to be constant in the snippet
524 * @param startOffset offset to begin zeroing. May not be word aligned.
525 * @param manualUnroll maximally unroll zeroing
526 */
527 private static void zeroMemory(int size, Word memory, boolean constantSize, int startOffset, boolean manualUnroll, Counters counters) {
528 fillMemory(0, size, memory, constantSize, startOffset, manualUnroll, counters);
529 }
530
531 private static void fillMemory(long value, int size, Word memory, boolean constantSize, int startOffset, boolean manualUnroll, Counters counters) {
532 ReplacementsUtil.runtimeAssert((size & 0x7) == 0, "unaligned object size");
533 int offset = startOffset;
534 if ((offset & 0x7) != 0) {
535 memory.writeInt(offset, (int) value, LocationIdentity.init());
536 offset += 4;
537 }
538 ReplacementsUtil.runtimeAssert((offset & 0x7) == 0, "unaligned offset");
539 if (manualUnroll && ((size - offset) / 8) <= MAX_UNROLLED_OBJECT_ZEROING_STORES) {
540 ReplacementsUtil.staticAssert(!constantSize, "size shouldn't be constant at instantiation time");
541 // This case handles arrays of constant length. Instead of having a snippet variant for
542 // each length, generate a chain of stores of maximum length. Once it's inlined the
543 // break statement will trim excess stores.
544 if (counters != null && counters.instanceSeqInit != null) {
545 counters.instanceSeqInit.inc();
546 }
547
548 explodeLoop();
549 for (int i = 0; i < MAX_UNROLLED_OBJECT_ZEROING_STORES; i++, offset += 8) {
550 if (offset == size) {
551 break;
552 }
553 memory.initializeLong(offset, value, LocationIdentity.init());
554 }
555 } else {
556 // Use Word instead of int to avoid extension to long in generated code
557 Word off = WordFactory.signed(offset);
558 if (constantSize && ((size - offset) / 8) <= MAX_UNROLLED_OBJECT_ZEROING_STORES) {
559 if (counters != null && counters.instanceSeqInit != null) {
560 counters.instanceSeqInit.inc();
561 }
562 explodeLoop();
563 } else {
564 if (counters != null && counters.instanceLoopInit != null) {
565 counters.instanceLoopInit.inc();
566 }
567 }
568 for (; off.rawValue() < size; off = off.add(8)) {
569 memory.initializeLong(off, value, LocationIdentity.init());
570 }
571 }
572 }
573
574 /**
575 * Fill uninitialized memory with garbage value in a newly allocated object, unrolling as
576 * necessary and ensuring that stores are aligned.
577 *
578 * @param size number of bytes to zero
579 * @param memory beginning of object which is being zeroed
580 * @param constantSize is {@code size} known to be constant in the snippet
581 * @param startOffset offset to begin zeroing. May not be word aligned.
582 * @param manualUnroll maximally unroll zeroing
583 */
584 private static void fillWithGarbage(int size, Word memory, boolean constantSize, int startOffset, boolean manualUnroll, Counters counters) {
585 fillMemory(0xfefefefefefefefeL, size, memory, constantSize, startOffset, manualUnroll, counters);
677 /**
678 * Lowers a {@link NewInstanceNode}.
679 */
680 public void lower(NewInstanceNode newInstanceNode, HotSpotRegistersProvider registers, LoweringTool tool) {
681 StructuredGraph graph = newInstanceNode.graph();
682 HotSpotResolvedObjectType type = (HotSpotResolvedObjectType) newInstanceNode.instanceClass();
683 assert !type.isArray();
684 ConstantNode hub = ConstantNode.forConstant(KlassPointerStamp.klassNonNull(), type.klass(), providers.getMetaAccess(), graph);
685 int size = instanceSize(type);
686
687 OptionValues localOptions = graph.getOptions();
688 SnippetInfo snippet = GeneratePIC.getValue(localOptions) ? allocateInstancePIC : allocateInstance;
689 Arguments args = new Arguments(snippet, graph.getGuardsStage(), tool.getLoweringStage());
690 args.addConst("size", size);
691 args.add("hub", hub);
692 args.add("prototypeMarkWord", type.prototypeMarkWord());
693 args.addConst("fillContents", newInstanceNode.fillContents());
694 args.addConst("threadRegister", registers.getThreadRegister());
695 args.addConst("constantSize", true);
696 args.addConst("typeContext", ProfileAllocations.getValue(localOptions) ? type.toJavaName(false) : "");
697 args.addConst("options", localOptions);
698 args.addConst("counters", counters);
699
700 SnippetTemplate template = template(newInstanceNode, args);
701 graph.getDebug().log("Lowering allocateInstance in %s: node=%s, template=%s, arguments=%s", graph, newInstanceNode, template, args);
702 template.instantiate(providers.getMetaAccess(), newInstanceNode, DEFAULT_REPLACER, args);
703 }
704
705 /**
706 * Lowers a {@link NewArrayNode}.
707 */
708 public void lower(NewArrayNode newArrayNode, HotSpotRegistersProvider registers, LoweringTool tool) {
709 StructuredGraph graph = newArrayNode.graph();
710 ResolvedJavaType elementType = newArrayNode.elementType();
711 HotSpotResolvedObjectType arrayType = (HotSpotResolvedObjectType) elementType.getArrayClass();
712 JavaKind elementKind = elementType.getJavaKind();
713 ConstantNode hub = ConstantNode.forConstant(KlassPointerStamp.klassNonNull(), arrayType.klass(), providers.getMetaAccess(), graph);
714 final int headerSize = tool.getMetaAccess().getArrayBaseOffset(elementKind);
715 int log2ElementSize = CodeUtil.log2(tool.getMetaAccess().getArrayIndexScale(elementKind));
716
717 OptionValues localOptions = graph.getOptions();
721 snippet = allocatePrimitiveArrayPIC;
722 } else {
723 snippet = allocateArrayPIC;
724 }
725 } else {
726 snippet = allocateArray;
727 }
728
729 Arguments args = new Arguments(snippet, graph.getGuardsStage(), tool.getLoweringStage());
730 args.add("hub", hub);
731 ValueNode length = newArrayNode.length();
732 args.add("length", length.isAlive() ? length : graph.addOrUniqueWithInputs(length));
733 assert arrayType.prototypeMarkWord() == lookupArrayClass(tool, JavaKind.Object).prototypeMarkWord() : "all array types are assumed to have the same prototypeMarkWord";
734 args.add("prototypeMarkWord", arrayType.prototypeMarkWord());
735 args.addConst("headerSize", headerSize);
736 args.addConst("log2ElementSize", log2ElementSize);
737 args.addConst("fillContents", newArrayNode.fillContents());
738 args.addConst("threadRegister", registers.getThreadRegister());
739 args.addConst("maybeUnroll", length.isConstant());
740 args.addConst("typeContext", ProfileAllocations.getValue(localOptions) ? arrayType.toJavaName(false) : "");
741 args.addConst("options", localOptions);
742 args.addConst("counters", counters);
743 SnippetTemplate template = template(newArrayNode, args);
744 graph.getDebug().log("Lowering allocateArray in %s: node=%s, template=%s, arguments=%s", graph, newArrayNode, template, args);
745 template.instantiate(providers.getMetaAccess(), newArrayNode, DEFAULT_REPLACER, args);
746 }
747
748 public void lower(DynamicNewInstanceNode newInstanceNode, HotSpotRegistersProvider registers, LoweringTool tool) {
749 Arguments args = new Arguments(allocateInstanceDynamic, newInstanceNode.graph().getGuardsStage(), tool.getLoweringStage());
750 OptionValues localOptions = newInstanceNode.getOptions();
751 args.add("type", newInstanceNode.getInstanceType());
752 ValueNode classClass = newInstanceNode.getClassClass();
753 assert classClass != null;
754 args.add("classClass", classClass);
755 args.addConst("fillContents", newInstanceNode.fillContents());
756 args.addConst("threadRegister", registers.getThreadRegister());
757 args.addConst("options", localOptions);
758 args.addConst("counters", counters);
759
760 SnippetTemplate template = template(newInstanceNode, args);
761 template.instantiate(providers.getMetaAccess(), newInstanceNode, DEFAULT_REPLACER, args);
762 }
763
764 public void lower(DynamicNewArrayNode newArrayNode, HotSpotRegistersProvider registers, LoweringTool tool) {
765 StructuredGraph graph = newArrayNode.graph();
766 OptionValues localOptions = graph.getOptions();
767 Arguments args = new Arguments(allocateArrayDynamic, newArrayNode.graph().getGuardsStage(), tool.getLoweringStage());
768 args.add("elementType", newArrayNode.getElementType());
769 ValueNode voidClass = newArrayNode.getVoidClass();
770 assert voidClass != null;
771 args.add("voidClass", voidClass);
772 ValueNode length = newArrayNode.length();
773 args.add("length", length.isAlive() ? length : graph.addOrUniqueWithInputs(length));
774 args.addConst("fillContents", newArrayNode.fillContents());
775 args.addConst("threadRegister", registers.getThreadRegister());
776 /*
777 * We use Kind.Illegal as a marker value instead of null because constant snippet
778 * parameters cannot be null.
779 */
780 args.addConst("knownElementKind", newArrayNode.getKnownElementKind() == null ? JavaKind.Illegal : newArrayNode.getKnownElementKind());
781 if (newArrayNode.getKnownElementKind() != null) {
782 args.addConst("knownLayoutHelper", lookupArrayClass(tool, newArrayNode.getKnownElementKind()).layoutHelper());
783 } else {
784 args.addConst("knownLayoutHelper", 0);
785 }
786 args.add("prototypeMarkWord", lookupArrayClass(tool, JavaKind.Object).prototypeMarkWord());
787 args.addConst("options", localOptions);
788 args.addConst("counters", counters);
789 SnippetTemplate template = template(newArrayNode, args);
790 template.instantiate(providers.getMetaAccess(), newArrayNode, DEFAULT_REPLACER, args);
791 }
792
793 private static HotSpotResolvedObjectType lookupArrayClass(LoweringTool tool, JavaKind kind) {
794 return (HotSpotResolvedObjectType) tool.getMetaAccess().lookupJavaType(kind == JavaKind.Object ? Object.class : kind.toJavaClass()).getArrayClass();
795 }
796
797 public void lower(NewMultiArrayNode newmultiarrayNode, LoweringTool tool) {
798 StructuredGraph graph = newmultiarrayNode.graph();
799 OptionValues localOptions = graph.getOptions();
800 int rank = newmultiarrayNode.dimensionCount();
801 ValueNode[] dims = new ValueNode[rank];
802 for (int i = 0; i < newmultiarrayNode.dimensionCount(); i++) {
803 dims[i] = newmultiarrayNode.dimension(i);
804 }
805 HotSpotResolvedObjectType type = (HotSpotResolvedObjectType) newmultiarrayNode.type();
806 ConstantNode hub = ConstantNode.forConstant(KlassPointerStamp.klassNonNull(), type.klass(), providers.getMetaAccess(), graph);
807
|
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23
24
25 package org.graalvm.compiler.hotspot.replacements;
26
27 import static jdk.vm.ci.meta.DeoptimizationAction.None;
28 import static jdk.vm.ci.meta.DeoptimizationReason.RuntimeConstraint;
29 import static org.graalvm.compiler.core.common.GraalOptions.GeneratePIC;
30 import static org.graalvm.compiler.core.common.calc.UnsignedMath.belowThan;
31 import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfigBase.INJECTED_OPTIONVALUES;
32 import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfigBase.INJECTED_VMCONFIG;
33 import static org.graalvm.compiler.hotspot.HotSpotBackend.NEW_ARRAY;
34 import static org.graalvm.compiler.hotspot.HotSpotBackend.NEW_ARRAY_OR_NULL;
35 import static org.graalvm.compiler.hotspot.HotSpotBackend.NEW_INSTANCE;
36 import static org.graalvm.compiler.hotspot.HotSpotBackend.NEW_INSTANCE_OR_NULL;
37 import static org.graalvm.compiler.hotspot.HotSpotBackend.NEW_MULTI_ARRAY;
38 import static org.graalvm.compiler.hotspot.HotSpotBackend.NEW_MULTI_ARRAY_OR_NULL;
39 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.CLASS_ARRAY_KLASS_LOCATION;
40 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.HUB_WRITE_LOCATION;
41 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.MARK_WORD_LOCATION;
42 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.PROTOTYPE_MARK_WORD_LOCATION;
43 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_END_LOCATION;
44 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_TOP_LOCATION;
45 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.allocateInstancePrefetchLines;
46 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.allocatePrefetchDistance;
47 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.allocatePrefetchLines;
48 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.allocatePrefetchStepSize;
49 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.allocatePrefetchStyle;
50 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayAllocationSize;
51 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayKlassOffset;
52 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayLengthOffset;
53 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.initializeObjectHeader;
54 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.instanceHeaderSize;
55 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.isInstanceKlassFullyInitialized;
56 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperHeaderSizeMask;
57 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperHeaderSizeShift;
58 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperLog2ElementSizeMask;
59 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperLog2ElementSizeShift;
60 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.loadKlassFromObject;
61 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.prototypeMarkWordOffset;
62 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readLayoutHelper;
63 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readTlabEnd;
64 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readTlabTop;
65 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.registerAsWord;
66 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.useBiasedLocking;
67 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.useTLAB;
68 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.verifyOop;
69 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.writeTlabTop;
70 import static org.graalvm.compiler.hotspot.replacements.HotspotSnippetsOptions.ProfileAllocations;
71 import static org.graalvm.compiler.hotspot.replacements.HotspotSnippetsOptions.ProfileAllocationsContext;
72 import static org.graalvm.compiler.nodes.PiArrayNode.piArrayCastToSnippetReplaceeStamp;
139 import jdk.vm.ci.code.Register;
140 import jdk.vm.ci.code.TargetDescription;
141 import jdk.vm.ci.hotspot.HotSpotResolvedObjectType;
142 import jdk.vm.ci.meta.JavaKind;
143 import jdk.vm.ci.meta.ResolvedJavaType;
144
145 /**
146 * Snippets used for implementing NEW, ANEWARRAY and NEWARRAY.
147 */
148 public class NewObjectSnippets implements Snippets {
149
150 enum ProfileContext {
151 AllocatingMethod,
152 InstanceOrArray,
153 AllocatedType,
154 AllocatedTypesInMethod,
155 Total
156 }
157
158 @Fold
159 static String createName(@Fold.InjectedParameter OptionValues options, String path, String typeContext) {
160 switch (ProfileAllocationsContext.getValue(options)) {
161 case AllocatingMethod:
162 return "";
163 case InstanceOrArray:
164 return path;
165 case AllocatedType:
166 case AllocatedTypesInMethod:
167 return typeContext;
168 case Total:
169 return "bytes";
170 default:
171 throw GraalError.shouldNotReachHere();
172 }
173 }
174
175 @Fold
176 static boolean doProfile(@Fold.InjectedParameter OptionValues options) {
177 return ProfileAllocations.getValue(options);
178 }
179
180 @Fold
181 static boolean withContext(@Fold.InjectedParameter OptionValues options) {
182 ProfileContext context = ProfileAllocationsContext.getValue(options);
183 return context == ProfileContext.AllocatingMethod || context == ProfileContext.AllocatedTypesInMethod;
184 }
185
186 protected static void profileAllocation(String path, long size, String typeContext) {
187 if (doProfile(INJECTED_OPTIONVALUES)) {
188 String name = createName(INJECTED_OPTIONVALUES, path, typeContext);
189
190 boolean context = withContext(INJECTED_OPTIONVALUES);
191 DynamicCounterNode.counter("number of bytes allocated", name, size, context);
192 DynamicCounterNode.counter("number of allocations", name, 1, context);
193 }
194 }
195
196 public static void emitPrefetchAllocate(Word address, boolean isArray) {
197 if (allocatePrefetchStyle(INJECTED_VMCONFIG) > 0) {
198 // Insert a prefetch for each allocation only on the fast-path
199 // Generate several prefetch instructions.
200 int lines = isArray ? allocatePrefetchLines(INJECTED_VMCONFIG) : allocateInstancePrefetchLines(INJECTED_VMCONFIG);
201 int stepSize = allocatePrefetchStepSize(INJECTED_VMCONFIG);
202 int distance = allocatePrefetchDistance(INJECTED_VMCONFIG);
203 ExplodeLoopNode.explodeLoop();
204 for (int i = 0; i < lines; i++) {
205 PrefetchAllocateNode.prefetch(OffsetAddressNode.address(address, distance));
206 distance += stepSize;
207 }
208 }
209 }
210
211 @Snippet
212 public static Object allocateInstance(@ConstantParameter int size, KlassPointer hub, Word prototypeMarkWord, @ConstantParameter boolean fillContents,
213 @ConstantParameter Register threadRegister, @ConstantParameter boolean constantSize, @ConstantParameter String typeContext,
214 @ConstantParameter Counters counters) {
215 return piCastToSnippetReplaceeStamp(allocateInstanceHelper(size, hub, prototypeMarkWord, fillContents, threadRegister, constantSize, typeContext, counters));
216 }
217
218 public static Object allocateInstanceHelper(int size, KlassPointer hub, Word prototypeMarkWord, boolean fillContents,
219 Register threadRegister, boolean constantSize, String typeContext, Counters counters) {
220 Object result;
221 Word thread = registerAsWord(threadRegister);
222 Word top = readTlabTop(thread);
223 Word end = readTlabEnd(thread);
224 Word newTop = top.add(size);
225 if (useTLAB(INJECTED_VMCONFIG) && probability(FAST_PATH_PROBABILITY, newTop.belowOrEqual(end))) {
226 writeTlabTop(thread, newTop);
227 emitPrefetchAllocate(newTop, false);
228 result = formatObject(hub, size, top, prototypeMarkWord, fillContents, constantSize, counters);
229 } else {
230 Counters theCounters = counters;
231 if (theCounters != null && theCounters.stub != null) {
232 theCounters.stub.inc();
233 }
234 result = newInstanceStub(hub);
235 }
236 profileAllocation("instance", size, typeContext);
237 return verifyOop(result);
238 }
239
240 public static Object newInstanceStub(KlassPointer hub) {
241 if (useNullAllocationStubs(INJECTED_VMCONFIG)) {
242 return nonNullOrDeopt(newInstanceOrNull(NEW_INSTANCE_OR_NULL, hub));
243 } else {
244 return newInstance(NEW_INSTANCE, hub);
245 }
246 }
247
248 @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = true)
249 private static native Object newInstance(@ConstantNodeParameter ForeignCallDescriptor descriptor, KlassPointer hub);
250
251 @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = false)
252 private static native Object newInstanceOrNull(@ConstantNodeParameter ForeignCallDescriptor descriptor, KlassPointer hub);
253
254 @Snippet
255 public static Object allocateInstancePIC(@ConstantParameter int size, KlassPointer hub, Word prototypeMarkWord, @ConstantParameter boolean fillContents,
256 @ConstantParameter Register threadRegister, @ConstantParameter boolean constantSize, @ConstantParameter String typeContext,
257 @ConstantParameter Counters counters) {
258 // Klass must be initialized by the time the first instance is allocated, therefore we can
259 // just load it from the corresponding cell and avoid the resolution check. We have to use a
260 // fixed load though, to prevent it from floating above the initialization.
261 KlassPointer picHub = LoadConstantIndirectlyFixedNode.loadKlass(hub);
262 return piCastToSnippetReplaceeStamp(allocateInstanceHelper(size, picHub, prototypeMarkWord, fillContents, threadRegister, constantSize, typeContext, counters));
263 }
264
265 @Snippet
266 public static Object allocateInstanceDynamic(Class<?> type, Class<?> classClass, @ConstantParameter boolean fillContents, @ConstantParameter Register threadRegister,
267 @ConstantParameter Counters counters) {
268 if (probability(SLOW_PATH_PROBABILITY, type == null)) {
269 DeoptimizeNode.deopt(None, RuntimeConstraint);
270 }
271 Class<?> nonNullType = PiNode.piCastNonNullClass(type, SnippetAnchorNode.anchor());
272
273 if (probability(SLOW_PATH_PROBABILITY, DynamicNewInstanceNode.throwsInstantiationException(type, classClass))) {
274 DeoptimizeNode.deopt(None, RuntimeConstraint);
275 }
276
277 return PiNode.piCastToSnippetReplaceeStamp(allocateInstanceDynamicHelper(type, fillContents, threadRegister, counters, nonNullType));
278 }
279
280 private static Object allocateInstanceDynamicHelper(Class<?> type, boolean fillContents, Register threadRegister, Counters counters, Class<?> nonNullType) {
281 KlassPointer hub = ClassGetHubNode.readClass(nonNullType);
282 if (probability(FAST_PATH_PROBABILITY, !hub.isNull())) {
283 KlassPointer nonNullHub = ClassGetHubNode.piCastNonNull(hub, SnippetAnchorNode.anchor());
284
285 if (probability(FAST_PATH_PROBABILITY, isInstanceKlassFullyInitialized(nonNullHub))) {
286 int layoutHelper = readLayoutHelper(nonNullHub);
287 /*
288 * src/share/vm/oops/klass.hpp: For instances, layout helper is a positive number,
289 * the instance size. This size is already passed through align_object_size and
290 * scaled to bytes. The low order bit is set if instances of this class cannot be
291 * allocated using the fastpath.
292 */
293 if (probability(FAST_PATH_PROBABILITY, (layoutHelper & 1) == 0)) {
294 Word prototypeMarkWord = nonNullHub.readWord(prototypeMarkWordOffset(INJECTED_VMCONFIG), PROTOTYPE_MARK_WORD_LOCATION);
295 /*
296 * FIXME(je,ds): we should actually pass typeContext instead of "" but late
297 * binding of parameters is not yet supported by the GraphBuilderPlugin system.
298 */
299 return allocateInstanceHelper(layoutHelper, nonNullHub, prototypeMarkWord, fillContents, threadRegister, false, "", counters);
300 }
301 } else {
302 DeoptimizeNode.deopt(None, RuntimeConstraint);
303 }
304 }
305 return dynamicNewInstanceStub(type);
306 }
307
308 /**
309 * Maximum array length for which fast path allocation is used.
310 */
311 public static final int MAX_ARRAY_FAST_PATH_ALLOCATION_LENGTH = 0x00FFFFFF;
312
313 @Snippet
314 public static Object allocatePrimitiveArrayPIC(KlassPointer hub, int length, Word prototypeMarkWord, @ConstantParameter int headerSize, @ConstantParameter int log2ElementSize,
315 @ConstantParameter boolean fillContents, @ConstantParameter Register threadRegister, @ConstantParameter boolean maybeUnroll, @ConstantParameter String typeContext,
316 @ConstantParameter Counters counters) {
317 // Primitive array types are eagerly pre-resolved. We can use a floating load.
318 KlassPointer picHub = LoadConstantIndirectlyNode.loadKlass(hub);
319 return allocateArrayImpl(picHub, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents, threadRegister, maybeUnroll, typeContext, false, counters);
320 }
321
322 @Snippet
323 public static Object allocateArrayPIC(KlassPointer hub, int length, Word prototypeMarkWord, @ConstantParameter int headerSize, @ConstantParameter int log2ElementSize,
324 @ConstantParameter boolean fillContents, @ConstantParameter Register threadRegister, @ConstantParameter boolean maybeUnroll, @ConstantParameter String typeContext,
325 @ConstantParameter Counters counters) {
326 // Array type would be resolved by dominating resolution.
327 KlassPointer picHub = LoadConstantIndirectlyFixedNode.loadKlass(hub);
328 return allocateArrayImpl(picHub, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents, threadRegister, maybeUnroll, typeContext, false, counters);
329 }
330
331 @Snippet
332 public static Object allocateArray(KlassPointer hub,
333 int length,
334 Word prototypeMarkWord,
335 @ConstantParameter int headerSize,
336 @ConstantParameter int log2ElementSize,
337 @ConstantParameter boolean fillContents,
338 @ConstantParameter Register threadRegister,
339 @ConstantParameter boolean maybeUnroll,
340 @ConstantParameter String typeContext,
341 @ConstantParameter Counters counters) {
342 Object result = allocateArrayImpl(hub,
343 length,
344 prototypeMarkWord,
345 headerSize,
346 log2ElementSize,
347 fillContents,
348 threadRegister,
349 maybeUnroll,
350 typeContext,
351 false,
352
353 counters);
354 return piArrayCastToSnippetReplaceeStamp(verifyOop(result), length);
355 }
356
357 /**
358 * When allocating on the slow path, determines whether to use a version of the runtime call
359 * that returns {@code null} on a failed allocation instead of raising an OutOfMemoryError.
360 */
361 @Fold
362 static boolean useNullAllocationStubs(@InjectedParameter GraalHotSpotVMConfig config) {
363 return config.areNullAllocationStubsAvailable();
364 }
365
366 private static Object allocateArrayImpl(KlassPointer hub, int length, Word prototypeMarkWord, int headerSize, int log2ElementSize, boolean fillContents, Register threadRegister,
367 boolean maybeUnroll, String typeContext, boolean skipNegativeCheck, Counters counters) {
368 Object result;
369 int allocationSize = arrayAllocationSize(length, headerSize, log2ElementSize);
370 Word thread = registerAsWord(threadRegister);
371 Word top = readTlabTop(thread);
372 Word end = readTlabEnd(thread);
373 Word newTop = top.add(allocationSize);
374 if (probability(FREQUENT_PROBABILITY, skipNegativeCheck || belowThan(length, MAX_ARRAY_FAST_PATH_ALLOCATION_LENGTH)) && useTLAB(INJECTED_VMCONFIG) &&
375 probability(FAST_PATH_PROBABILITY, newTop.belowOrEqual(end))) {
376 writeTlabTop(thread, newTop);
377 emitPrefetchAllocate(newTop, true);
378 Counters theCounters = counters;
379 if (theCounters != null && theCounters.arrayLoopInit != null) {
380 theCounters.arrayLoopInit.inc();
381 }
382 result = formatArray(hub, allocationSize, length, headerSize, top, prototypeMarkWord, fillContents, maybeUnroll, counters);
383 } else {
384 result = newArrayStub(hub, length);
385 }
386 profileAllocation("array", allocationSize, typeContext);
387 return result;
388 }
389
390 public static Object newArrayStub(KlassPointer hub, int length) {
391 if (useNullAllocationStubs(INJECTED_VMCONFIG)) {
392 return nonNullOrDeopt(newArrayOrNull(NEW_ARRAY_OR_NULL, hub, length));
393 } else {
394 return newArray(NEW_ARRAY, hub, length);
395 }
396 }
397
398 @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = true)
399 private static native Object newArray(@ConstantNodeParameter ForeignCallDescriptor descriptor, KlassPointer hub, int length);
400
401 @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = false)
402 private static native Object newArrayOrNull(@ConstantNodeParameter ForeignCallDescriptor descriptor, KlassPointer hub, int length);
403
404 /**
405 * New dynamic array stub that throws an {@link OutOfMemoryError} on allocation failure.
406 */
420 }
421
422 /**
423 * Deoptimizes if {@code obj == null} otherwise returns {@code obj}.
424 */
425 private static Object nonNullOrDeopt(Object obj) {
426 if (obj == null) {
427 DeoptimizeNode.deopt(None, RuntimeConstraint);
428 }
429 return obj;
430 }
431
432 @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = true)
433 public static native Object dynamicNewInstance(@ConstantNodeParameter ForeignCallDescriptor descriptor, Class<?> elementType);
434
435 @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = false)
436 public static native Object dynamicNewInstanceOrNull(@ConstantNodeParameter ForeignCallDescriptor descriptor, Class<?> elementType);
437
438 @Snippet
439 public static Object allocateArrayDynamic(Class<?> elementType, Class<?> voidClass, int length, @ConstantParameter boolean fillContents, @ConstantParameter Register threadRegister,
440 @ConstantParameter JavaKind knownElementKind, @ConstantParameter int knownLayoutHelper, Word prototypeMarkWord,
441 @ConstantParameter Counters counters) {
442 Object result = allocateArrayDynamicImpl(elementType, voidClass, length, fillContents, threadRegister, knownElementKind, knownLayoutHelper, prototypeMarkWord, counters);
443 return result;
444 }
445
446 private static Object allocateArrayDynamicImpl(Class<?> elementType, Class<?> voidClass, int length, boolean fillContents, Register threadRegister, JavaKind knownElementKind,
447 int knownLayoutHelper, Word prototypeMarkWord, Counters counters) {
448 /*
449 * We only need the dynamic check for void when we have no static information from
450 * knownElementKind.
451 */
452 staticAssert(knownElementKind != JavaKind.Void, "unsupported knownElementKind");
453 if (knownElementKind == JavaKind.Illegal && probability(SLOW_PATH_PROBABILITY, elementType == null || DynamicNewArrayNode.throwsIllegalArgumentException(elementType, voidClass))) {
454 DeoptimizeNode.deopt(None, RuntimeConstraint);
455 }
456
457 KlassPointer klass = loadKlassFromObject(elementType, arrayKlassOffset(INJECTED_VMCONFIG), CLASS_ARRAY_KLASS_LOCATION);
458 if (klass.isNull()) {
459 DeoptimizeNode.deopt(None, RuntimeConstraint);
460 }
461 KlassPointer nonNullKlass = ClassGetHubNode.piCastNonNull(klass, SnippetAnchorNode.anchor());
462
463 if (length < 0) {
464 DeoptimizeNode.deopt(None, RuntimeConstraint);
465 }
466 int layoutHelper;
467 if (knownElementKind == JavaKind.Illegal) {
469 } else {
470 runtimeAssert(knownLayoutHelper == readLayoutHelper(nonNullKlass), "layout mismatch");
471 layoutHelper = knownLayoutHelper;
472 }
473 //@formatter:off
474 // from src/share/vm/oops/klass.hpp:
475 //
476 // For arrays, layout helper is a negative number, containing four
477 // distinct bytes, as follows:
478 // MSB:[tag, hsz, ebt, log2(esz)]:LSB
479 // where:
480 // tag is 0x80 if the elements are oops, 0xC0 if non-oops
481 // hsz is array header size in bytes (i.e., offset of first element)
482 // ebt is the BasicType of the elements
483 // esz is the element size in bytes
484 //@formatter:on
485
486 int headerSize = (layoutHelper >> layoutHelperHeaderSizeShift(INJECTED_VMCONFIG)) & layoutHelperHeaderSizeMask(INJECTED_VMCONFIG);
487 int log2ElementSize = (layoutHelper >> layoutHelperLog2ElementSizeShift(INJECTED_VMCONFIG)) & layoutHelperLog2ElementSizeMask(INJECTED_VMCONFIG);
488
489 Object result = allocateArrayImpl(nonNullKlass, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents, threadRegister, false, "dynamic type", true, counters);
490 return piArrayCastToSnippetReplaceeStamp(verifyOop(result), length);
491 }
492
493 /**
494 * Calls the runtime stub for implementing MULTIANEWARRAY.
495 */
496 @Snippet
497 private static Object newmultiarray(KlassPointer hub, @ConstantParameter int rank, @VarargsParameter int[] dimensions) {
498 Word dims = DimensionsNode.allocaDimsArray(rank);
499 ExplodeLoopNode.explodeLoop();
500 for (int i = 0; i < rank; i++) {
501 dims.writeInt(i * 4, dimensions[i], LocationIdentity.init());
502 }
503 return newMultiArrayStub(hub, rank, dims);
504 }
505
506 private static Object newMultiArrayStub(KlassPointer hub, int rank, Word dims) {
507 if (useNullAllocationStubs(INJECTED_VMCONFIG)) {
508 return nonNullOrDeopt(newMultiArrayOrNull(NEW_MULTI_ARRAY_OR_NULL, hub, rank, dims));
509 } else {
535 * that stores are aligned.
536 *
537 * @param size number of bytes to zero
538 * @param memory beginning of object which is being zeroed
539 * @param constantSize is {@code size} known to be constant in the snippet
540 * @param startOffset offset to begin zeroing. May not be word aligned.
541 * @param manualUnroll maximally unroll zeroing
542 */
543 private static void zeroMemory(int size, Word memory, boolean constantSize, int startOffset, boolean manualUnroll, Counters counters) {
544 fillMemory(0, size, memory, constantSize, startOffset, manualUnroll, counters);
545 }
546
547 private static void fillMemory(long value, int size, Word memory, boolean constantSize, int startOffset, boolean manualUnroll, Counters counters) {
548 ReplacementsUtil.runtimeAssert((size & 0x7) == 0, "unaligned object size");
549 int offset = startOffset;
550 if ((offset & 0x7) != 0) {
551 memory.writeInt(offset, (int) value, LocationIdentity.init());
552 offset += 4;
553 }
554 ReplacementsUtil.runtimeAssert((offset & 0x7) == 0, "unaligned offset");
555 Counters theCounters = counters;
556 if (manualUnroll && ((size - offset) / 8) <= MAX_UNROLLED_OBJECT_ZEROING_STORES) {
557 ReplacementsUtil.staticAssert(!constantSize, "size shouldn't be constant at instantiation time");
558 // This case handles arrays of constant length. Instead of having a snippet variant for
559 // each length, generate a chain of stores of maximum length. Once it's inlined the
560 // break statement will trim excess stores.
561 if (theCounters != null && theCounters.instanceSeqInit != null) {
562 theCounters.instanceSeqInit.inc();
563 }
564
565 explodeLoop();
566 for (int i = 0; i < MAX_UNROLLED_OBJECT_ZEROING_STORES; i++, offset += 8) {
567 if (offset == size) {
568 break;
569 }
570 memory.initializeLong(offset, value, LocationIdentity.init());
571 }
572 } else {
573 // Use Word instead of int to avoid extension to long in generated code
574 Word off = WordFactory.signed(offset);
575 if (constantSize && ((size - offset) / 8) <= MAX_UNROLLED_OBJECT_ZEROING_STORES) {
576 if (theCounters != null && theCounters.instanceSeqInit != null) {
577 theCounters.instanceSeqInit.inc();
578 }
579 explodeLoop();
580 } else {
581 if (theCounters != null && theCounters.instanceLoopInit != null) {
582 theCounters.instanceLoopInit.inc();
583 }
584 }
585 for (; off.rawValue() < size; off = off.add(8)) {
586 memory.initializeLong(off, value, LocationIdentity.init());
587 }
588 }
589 }
590
591 /**
592 * Fill uninitialized memory with garbage value in a newly allocated object, unrolling as
593 * necessary and ensuring that stores are aligned.
594 *
595 * @param size number of bytes to zero
596 * @param memory beginning of object which is being zeroed
597 * @param constantSize is {@code size} known to be constant in the snippet
598 * @param startOffset offset to begin zeroing. May not be word aligned.
599 * @param manualUnroll maximally unroll zeroing
600 */
601 private static void fillWithGarbage(int size, Word memory, boolean constantSize, int startOffset, boolean manualUnroll, Counters counters) {
602 fillMemory(0xfefefefefefefefeL, size, memory, constantSize, startOffset, manualUnroll, counters);
694 /**
695 * Lowers a {@link NewInstanceNode}.
696 */
697 public void lower(NewInstanceNode newInstanceNode, HotSpotRegistersProvider registers, LoweringTool tool) {
698 StructuredGraph graph = newInstanceNode.graph();
699 HotSpotResolvedObjectType type = (HotSpotResolvedObjectType) newInstanceNode.instanceClass();
700 assert !type.isArray();
701 ConstantNode hub = ConstantNode.forConstant(KlassPointerStamp.klassNonNull(), type.klass(), providers.getMetaAccess(), graph);
702 int size = instanceSize(type);
703
704 OptionValues localOptions = graph.getOptions();
705 SnippetInfo snippet = GeneratePIC.getValue(localOptions) ? allocateInstancePIC : allocateInstance;
706 Arguments args = new Arguments(snippet, graph.getGuardsStage(), tool.getLoweringStage());
707 args.addConst("size", size);
708 args.add("hub", hub);
709 args.add("prototypeMarkWord", type.prototypeMarkWord());
710 args.addConst("fillContents", newInstanceNode.fillContents());
711 args.addConst("threadRegister", registers.getThreadRegister());
712 args.addConst("constantSize", true);
713 args.addConst("typeContext", ProfileAllocations.getValue(localOptions) ? type.toJavaName(false) : "");
714 args.addConst("counters", counters);
715
716 SnippetTemplate template = template(newInstanceNode, args);
717 graph.getDebug().log("Lowering allocateInstance in %s: node=%s, template=%s, arguments=%s", graph, newInstanceNode, template, args);
718 template.instantiate(providers.getMetaAccess(), newInstanceNode, DEFAULT_REPLACER, args);
719 }
720
721 /**
722 * Lowers a {@link NewArrayNode}.
723 */
724 public void lower(NewArrayNode newArrayNode, HotSpotRegistersProvider registers, LoweringTool tool) {
725 StructuredGraph graph = newArrayNode.graph();
726 ResolvedJavaType elementType = newArrayNode.elementType();
727 HotSpotResolvedObjectType arrayType = (HotSpotResolvedObjectType) elementType.getArrayClass();
728 JavaKind elementKind = elementType.getJavaKind();
729 ConstantNode hub = ConstantNode.forConstant(KlassPointerStamp.klassNonNull(), arrayType.klass(), providers.getMetaAccess(), graph);
730 final int headerSize = tool.getMetaAccess().getArrayBaseOffset(elementKind);
731 int log2ElementSize = CodeUtil.log2(tool.getMetaAccess().getArrayIndexScale(elementKind));
732
733 OptionValues localOptions = graph.getOptions();
737 snippet = allocatePrimitiveArrayPIC;
738 } else {
739 snippet = allocateArrayPIC;
740 }
741 } else {
742 snippet = allocateArray;
743 }
744
745 Arguments args = new Arguments(snippet, graph.getGuardsStage(), tool.getLoweringStage());
746 args.add("hub", hub);
747 ValueNode length = newArrayNode.length();
748 args.add("length", length.isAlive() ? length : graph.addOrUniqueWithInputs(length));
749 assert arrayType.prototypeMarkWord() == lookupArrayClass(tool, JavaKind.Object).prototypeMarkWord() : "all array types are assumed to have the same prototypeMarkWord";
750 args.add("prototypeMarkWord", arrayType.prototypeMarkWord());
751 args.addConst("headerSize", headerSize);
752 args.addConst("log2ElementSize", log2ElementSize);
753 args.addConst("fillContents", newArrayNode.fillContents());
754 args.addConst("threadRegister", registers.getThreadRegister());
755 args.addConst("maybeUnroll", length.isConstant());
756 args.addConst("typeContext", ProfileAllocations.getValue(localOptions) ? arrayType.toJavaName(false) : "");
757 args.addConst("counters", counters);
758 SnippetTemplate template = template(newArrayNode, args);
759 graph.getDebug().log("Lowering allocateArray in %s: node=%s, template=%s, arguments=%s", graph, newArrayNode, template, args);
760 template.instantiate(providers.getMetaAccess(), newArrayNode, DEFAULT_REPLACER, args);
761 }
762
763 public void lower(DynamicNewInstanceNode newInstanceNode, HotSpotRegistersProvider registers, LoweringTool tool) {
764 Arguments args = new Arguments(allocateInstanceDynamic, newInstanceNode.graph().getGuardsStage(), tool.getLoweringStage());
765 args.add("type", newInstanceNode.getInstanceType());
766 ValueNode classClass = newInstanceNode.getClassClass();
767 assert classClass != null;
768 args.add("classClass", classClass);
769 args.addConst("fillContents", newInstanceNode.fillContents());
770 args.addConst("threadRegister", registers.getThreadRegister());
771 args.addConst("counters", counters);
772
773 SnippetTemplate template = template(newInstanceNode, args);
774 template.instantiate(providers.getMetaAccess(), newInstanceNode, DEFAULT_REPLACER, args);
775 }
776
777 public void lower(DynamicNewArrayNode newArrayNode, HotSpotRegistersProvider registers, LoweringTool tool) {
778 StructuredGraph graph = newArrayNode.graph();
779 Arguments args = new Arguments(allocateArrayDynamic, newArrayNode.graph().getGuardsStage(), tool.getLoweringStage());
780 args.add("elementType", newArrayNode.getElementType());
781 ValueNode voidClass = newArrayNode.getVoidClass();
782 assert voidClass != null;
783 args.add("voidClass", voidClass);
784 ValueNode length = newArrayNode.length();
785 args.add("length", length.isAlive() ? length : graph.addOrUniqueWithInputs(length));
786 args.addConst("fillContents", newArrayNode.fillContents());
787 args.addConst("threadRegister", registers.getThreadRegister());
788 /*
789 * We use Kind.Illegal as a marker value instead of null because constant snippet
790 * parameters cannot be null.
791 */
792 args.addConst("knownElementKind", newArrayNode.getKnownElementKind() == null ? JavaKind.Illegal : newArrayNode.getKnownElementKind());
793 if (newArrayNode.getKnownElementKind() != null) {
794 args.addConst("knownLayoutHelper", lookupArrayClass(tool, newArrayNode.getKnownElementKind()).layoutHelper());
795 } else {
796 args.addConst("knownLayoutHelper", 0);
797 }
798 args.add("prototypeMarkWord", lookupArrayClass(tool, JavaKind.Object).prototypeMarkWord());
799 args.addConst("counters", counters);
800 SnippetTemplate template = template(newArrayNode, args);
801 template.instantiate(providers.getMetaAccess(), newArrayNode, DEFAULT_REPLACER, args);
802 }
803
804 private static HotSpotResolvedObjectType lookupArrayClass(LoweringTool tool, JavaKind kind) {
805 return (HotSpotResolvedObjectType) tool.getMetaAccess().lookupJavaType(kind == JavaKind.Object ? Object.class : kind.toJavaClass()).getArrayClass();
806 }
807
808 public void lower(NewMultiArrayNode newmultiarrayNode, LoweringTool tool) {
809 StructuredGraph graph = newmultiarrayNode.graph();
810 OptionValues localOptions = graph.getOptions();
811 int rank = newmultiarrayNode.dimensionCount();
812 ValueNode[] dims = new ValueNode[rank];
813 for (int i = 0; i < newmultiarrayNode.dimensionCount(); i++) {
814 dims[i] = newmultiarrayNode.dimension(i);
815 }
816 HotSpotResolvedObjectType type = (HotSpotResolvedObjectType) newmultiarrayNode.type();
817 ConstantNode hub = ConstantNode.forConstant(KlassPointerStamp.klassNonNull(), type.klass(), providers.getMetaAccess(), graph);
818
|