< prev index next >
src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/NewObjectSnippets.java
Print this page
rev 56282 : [mq]: graal
*** 22,31 ****
--- 22,32 ----
*/
package org.graalvm.compiler.hotspot.replacements;
+ import static jdk.vm.ci.meta.DeoptimizationAction.InvalidateRecompile;
import static jdk.vm.ci.meta.DeoptimizationAction.None;
import static jdk.vm.ci.meta.DeoptimizationReason.RuntimeConstraint;
import static org.graalvm.compiler.core.common.GraalOptions.GeneratePIC;
import static org.graalvm.compiler.core.common.GraalOptions.MinimalBulkZeroingSize;
import static org.graalvm.compiler.core.common.calc.UnsignedMath.belowThan;
*** 36,46 ****
import static org.graalvm.compiler.hotspot.HotSpotBackend.NEW_INSTANCE;
import static org.graalvm.compiler.hotspot.HotSpotBackend.NEW_INSTANCE_OR_NULL;
import static org.graalvm.compiler.hotspot.HotSpotBackend.NEW_MULTI_ARRAY;
import static org.graalvm.compiler.hotspot.HotSpotBackend.NEW_MULTI_ARRAY_OR_NULL;
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.CLASS_ARRAY_KLASS_LOCATION;
! import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.CLASS_STATE_LOCATION;
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.HUB_WRITE_LOCATION;
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.MARK_WORD_LOCATION;
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.PROTOTYPE_MARK_WORD_LOCATION;
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_END_LOCATION;
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_TOP_LOCATION;
--- 37,47 ----
import static org.graalvm.compiler.hotspot.HotSpotBackend.NEW_INSTANCE;
import static org.graalvm.compiler.hotspot.HotSpotBackend.NEW_INSTANCE_OR_NULL;
import static org.graalvm.compiler.hotspot.HotSpotBackend.NEW_MULTI_ARRAY;
import static org.graalvm.compiler.hotspot.HotSpotBackend.NEW_MULTI_ARRAY_OR_NULL;
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.CLASS_ARRAY_KLASS_LOCATION;
! import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.CLASS_INIT_STATE_LOCATION;
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.HUB_WRITE_LOCATION;
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.MARK_WORD_LOCATION;
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.PROTOTYPE_MARK_WORD_LOCATION;
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_END_LOCATION;
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_TOP_LOCATION;
*** 52,68 ****
--- 53,72 ----
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayAllocationSize;
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayKlassOffset;
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayLengthOffset;
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.initializeObjectHeader;
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.instanceHeaderSize;
+ import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.instanceKlassStateBeingInitialized;
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.isInstanceKlassFullyInitialized;
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperHeaderSizeMask;
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperHeaderSizeShift;
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperLog2ElementSizeMask;
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperLog2ElementSizeShift;
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.loadKlassFromObject;
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.prototypeMarkWordOffset;
+ import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readInstanceKlassInitState;
+ import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readInstanceKlassInitThread;
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readLayoutHelper;
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readTlabEnd;
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readTlabTop;
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.registerAsWord;
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.useBiasedLocking;
*** 71,83 ****
--- 75,89 ----
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.writeTlabTop;
import static org.graalvm.compiler.hotspot.replacements.HotspotSnippetsOptions.ProfileAllocations;
import static org.graalvm.compiler.hotspot.replacements.HotspotSnippetsOptions.ProfileAllocationsContext;
import static org.graalvm.compiler.nodes.PiArrayNode.piArrayCastToSnippetReplaceeStamp;
import static org.graalvm.compiler.nodes.PiNode.piCastToSnippetReplaceeStamp;
+ import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.DEOPT_PROBABILITY;
import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.FAST_PATH_PROBABILITY;
import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.FREQUENT_PROBABILITY;
import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.SLOW_PATH_PROBABILITY;
+ import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.VERY_FAST_PATH_PROBABILITY;
import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.probability;
import static org.graalvm.compiler.replacements.ReplacementsUtil.REPLACEMENTS_ASSERTIONS_ENABLED;
import static org.graalvm.compiler.replacements.ReplacementsUtil.runtimeAssert;
import static org.graalvm.compiler.replacements.ReplacementsUtil.staticAssert;
import static org.graalvm.compiler.replacements.SnippetTemplate.DEFAULT_REPLACER;
*** 97,106 ****
--- 103,113 ----
import org.graalvm.compiler.graph.Node.NodeIntrinsic;
import org.graalvm.compiler.hotspot.GraalHotSpotVMConfig;
import org.graalvm.compiler.hotspot.meta.HotSpotProviders;
import org.graalvm.compiler.hotspot.meta.HotSpotRegistersProvider;
import org.graalvm.compiler.hotspot.nodes.DimensionsNode;
+ import org.graalvm.compiler.hotspot.nodes.KlassBeingInitializedCheckNode;
import org.graalvm.compiler.hotspot.nodes.aot.LoadConstantIndirectlyFixedNode;
import org.graalvm.compiler.hotspot.nodes.aot.LoadConstantIndirectlyNode;
import org.graalvm.compiler.hotspot.nodes.type.KlassPointerStamp;
import org.graalvm.compiler.hotspot.word.KlassPointer;
import org.graalvm.compiler.nodes.ConstantNode;
*** 110,119 ****
--- 117,127 ----
import org.graalvm.compiler.nodes.SnippetAnchorNode;
import org.graalvm.compiler.nodes.StructuredGraph;
import org.graalvm.compiler.nodes.ValueNode;
import org.graalvm.compiler.nodes.debug.DynamicCounterNode;
import org.graalvm.compiler.nodes.debug.VerifyHeapNode;
+ import org.graalvm.compiler.nodes.extended.BranchProbabilityNode;
import org.graalvm.compiler.nodes.extended.ForeignCallNode;
import org.graalvm.compiler.nodes.extended.MembarNode;
import org.graalvm.compiler.nodes.java.DynamicNewArrayNode;
import org.graalvm.compiler.nodes.java.DynamicNewInstanceNode;
import org.graalvm.compiler.nodes.java.NewArrayNode;
*** 288,303 ****
public static Object allocateInstanceDynamic(Class<?> type, Class<?> classClass,
@ConstantParameter boolean fillContents,
@ConstantParameter boolean emitMemoryBarrier,
@ConstantParameter Register threadRegister,
@ConstantParameter Counters counters) {
! if (probability(SLOW_PATH_PROBABILITY, type == null)) {
DeoptimizeNode.deopt(None, RuntimeConstraint);
}
Class<?> nonNullType = PiNode.piCastNonNullClass(type, SnippetAnchorNode.anchor());
! if (probability(SLOW_PATH_PROBABILITY, DynamicNewInstanceNode.throwsInstantiationException(type, classClass))) {
DeoptimizeNode.deopt(None, RuntimeConstraint);
}
return PiNode.piCastToSnippetReplaceeStamp(allocateInstanceDynamicHelper(type, fillContents, emitMemoryBarrier, threadRegister, counters, nonNullType));
}
--- 296,311 ----
public static Object allocateInstanceDynamic(Class<?> type, Class<?> classClass,
@ConstantParameter boolean fillContents,
@ConstantParameter boolean emitMemoryBarrier,
@ConstantParameter Register threadRegister,
@ConstantParameter Counters counters) {
! if (probability(DEOPT_PROBABILITY, type == null)) {
DeoptimizeNode.deopt(None, RuntimeConstraint);
}
Class<?> nonNullType = PiNode.piCastNonNullClass(type, SnippetAnchorNode.anchor());
! if (probability(DEOPT_PROBABILITY, DynamicNewInstanceNode.throwsInstantiationException(type, classClass))) {
DeoptimizeNode.deopt(None, RuntimeConstraint);
}
return PiNode.piCastToSnippetReplaceeStamp(allocateInstanceDynamicHelper(type, fillContents, emitMemoryBarrier, threadRegister, counters, nonNullType));
}
*** 310,320 ****
Class<?> nonNullType) {
KlassPointer hub = ClassGetHubNode.readClass(nonNullType);
if (probability(FAST_PATH_PROBABILITY, !hub.isNull())) {
KlassPointer nonNullHub = ClassGetHubNode.piCastNonNull(hub, SnippetAnchorNode.anchor());
! if (probability(FAST_PATH_PROBABILITY, isInstanceKlassFullyInitialized(nonNullHub))) {
int layoutHelper = readLayoutHelper(nonNullHub);
/*
* src/share/vm/oops/klass.hpp: For instances, layout helper is a positive number,
* the instance size. This size is already passed through align_object_size and
* scaled to bytes. The low order bit is set if instances of this class cannot be
--- 318,328 ----
Class<?> nonNullType) {
KlassPointer hub = ClassGetHubNode.readClass(nonNullType);
if (probability(FAST_PATH_PROBABILITY, !hub.isNull())) {
KlassPointer nonNullHub = ClassGetHubNode.piCastNonNull(hub, SnippetAnchorNode.anchor());
! if (probability(VERY_FAST_PATH_PROBABILITY, isInstanceKlassFullyInitialized(nonNullHub))) {
int layoutHelper = readLayoutHelper(nonNullHub);
/*
* src/share/vm/oops/klass.hpp: For instances, layout helper is a positive number,
* the instance size. This size is already passed through align_object_size and
* scaled to bytes. The low order bit is set if instances of this class cannot be
*** 349,364 ****
@ConstantParameter boolean fillContents,
@ConstantParameter boolean emitMemoryBarrier,
@ConstantParameter Register threadRegister,
@ConstantParameter boolean maybeUnroll,
@ConstantParameter String typeContext,
! @ConstantParameter boolean useBulkZeroing,
@ConstantParameter Counters counters) {
// Primitive array types are eagerly pre-resolved. We can use a floating load.
KlassPointer picHub = LoadConstantIndirectlyNode.loadKlass(hub);
return allocateArrayImpl(picHub, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents,
! emitMemoryBarrier, threadRegister, maybeUnroll, typeContext, useBulkZeroing, counters);
}
@Snippet
public static Object allocateArrayPIC(KlassPointer hub,
int length,
--- 357,372 ----
@ConstantParameter boolean fillContents,
@ConstantParameter boolean emitMemoryBarrier,
@ConstantParameter Register threadRegister,
@ConstantParameter boolean maybeUnroll,
@ConstantParameter String typeContext,
! @ConstantParameter int bulkZeroingStride,
@ConstantParameter Counters counters) {
// Primitive array types are eagerly pre-resolved. We can use a floating load.
KlassPointer picHub = LoadConstantIndirectlyNode.loadKlass(hub);
return allocateArrayImpl(picHub, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents,
! emitMemoryBarrier, threadRegister, maybeUnroll, typeContext, bulkZeroingStride, counters);
}
@Snippet
public static Object allocateArrayPIC(KlassPointer hub,
int length,
*** 368,383 ****
@ConstantParameter boolean fillContents,
@ConstantParameter boolean emitMemoryBarrier,
@ConstantParameter Register threadRegister,
@ConstantParameter boolean maybeUnroll,
@ConstantParameter String typeContext,
! @ConstantParameter boolean useBulkZeroing,
@ConstantParameter Counters counters) {
// Array type would be resolved by dominating resolution.
KlassPointer picHub = LoadConstantIndirectlyFixedNode.loadKlass(hub);
return allocateArrayImpl(picHub, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents,
! emitMemoryBarrier, threadRegister, maybeUnroll, typeContext, useBulkZeroing, counters);
}
@Snippet
public static Object allocateArray(KlassPointer hub,
int length,
--- 376,391 ----
@ConstantParameter boolean fillContents,
@ConstantParameter boolean emitMemoryBarrier,
@ConstantParameter Register threadRegister,
@ConstantParameter boolean maybeUnroll,
@ConstantParameter String typeContext,
! @ConstantParameter int bulkZeroingStride,
@ConstantParameter Counters counters) {
// Array type would be resolved by dominating resolution.
KlassPointer picHub = LoadConstantIndirectlyFixedNode.loadKlass(hub);
return allocateArrayImpl(picHub, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents,
! emitMemoryBarrier, threadRegister, maybeUnroll, typeContext, bulkZeroingStride, counters);
}
@Snippet
public static Object allocateArray(KlassPointer hub,
int length,
*** 387,408 ****
@ConstantParameter boolean fillContents,
@ConstantParameter boolean emitMemoryBarrier,
@ConstantParameter Register threadRegister,
@ConstantParameter boolean maybeUnroll,
@ConstantParameter String typeContext,
! @ConstantParameter boolean useBulkZeroing,
@ConstantParameter Counters counters) {
Object result = allocateArrayImpl(hub,
length,
prototypeMarkWord,
headerSize,
log2ElementSize,
fillContents,
emitMemoryBarrier, threadRegister,
maybeUnroll,
typeContext,
! useBulkZeroing,
counters);
return piArrayCastToSnippetReplaceeStamp(verifyOop(result), length);
}
/**
--- 395,416 ----
@ConstantParameter boolean fillContents,
@ConstantParameter boolean emitMemoryBarrier,
@ConstantParameter Register threadRegister,
@ConstantParameter boolean maybeUnroll,
@ConstantParameter String typeContext,
! @ConstantParameter int bulkZeroingStride,
@ConstantParameter Counters counters) {
Object result = allocateArrayImpl(hub,
length,
prototypeMarkWord,
headerSize,
log2ElementSize,
fillContents,
emitMemoryBarrier, threadRegister,
maybeUnroll,
typeContext,
! bulkZeroingStride,
counters);
return piArrayCastToSnippetReplaceeStamp(verifyOop(result), length);
}
/**
*** 422,432 ****
boolean fillContents,
boolean emitMemoryBarrier,
Register threadRegister,
boolean maybeUnroll,
String typeContext,
! boolean useBulkZeroing,
Counters counters) {
Object result;
long allocationSize = arrayAllocationSize(length, headerSize, log2ElementSize);
Word thread = registerAsWord(threadRegister);
Word top = readTlabTop(thread);
--- 430,440 ----
boolean fillContents,
boolean emitMemoryBarrier,
Register threadRegister,
boolean maybeUnroll,
String typeContext,
! int bulkZeroingStride,
Counters counters) {
Object result;
long allocationSize = arrayAllocationSize(length, headerSize, log2ElementSize);
Word thread = registerAsWord(threadRegister);
Word top = readTlabTop(thread);
*** 438,448 ****
emitPrefetchAllocate(newTop, true);
Counters theCounters = counters;
if (theCounters != null && theCounters.arrayLoopInit != null) {
theCounters.arrayLoopInit.inc();
}
! result = formatArray(hub, allocationSize, length, headerSize, top, prototypeMarkWord, fillContents, emitMemoryBarrier, maybeUnroll, useBulkZeroing, counters);
} else {
result = newArrayStub(hub, length);
}
profileAllocation("array", allocationSize, typeContext);
return result;
--- 446,456 ----
emitPrefetchAllocate(newTop, true);
Counters theCounters = counters;
if (theCounters != null && theCounters.arrayLoopInit != null) {
theCounters.arrayLoopInit.inc();
}
! result = formatArray(hub, allocationSize, length, headerSize, top, prototypeMarkWord, fillContents, emitMemoryBarrier, maybeUnroll, bulkZeroingStride, counters);
} else {
result = newArrayStub(hub, length);
}
profileAllocation("array", allocationSize, typeContext);
return result;
*** 482,492 ****
/**
* Deoptimizes if {@code obj == null} otherwise returns {@code obj}.
*/
private static Object nonNullOrDeopt(Object obj) {
! if (obj == null) {
DeoptimizeNode.deopt(None, RuntimeConstraint);
}
return obj;
}
--- 490,500 ----
/**
* Deoptimizes if {@code obj == null} otherwise returns {@code obj}.
*/
private static Object nonNullOrDeopt(Object obj) {
! if (BranchProbabilityNode.probability(BranchProbabilityNode.DEOPT_PROBABILITY, obj == null)) {
DeoptimizeNode.deopt(None, RuntimeConstraint);
}
return obj;
}
*** 503,517 ****
@ConstantParameter boolean fillContents,
@ConstantParameter boolean emitMemoryBarrier,
@ConstantParameter Register threadRegister,
@ConstantParameter JavaKind knownElementKind,
@ConstantParameter int knownLayoutHelper,
! @ConstantParameter boolean useBulkZeroing,
Word prototypeMarkWord,
@ConstantParameter Counters counters) {
Object result = allocateArrayDynamicImpl(elementType, voidClass, length, fillContents, emitMemoryBarrier, threadRegister, knownElementKind,
! knownLayoutHelper, useBulkZeroing, prototypeMarkWord, counters);
return result;
}
private static Object allocateArrayDynamicImpl(Class<?> elementType,
Class<?> voidClass,
--- 511,525 ----
@ConstantParameter boolean fillContents,
@ConstantParameter boolean emitMemoryBarrier,
@ConstantParameter Register threadRegister,
@ConstantParameter JavaKind knownElementKind,
@ConstantParameter int knownLayoutHelper,
! @ConstantParameter int bulkZeroingStride,
Word prototypeMarkWord,
@ConstantParameter Counters counters) {
Object result = allocateArrayDynamicImpl(elementType, voidClass, length, fillContents, emitMemoryBarrier, threadRegister, knownElementKind,
! knownLayoutHelper, bulkZeroingStride, prototypeMarkWord, counters);
return result;
}
private static Object allocateArrayDynamicImpl(Class<?> elementType,
Class<?> voidClass,
*** 519,529 ****
boolean fillContents,
boolean emitMemoryBarrier,
Register threadRegister,
JavaKind knownElementKind,
int knownLayoutHelper,
! boolean useBulkZeroing,
Word prototypeMarkWord,
Counters counters) {
/*
* We only need the dynamic check for void when we have no static information from
* knownElementKind.
--- 527,537 ----
boolean fillContents,
boolean emitMemoryBarrier,
Register threadRegister,
JavaKind knownElementKind,
int knownLayoutHelper,
! int bulkZeroingStride,
Word prototypeMarkWord,
Counters counters) {
/*
* We only need the dynamic check for void when we have no static information from
* knownElementKind.
*** 532,547 ****
if (knownElementKind == JavaKind.Illegal && probability(SLOW_PATH_PROBABILITY, elementType == null || DynamicNewArrayNode.throwsIllegalArgumentException(elementType, voidClass))) {
DeoptimizeNode.deopt(None, RuntimeConstraint);
}
KlassPointer klass = loadKlassFromObject(elementType, arrayKlassOffset(INJECTED_VMCONFIG), CLASS_ARRAY_KLASS_LOCATION);
! if (klass.isNull()) {
DeoptimizeNode.deopt(None, RuntimeConstraint);
}
KlassPointer nonNullKlass = ClassGetHubNode.piCastNonNull(klass, SnippetAnchorNode.anchor());
! if (length < 0) {
DeoptimizeNode.deopt(None, RuntimeConstraint);
}
int layoutHelper;
if (knownElementKind == JavaKind.Illegal) {
layoutHelper = readLayoutHelper(nonNullKlass);
--- 540,555 ----
if (knownElementKind == JavaKind.Illegal && probability(SLOW_PATH_PROBABILITY, elementType == null || DynamicNewArrayNode.throwsIllegalArgumentException(elementType, voidClass))) {
DeoptimizeNode.deopt(None, RuntimeConstraint);
}
KlassPointer klass = loadKlassFromObject(elementType, arrayKlassOffset(INJECTED_VMCONFIG), CLASS_ARRAY_KLASS_LOCATION);
! if (probability(DEOPT_PROBABILITY, klass.isNull())) {
DeoptimizeNode.deopt(None, RuntimeConstraint);
}
KlassPointer nonNullKlass = ClassGetHubNode.piCastNonNull(klass, SnippetAnchorNode.anchor());
! if (probability(DEOPT_PROBABILITY, length < 0)) {
DeoptimizeNode.deopt(None, RuntimeConstraint);
}
int layoutHelper;
if (knownElementKind == JavaKind.Illegal) {
layoutHelper = readLayoutHelper(nonNullKlass);
*** 564,574 ****
int headerSize = (layoutHelper >> layoutHelperHeaderSizeShift(INJECTED_VMCONFIG)) & layoutHelperHeaderSizeMask(INJECTED_VMCONFIG);
int log2ElementSize = (layoutHelper >> layoutHelperLog2ElementSizeShift(INJECTED_VMCONFIG)) & layoutHelperLog2ElementSizeMask(INJECTED_VMCONFIG);
Object result = allocateArrayImpl(nonNullKlass, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents,
! emitMemoryBarrier, threadRegister, false, "dynamic type", useBulkZeroing, counters);
return piArrayCastToSnippetReplaceeStamp(verifyOop(result), length);
}
/**
* Calls the runtime stub for implementing MULTIANEWARRAY.
--- 572,582 ----
int headerSize = (layoutHelper >> layoutHelperHeaderSizeShift(INJECTED_VMCONFIG)) & layoutHelperHeaderSizeMask(INJECTED_VMCONFIG);
int log2ElementSize = (layoutHelper >> layoutHelperLog2ElementSizeShift(INJECTED_VMCONFIG)) & layoutHelperLog2ElementSizeMask(INJECTED_VMCONFIG);
Object result = allocateArrayImpl(nonNullKlass, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents,
! emitMemoryBarrier, threadRegister, false, "dynamic type", bulkZeroingStride, counters);
return piArrayCastToSnippetReplaceeStamp(verifyOop(result), length);
}
/**
* Calls the runtime stub for implementing MULTIANEWARRAY.
*** 612,679 ****
/**
* Zero uninitialized memory in a newly allocated object, unrolling as necessary and ensuring
* that stores are aligned.
*
- * @param size number of bytes to zero
* @param memory beginning of object which is being zeroed
! * @param constantSize is {@code size} known to be constant in the snippet
! * @param startOffset offset to begin zeroing. May not be word aligned.
* @param manualUnroll maximally unroll zeroing
! * @param useBulkZeroing apply bulk zeroing
*/
! private static void zeroMemory(long size, Word memory, boolean constantSize, int startOffset, boolean manualUnroll,
! boolean useBulkZeroing, Counters counters) {
! fillMemory(0, size, memory, constantSize, startOffset, manualUnroll, useBulkZeroing, counters);
}
! private static void fillMemory(long value, long size, Word memory, boolean constantSize, int startOffset, boolean manualUnroll,
! boolean useBulkZeroing, Counters counters) {
! ReplacementsUtil.runtimeAssert((size & 0x7) == 0, "unaligned object size");
int offset = startOffset;
if ((offset & 0x7) != 0) {
memory.writeInt(offset, (int) value, LocationIdentity.init());
offset += 4;
}
ReplacementsUtil.runtimeAssert((offset & 0x7) == 0, "unaligned offset");
Counters theCounters = counters;
! if (manualUnroll && ((size - offset) / 8) <= MAX_UNROLLED_OBJECT_ZEROING_STORES) {
! ReplacementsUtil.staticAssert(!constantSize, "size shouldn't be constant at instantiation time");
// This case handles arrays of constant length. Instead of having a snippet variant for
// each length, generate a chain of stores of maximum length. Once it's inlined the
// break statement will trim excess stores.
if (theCounters != null && theCounters.instanceSeqInit != null) {
theCounters.instanceSeqInit.inc();
}
explodeLoop();
for (int i = 0; i < MAX_UNROLLED_OBJECT_ZEROING_STORES; i++, offset += 8) {
! if (offset == size) {
break;
}
memory.initializeLong(offset, value, LocationIdentity.init());
}
} else {
// Use Word instead of int to avoid extension to long in generated code
Word off = WordFactory.signed(offset);
! if (useBulkZeroing && value == 0 && probability(SLOW_PATH_PROBABILITY, (size - offset) >= getMinimalBulkZeroingSize(INJECTED_OPTIONVALUES))) {
if (theCounters != null && theCounters.instanceBulkInit != null) {
theCounters.instanceBulkInit.inc();
}
! ZeroMemoryNode.zero(memory.add(off), size - offset, LocationIdentity.init());
} else {
! if (constantSize && ((size - offset) / 8) <= MAX_UNROLLED_OBJECT_ZEROING_STORES) {
if (theCounters != null && theCounters.instanceSeqInit != null) {
theCounters.instanceSeqInit.inc();
}
explodeLoop();
} else {
if (theCounters != null && theCounters.instanceLoopInit != null) {
theCounters.instanceLoopInit.inc();
}
}
! for (; off.rawValue() < size; off = off.add(8)) {
memory.initializeLong(off, value, LocationIdentity.init());
}
}
}
}
--- 620,687 ----
/**
* Zero uninitialized memory in a newly allocated object, unrolling as necessary and ensuring
* that stores are aligned.
*
* @param memory beginning of object which is being zeroed
! * @param startOffset offset to begin zeroing (inclusive). May not be word aligned.
! * @param endOffset offset to stop zeroing (exclusive). May not be word aligned.
! * @param isEndOffsetConstant is {@code endOffset} known to be constant in the snippet
* @param manualUnroll maximally unroll zeroing
! * @param bulkZeroingStride stride of bulk zeroing supported by the backend
*/
! private static void zeroMemory(Word memory, int startOffset, long endOffset, boolean isEndOffsetConstant, boolean manualUnroll,
! int bulkZeroingStride, Counters counters) {
! fillMemory(0, memory, startOffset, endOffset, isEndOffsetConstant, manualUnroll, bulkZeroingStride, counters);
}
! private static void fillMemory(long value, Word memory, int startOffset, long offsetLimit, boolean constantOffsetLimit, boolean manualUnroll,
! int bulkZeroingStride, Counters counters) {
! ReplacementsUtil.runtimeAssert((offsetLimit & 0x7) == 0, "unaligned object size");
int offset = startOffset;
if ((offset & 0x7) != 0) {
memory.writeInt(offset, (int) value, LocationIdentity.init());
offset += 4;
}
ReplacementsUtil.runtimeAssert((offset & 0x7) == 0, "unaligned offset");
Counters theCounters = counters;
! if (manualUnroll && ((offsetLimit - offset) / 8) <= MAX_UNROLLED_OBJECT_ZEROING_STORES) {
! ReplacementsUtil.staticAssert(!constantOffsetLimit, "size shouldn't be constant at instantiation time");
// This case handles arrays of constant length. Instead of having a snippet variant for
// each length, generate a chain of stores of maximum length. Once it's inlined the
// break statement will trim excess stores.
if (theCounters != null && theCounters.instanceSeqInit != null) {
theCounters.instanceSeqInit.inc();
}
explodeLoop();
for (int i = 0; i < MAX_UNROLLED_OBJECT_ZEROING_STORES; i++, offset += 8) {
! if (offset == offsetLimit) {
break;
}
memory.initializeLong(offset, value, LocationIdentity.init());
}
} else {
// Use Word instead of int to avoid extension to long in generated code
Word off = WordFactory.signed(offset);
! if (bulkZeroingStride > 0 && value == 0 && probability(SLOW_PATH_PROBABILITY, (offsetLimit - offset) >= getMinimalBulkZeroingSize(INJECTED_OPTIONVALUES))) {
if (theCounters != null && theCounters.instanceBulkInit != null) {
theCounters.instanceBulkInit.inc();
}
! ZeroMemoryNode.zero(memory.add(off), offsetLimit - offset, LocationIdentity.init());
} else {
! if (constantOffsetLimit && ((offsetLimit - offset) / 8) <= MAX_UNROLLED_OBJECT_ZEROING_STORES) {
if (theCounters != null && theCounters.instanceSeqInit != null) {
theCounters.instanceSeqInit.inc();
}
explodeLoop();
} else {
if (theCounters != null && theCounters.instanceLoopInit != null) {
theCounters.instanceLoopInit.inc();
}
}
! for (; off.rawValue() < offsetLimit; off = off.add(8)) {
memory.initializeLong(off, value, LocationIdentity.init());
}
}
}
}
*** 685,702 ****
/**
* Fill uninitialized memory with garbage value in a newly allocated object, unrolling as
* necessary and ensuring that stores are aligned.
*
- * @param size number of bytes to zero
* @param memory beginning of object which is being zeroed
! * @param constantSize is {@code size} known to be constant in the snippet
! * @param startOffset offset to begin zeroing. May not be word aligned.
* @param manualUnroll maximally unroll zeroing
*/
! private static void fillWithGarbage(long size, Word memory, boolean constantSize, int startOffset, boolean manualUnroll, Counters counters) {
! fillMemory(0xfefefefefefefefeL, size, memory, constantSize, startOffset, manualUnroll, false, counters);
}
/**
* Formats some allocated memory with an object header and zeroes out the rest.
*/
--- 693,711 ----
/**
* Fill uninitialized memory with garbage value in a newly allocated object, unrolling as
* necessary and ensuring that stores are aligned.
*
* @param memory beginning of object which is being zeroed
! * @param startOffset offset to begin filling garbage value (inclusive). May not be word
! * aligned.
! * @param endOffset offset to stop filling garbage value (exclusive). May not be word aligned.
! * @param isEndOffsetConstant is {@code endOffset} known to be constant in the snippet
* @param manualUnroll maximally unroll zeroing
*/
! private static void fillWithGarbage(Word memory, int startOffset, long endOffset, boolean isEndOffsetConstant, boolean manualUnroll, Counters counters) {
! fillMemory(0xfefefefefefefefeL, memory, startOffset, endOffset, isEndOffsetConstant, manualUnroll, 0, counters);
}
/**
* Formats some allocated memory with an object header and zeroes out the rest.
*/
*** 709,721 ****
boolean constantSize,
Counters counters) {
Word prototypeMarkWord = useBiasedLocking(INJECTED_VMCONFIG) ? hub.readWord(prototypeMarkWordOffset(INJECTED_VMCONFIG), PROTOTYPE_MARK_WORD_LOCATION) : compileTimePrototypeMarkWord;
initializeObjectHeader(memory, prototypeMarkWord, hub);
if (fillContents) {
! zeroMemory(size, memory, constantSize, instanceHeaderSize(INJECTED_VMCONFIG), false, false, counters);
} else if (REPLACEMENTS_ASSERTIONS_ENABLED) {
! fillWithGarbage(size, memory, constantSize, instanceHeaderSize(INJECTED_VMCONFIG), false, counters);
}
if (emitMemoryBarrier) {
MembarNode.memoryBarrier(MemoryBarriers.STORE_STORE, LocationIdentity.init());
}
return memory.toObjectNonNull();
--- 718,730 ----
boolean constantSize,
Counters counters) {
Word prototypeMarkWord = useBiasedLocking(INJECTED_VMCONFIG) ? hub.readWord(prototypeMarkWordOffset(INJECTED_VMCONFIG), PROTOTYPE_MARK_WORD_LOCATION) : compileTimePrototypeMarkWord;
initializeObjectHeader(memory, prototypeMarkWord, hub);
if (fillContents) {
! zeroMemory(memory, instanceHeaderSize(INJECTED_VMCONFIG), size, constantSize, false, 0, counters);
} else if (REPLACEMENTS_ASSERTIONS_ENABLED) {
! fillWithGarbage(memory, instanceHeaderSize(INJECTED_VMCONFIG), size, constantSize, false, counters);
}
if (emitMemoryBarrier) {
MembarNode.memoryBarrier(MemoryBarriers.STORE_STORE, LocationIdentity.init());
}
return memory.toObjectNonNull();
*** 731,740 ****
--- 740,762 ----
AssertionSnippets.vmMessageC(AssertionSnippets.ASSERTION_VM_MESSAGE_C, true, cstring("overzeroing of TLAB detected"), 0L, 0L, 0L);
}
}
}
+ @Snippet
+ private static void threadBeingInitializedCheck(@ConstantParameter Register threadRegister, KlassPointer klass) {
+ int state = readInstanceKlassInitState(klass);
+ if (state != instanceKlassStateBeingInitialized(INJECTED_VMCONFIG)) {
+ // The klass is no longer being initialized so force recompilation
+ DeoptimizeNode.deopt(InvalidateRecompile, RuntimeConstraint);
+ } else if (registerAsWord(threadRegister) != readInstanceKlassInitThread(klass)) {
+ // The klass is being initialized but this isn't the initializing thread so
+ // so deopt and allow execution to resume in the interpreter where it should block.
+ DeoptimizeNode.deopt(None, RuntimeConstraint);
+ }
+ }
+
/**
* Formats some allocated memory with an object header and zeroes out the rest.
*/
private static Object formatArray(KlassPointer hub,
long allocationSize,
*** 743,764 ****
Word memory,
Word prototypeMarkWord,
boolean fillContents,
boolean emitMemoryBarrier,
boolean maybeUnroll,
! boolean useBulkZeroing,
Counters counters) {
memory.writeInt(arrayLengthOffset(INJECTED_VMCONFIG), length, LocationIdentity.init());
/*
* store hub last as the concurrent garbage collectors assume length is valid if hub field
* is not null
*/
initializeObjectHeader(memory, prototypeMarkWord, hub);
if (fillContents) {
! zeroMemory(allocationSize, memory, false, headerSize, maybeUnroll, useBulkZeroing, counters);
} else if (REPLACEMENTS_ASSERTIONS_ENABLED) {
! fillWithGarbage(allocationSize, memory, false, headerSize, maybeUnroll, counters);
}
if (emitMemoryBarrier) {
MembarNode.memoryBarrier(MemoryBarriers.STORE_STORE, LocationIdentity.init());
}
return memory.toObjectNonNull();
--- 765,786 ----
Word memory,
Word prototypeMarkWord,
boolean fillContents,
boolean emitMemoryBarrier,
boolean maybeUnroll,
! int bulkZeroingStride,
Counters counters) {
memory.writeInt(arrayLengthOffset(INJECTED_VMCONFIG), length, LocationIdentity.init());
/*
* store hub last as the concurrent garbage collectors assume length is valid if hub field
* is not null
*/
initializeObjectHeader(memory, prototypeMarkWord, hub);
if (fillContents) {
! zeroMemory(memory, headerSize, allocationSize, false, maybeUnroll, bulkZeroingStride, counters);
} else if (REPLACEMENTS_ASSERTIONS_ENABLED) {
! fillWithGarbage(memory, headerSize, allocationSize, false, maybeUnroll, counters);
}
if (emitMemoryBarrier) {
MembarNode.memoryBarrier(MemoryBarriers.STORE_STORE, LocationIdentity.init());
}
return memory.toObjectNonNull();
*** 793,806 ****
private final SnippetInfo allocatePrimitiveArrayPIC = snippet(NewObjectSnippets.class, "allocatePrimitiveArrayPIC", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION,
TLAB_END_LOCATION);
private final SnippetInfo allocateArrayDynamic = snippet(NewObjectSnippets.class, "allocateArrayDynamic", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION,
TLAB_END_LOCATION);
private final SnippetInfo allocateInstanceDynamic = snippet(NewObjectSnippets.class, "allocateInstanceDynamic", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION,
! TLAB_END_LOCATION, PROTOTYPE_MARK_WORD_LOCATION, CLASS_STATE_LOCATION);
private final SnippetInfo newmultiarray = snippet(NewObjectSnippets.class, "newmultiarray", TLAB_TOP_LOCATION, TLAB_END_LOCATION);
private final SnippetInfo newmultiarrayPIC = snippet(NewObjectSnippets.class, "newmultiarrayPIC", TLAB_TOP_LOCATION, TLAB_END_LOCATION);
private final SnippetInfo verifyHeap = snippet(NewObjectSnippets.class, "verifyHeap");
private final GraalHotSpotVMConfig config;
private final Counters counters;
public Templates(OptionValues options, Iterable<DebugHandlersFactory> factories, SnippetCounter.Group.Factory factory, HotSpotProviders providers, TargetDescription target,
GraalHotSpotVMConfig config) {
--- 815,829 ----
private final SnippetInfo allocatePrimitiveArrayPIC = snippet(NewObjectSnippets.class, "allocatePrimitiveArrayPIC", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION,
TLAB_END_LOCATION);
private final SnippetInfo allocateArrayDynamic = snippet(NewObjectSnippets.class, "allocateArrayDynamic", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION,
TLAB_END_LOCATION);
private final SnippetInfo allocateInstanceDynamic = snippet(NewObjectSnippets.class, "allocateInstanceDynamic", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION,
! TLAB_END_LOCATION, PROTOTYPE_MARK_WORD_LOCATION, CLASS_INIT_STATE_LOCATION);
private final SnippetInfo newmultiarray = snippet(NewObjectSnippets.class, "newmultiarray", TLAB_TOP_LOCATION, TLAB_END_LOCATION);
private final SnippetInfo newmultiarrayPIC = snippet(NewObjectSnippets.class, "newmultiarrayPIC", TLAB_TOP_LOCATION, TLAB_END_LOCATION);
private final SnippetInfo verifyHeap = snippet(NewObjectSnippets.class, "verifyHeap");
+ private final SnippetInfo threadBeingInitializedCheck = snippet(NewObjectSnippets.class, "threadBeingInitializedCheck");
private final GraalHotSpotVMConfig config;
private final Counters counters;
public Templates(OptionValues options, Iterable<DebugHandlersFactory> factories, SnippetCounter.Group.Factory factory, HotSpotProviders providers, TargetDescription target,
GraalHotSpotVMConfig config) {
*** 872,882 ****
args.addConst("fillContents", newArrayNode.fillContents());
args.addConst("emitMemoryBarrier", newArrayNode.emitMemoryBarrier());
args.addConst("threadRegister", registers.getThreadRegister());
args.addConst("maybeUnroll", length.isConstant());
args.addConst("typeContext", ProfileAllocations.getValue(localOptions) ? arrayType.toJavaName(false) : "");
! args.addConst("useBulkZeroing", tool.getLowerer().supportBulkZeroing());
args.addConst("counters", counters);
SnippetTemplate template = template(newArrayNode, args);
graph.getDebug().log("Lowering allocateArray in %s: node=%s, template=%s, arguments=%s", graph, newArrayNode, template, args);
template.instantiate(providers.getMetaAccess(), newArrayNode, DEFAULT_REPLACER, args);
}
--- 895,905 ----
args.addConst("fillContents", newArrayNode.fillContents());
args.addConst("emitMemoryBarrier", newArrayNode.emitMemoryBarrier());
args.addConst("threadRegister", registers.getThreadRegister());
args.addConst("maybeUnroll", length.isConstant());
args.addConst("typeContext", ProfileAllocations.getValue(localOptions) ? arrayType.toJavaName(false) : "");
! args.addConst("bulkZeroingStride", tool.getLowerer().bulkZeroingStride());
args.addConst("counters", counters);
SnippetTemplate template = template(newArrayNode, args);
graph.getDebug().log("Lowering allocateArray in %s: node=%s, template=%s, arguments=%s", graph, newArrayNode, template, args);
template.instantiate(providers.getMetaAccess(), newArrayNode, DEFAULT_REPLACER, args);
}
*** 916,926 ****
if (newArrayNode.getKnownElementKind() != null) {
args.addConst("knownLayoutHelper", lookupArrayClass(tool, newArrayNode.getKnownElementKind()).layoutHelper());
} else {
args.addConst("knownLayoutHelper", 0);
}
! args.addConst("useBulkZeroing", tool.getLowerer().supportBulkZeroing());
args.add("prototypeMarkWord", lookupArrayClass(tool, JavaKind.Object).prototypeMarkWord());
args.addConst("counters", counters);
SnippetTemplate template = template(newArrayNode, args);
template.instantiate(providers.getMetaAccess(), newArrayNode, DEFAULT_REPLACER, args);
}
--- 939,949 ----
if (newArrayNode.getKnownElementKind() != null) {
args.addConst("knownLayoutHelper", lookupArrayClass(tool, newArrayNode.getKnownElementKind()).layoutHelper());
} else {
args.addConst("knownLayoutHelper", 0);
}
! args.addConst("bulkZeroingStride", tool.getLowerer().bulkZeroingStride());
args.add("prototypeMarkWord", lookupArrayClass(tool, JavaKind.Object).prototypeMarkWord());
args.addConst("counters", counters);
SnippetTemplate template = template(newArrayNode, args);
template.instantiate(providers.getMetaAccess(), newArrayNode, DEFAULT_REPLACER, args);
}
*** 963,969 ****
--- 986,1001 ----
template.instantiate(providers.getMetaAccess(), verifyHeapNode, DEFAULT_REPLACER, args);
} else {
GraphUtil.removeFixedWithUnusedInputs(verifyHeapNode);
}
}
+
+ public void lower(KlassBeingInitializedCheckNode verifyHeapNode, HotSpotRegistersProvider registers, LoweringTool tool) {
+ Arguments args = new Arguments(threadBeingInitializedCheck, verifyHeapNode.graph().getGuardsStage(), tool.getLoweringStage());
+ args.addConst("threadRegister", registers.getThreadRegister());
+ args.add("klass", verifyHeapNode.getKlass());
+
+ SnippetTemplate template = template(verifyHeapNode, args);
+ template.instantiate(providers.getMetaAccess(), verifyHeapNode, DEFAULT_REPLACER, args);
+ }
}
}
< prev index next >