< prev index next >
src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/NewObjectSnippets.java
Print this page
*** 26,35 ****
--- 26,36 ----
import static jdk.vm.ci.meta.DeoptimizationAction.None;
import static jdk.vm.ci.meta.DeoptimizationReason.RuntimeConstraint;
import static org.graalvm.compiler.core.common.GraalOptions.GeneratePIC;
import static org.graalvm.compiler.core.common.calc.UnsignedMath.belowThan;
+ import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfigBase.INJECTED_OPTIONVALUES;
import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfigBase.INJECTED_VMCONFIG;
import static org.graalvm.compiler.hotspot.HotSpotBackend.NEW_ARRAY;
import static org.graalvm.compiler.hotspot.HotSpotBackend.NEW_ARRAY_OR_NULL;
import static org.graalvm.compiler.hotspot.HotSpotBackend.NEW_INSTANCE;
import static org.graalvm.compiler.hotspot.HotSpotBackend.NEW_INSTANCE_OR_NULL;
*** 39,52 ****
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.HUB_WRITE_LOCATION;
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.MARK_WORD_LOCATION;
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.PROTOTYPE_MARK_WORD_LOCATION;
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_END_LOCATION;
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_TOP_LOCATION;
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayAllocationSize;
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayKlassOffset;
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayLengthOffset;
- import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.config;
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.initializeObjectHeader;
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.instanceHeaderSize;
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.isInstanceKlassFullyInitialized;
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperHeaderSizeMask;
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperHeaderSizeShift;
--- 40,57 ----
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.HUB_WRITE_LOCATION;
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.MARK_WORD_LOCATION;
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.PROTOTYPE_MARK_WORD_LOCATION;
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_END_LOCATION;
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_TOP_LOCATION;
+ import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.allocateInstancePrefetchLines;
+ import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.allocatePrefetchDistance;
+ import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.allocatePrefetchLines;
+ import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.allocatePrefetchStepSize;
+ import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.allocatePrefetchStyle;
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayAllocationSize;
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayKlassOffset;
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayLengthOffset;
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.initializeObjectHeader;
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.instanceHeaderSize;
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.isInstanceKlassFullyInitialized;
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperHeaderSizeMask;
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperHeaderSizeShift;
*** 149,159 ****
AllocatedTypesInMethod,
Total
}
@Fold
! static String createName(String path, String typeContext, OptionValues options) {
switch (ProfileAllocationsContext.getValue(options)) {
case AllocatingMethod:
return "";
case InstanceOrArray:
return path;
--- 154,164 ----
AllocatedTypesInMethod,
Total
}
@Fold
! static String createName(@Fold.InjectedParameter OptionValues options, String path, String typeContext) {
switch (ProfileAllocationsContext.getValue(options)) {
case AllocatingMethod:
return "";
case InstanceOrArray:
return path;
*** 166,236 ****
throw GraalError.shouldNotReachHere();
}
}
@Fold
! static boolean doProfile(OptionValues options) {
return ProfileAllocations.getValue(options);
}
@Fold
! static boolean withContext(OptionValues options) {
ProfileContext context = ProfileAllocationsContext.getValue(options);
return context == ProfileContext.AllocatingMethod || context == ProfileContext.AllocatedTypesInMethod;
}
! protected static void profileAllocation(String path, long size, String typeContext, OptionValues options) {
! if (doProfile(options)) {
! String name = createName(path, typeContext, options);
! boolean context = withContext(options);
DynamicCounterNode.counter("number of bytes allocated", name, size, context);
DynamicCounterNode.counter("number of allocations", name, 1, context);
}
}
public static void emitPrefetchAllocate(Word address, boolean isArray) {
! GraalHotSpotVMConfig config = config(INJECTED_VMCONFIG);
! if (config.allocatePrefetchStyle > 0) {
// Insert a prefetch for each allocation only on the fast-path
// Generate several prefetch instructions.
! int lines = isArray ? config.allocatePrefetchLines : config.allocateInstancePrefetchLines;
! int stepSize = config.allocatePrefetchStepSize;
! int distance = config.allocatePrefetchDistance;
ExplodeLoopNode.explodeLoop();
for (int i = 0; i < lines; i++) {
PrefetchAllocateNode.prefetch(OffsetAddressNode.address(address, distance));
distance += stepSize;
}
}
}
@Snippet
public static Object allocateInstance(@ConstantParameter int size, KlassPointer hub, Word prototypeMarkWord, @ConstantParameter boolean fillContents,
! @ConstantParameter Register threadRegister, @ConstantParameter boolean constantSize, @ConstantParameter String typeContext, @ConstantParameter OptionValues options,
@ConstantParameter Counters counters) {
! return piCastToSnippetReplaceeStamp(allocateInstanceHelper(size, hub, prototypeMarkWord, fillContents, threadRegister, constantSize, typeContext, options, counters));
}
public static Object allocateInstanceHelper(int size, KlassPointer hub, Word prototypeMarkWord, boolean fillContents,
! Register threadRegister, boolean constantSize, String typeContext, OptionValues options, Counters counters) {
Object result;
Word thread = registerAsWord(threadRegister);
Word top = readTlabTop(thread);
Word end = readTlabEnd(thread);
Word newTop = top.add(size);
if (useTLAB(INJECTED_VMCONFIG) && probability(FAST_PATH_PROBABILITY, newTop.belowOrEqual(end))) {
writeTlabTop(thread, newTop);
emitPrefetchAllocate(newTop, false);
result = formatObject(hub, size, top, prototypeMarkWord, fillContents, constantSize, counters);
} else {
! if (counters != null && counters.stub != null) {
! counters.stub.inc();
}
result = newInstanceStub(hub);
}
! profileAllocation("instance", size, typeContext, options);
return verifyOop(result);
}
public static Object newInstanceStub(KlassPointer hub) {
if (useNullAllocationStubs(INJECTED_VMCONFIG)) {
--- 171,241 ----
throw GraalError.shouldNotReachHere();
}
}
@Fold
! static boolean doProfile(@Fold.InjectedParameter OptionValues options) {
return ProfileAllocations.getValue(options);
}
@Fold
! static boolean withContext(@Fold.InjectedParameter OptionValues options) {
ProfileContext context = ProfileAllocationsContext.getValue(options);
return context == ProfileContext.AllocatingMethod || context == ProfileContext.AllocatedTypesInMethod;
}
! protected static void profileAllocation(String path, long size, String typeContext) {
! if (doProfile(INJECTED_OPTIONVALUES)) {
! String name = createName(INJECTED_OPTIONVALUES, path, typeContext);
! boolean context = withContext(INJECTED_OPTIONVALUES);
DynamicCounterNode.counter("number of bytes allocated", name, size, context);
DynamicCounterNode.counter("number of allocations", name, 1, context);
}
}
public static void emitPrefetchAllocate(Word address, boolean isArray) {
! if (allocatePrefetchStyle(INJECTED_VMCONFIG) > 0) {
// Insert a prefetch for each allocation only on the fast-path
// Generate several prefetch instructions.
! int lines = isArray ? allocatePrefetchLines(INJECTED_VMCONFIG) : allocateInstancePrefetchLines(INJECTED_VMCONFIG);
! int stepSize = allocatePrefetchStepSize(INJECTED_VMCONFIG);
! int distance = allocatePrefetchDistance(INJECTED_VMCONFIG);
ExplodeLoopNode.explodeLoop();
for (int i = 0; i < lines; i++) {
PrefetchAllocateNode.prefetch(OffsetAddressNode.address(address, distance));
distance += stepSize;
}
}
}
@Snippet
public static Object allocateInstance(@ConstantParameter int size, KlassPointer hub, Word prototypeMarkWord, @ConstantParameter boolean fillContents,
! @ConstantParameter Register threadRegister, @ConstantParameter boolean constantSize, @ConstantParameter String typeContext,
@ConstantParameter Counters counters) {
! return piCastToSnippetReplaceeStamp(allocateInstanceHelper(size, hub, prototypeMarkWord, fillContents, threadRegister, constantSize, typeContext, counters));
}
public static Object allocateInstanceHelper(int size, KlassPointer hub, Word prototypeMarkWord, boolean fillContents,
! Register threadRegister, boolean constantSize, String typeContext, Counters counters) {
Object result;
Word thread = registerAsWord(threadRegister);
Word top = readTlabTop(thread);
Word end = readTlabEnd(thread);
Word newTop = top.add(size);
if (useTLAB(INJECTED_VMCONFIG) && probability(FAST_PATH_PROBABILITY, newTop.belowOrEqual(end))) {
writeTlabTop(thread, newTop);
emitPrefetchAllocate(newTop, false);
result = formatObject(hub, size, top, prototypeMarkWord, fillContents, constantSize, counters);
} else {
! Counters theCounters = counters;
! if (theCounters != null && theCounters.stub != null) {
! theCounters.stub.inc();
}
result = newInstanceStub(hub);
}
! profileAllocation("instance", size, typeContext);
return verifyOop(result);
}
public static Object newInstanceStub(KlassPointer hub) {
if (useNullAllocationStubs(INJECTED_VMCONFIG)) {
*** 246,280 ****
@NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = false)
private static native Object newInstanceOrNull(@ConstantNodeParameter ForeignCallDescriptor descriptor, KlassPointer hub);
@Snippet
public static Object allocateInstancePIC(@ConstantParameter int size, KlassPointer hub, Word prototypeMarkWord, @ConstantParameter boolean fillContents,
! @ConstantParameter Register threadRegister, @ConstantParameter boolean constantSize, @ConstantParameter String typeContext, @ConstantParameter OptionValues options,
@ConstantParameter Counters counters) {
// Klass must be initialized by the time the first instance is allocated, therefore we can
// just load it from the corresponding cell and avoid the resolution check. We have to use a
// fixed load though, to prevent it from floating above the initialization.
KlassPointer picHub = LoadConstantIndirectlyFixedNode.loadKlass(hub);
! return piCastToSnippetReplaceeStamp(allocateInstanceHelper(size, picHub, prototypeMarkWord, fillContents, threadRegister, constantSize, typeContext, options, counters));
}
@Snippet
public static Object allocateInstanceDynamic(Class<?> type, Class<?> classClass, @ConstantParameter boolean fillContents, @ConstantParameter Register threadRegister,
! @ConstantParameter OptionValues options, @ConstantParameter Counters counters) {
if (probability(SLOW_PATH_PROBABILITY, type == null)) {
DeoptimizeNode.deopt(None, RuntimeConstraint);
}
Class<?> nonNullType = PiNode.piCastNonNullClass(type, SnippetAnchorNode.anchor());
if (probability(SLOW_PATH_PROBABILITY, DynamicNewInstanceNode.throwsInstantiationException(type, classClass))) {
DeoptimizeNode.deopt(None, RuntimeConstraint);
}
! return PiNode.piCastToSnippetReplaceeStamp(allocateInstanceDynamicHelper(type, fillContents, threadRegister, options, counters, nonNullType));
}
! private static Object allocateInstanceDynamicHelper(Class<?> type, boolean fillContents, Register threadRegister, OptionValues options, Counters counters, Class<?> nonNullType) {
KlassPointer hub = ClassGetHubNode.readClass(nonNullType);
if (probability(FAST_PATH_PROBABILITY, !hub.isNull())) {
KlassPointer nonNullHub = ClassGetHubNode.piCastNonNull(hub, SnippetAnchorNode.anchor());
if (probability(FAST_PATH_PROBABILITY, isInstanceKlassFullyInitialized(nonNullHub))) {
--- 251,285 ----
@NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = false)
private static native Object newInstanceOrNull(@ConstantNodeParameter ForeignCallDescriptor descriptor, KlassPointer hub);
@Snippet
public static Object allocateInstancePIC(@ConstantParameter int size, KlassPointer hub, Word prototypeMarkWord, @ConstantParameter boolean fillContents,
! @ConstantParameter Register threadRegister, @ConstantParameter boolean constantSize, @ConstantParameter String typeContext,
@ConstantParameter Counters counters) {
// Klass must be initialized by the time the first instance is allocated, therefore we can
// just load it from the corresponding cell and avoid the resolution check. We have to use a
// fixed load though, to prevent it from floating above the initialization.
KlassPointer picHub = LoadConstantIndirectlyFixedNode.loadKlass(hub);
! return piCastToSnippetReplaceeStamp(allocateInstanceHelper(size, picHub, prototypeMarkWord, fillContents, threadRegister, constantSize, typeContext, counters));
}
@Snippet
public static Object allocateInstanceDynamic(Class<?> type, Class<?> classClass, @ConstantParameter boolean fillContents, @ConstantParameter Register threadRegister,
! @ConstantParameter Counters counters) {
if (probability(SLOW_PATH_PROBABILITY, type == null)) {
DeoptimizeNode.deopt(None, RuntimeConstraint);
}
Class<?> nonNullType = PiNode.piCastNonNullClass(type, SnippetAnchorNode.anchor());
if (probability(SLOW_PATH_PROBABILITY, DynamicNewInstanceNode.throwsInstantiationException(type, classClass))) {
DeoptimizeNode.deopt(None, RuntimeConstraint);
}
! return PiNode.piCastToSnippetReplaceeStamp(allocateInstanceDynamicHelper(type, fillContents, threadRegister, counters, nonNullType));
}
! private static Object allocateInstanceDynamicHelper(Class<?> type, boolean fillContents, Register threadRegister, Counters counters, Class<?> nonNullType) {
KlassPointer hub = ClassGetHubNode.readClass(nonNullType);
if (probability(FAST_PATH_PROBABILITY, !hub.isNull())) {
KlassPointer nonNullHub = ClassGetHubNode.piCastNonNull(hub, SnippetAnchorNode.anchor());
if (probability(FAST_PATH_PROBABILITY, isInstanceKlassFullyInitialized(nonNullHub))) {
*** 289,299 ****
Word prototypeMarkWord = nonNullHub.readWord(prototypeMarkWordOffset(INJECTED_VMCONFIG), PROTOTYPE_MARK_WORD_LOCATION);
/*
* FIXME(je,ds): we should actually pass typeContext instead of "" but late
* binding of parameters is not yet supported by the GraphBuilderPlugin system.
*/
! return allocateInstanceHelper(layoutHelper, nonNullHub, prototypeMarkWord, fillContents, threadRegister, false, "", options, counters);
}
} else {
DeoptimizeNode.deopt(None, RuntimeConstraint);
}
}
--- 294,304 ----
Word prototypeMarkWord = nonNullHub.readWord(prototypeMarkWordOffset(INJECTED_VMCONFIG), PROTOTYPE_MARK_WORD_LOCATION);
/*
* FIXME(je,ds): we should actually pass typeContext instead of "" but late
* binding of parameters is not yet supported by the GraphBuilderPlugin system.
*/
! return allocateInstanceHelper(layoutHelper, nonNullHub, prototypeMarkWord, fillContents, threadRegister, false, "", counters);
}
} else {
DeoptimizeNode.deopt(None, RuntimeConstraint);
}
}
*** 306,328 ****
public static final int MAX_ARRAY_FAST_PATH_ALLOCATION_LENGTH = 0x00FFFFFF;
@Snippet
public static Object allocatePrimitiveArrayPIC(KlassPointer hub, int length, Word prototypeMarkWord, @ConstantParameter int headerSize, @ConstantParameter int log2ElementSize,
@ConstantParameter boolean fillContents, @ConstantParameter Register threadRegister, @ConstantParameter boolean maybeUnroll, @ConstantParameter String typeContext,
! @ConstantParameter OptionValues options, @ConstantParameter Counters counters) {
// Primitive array types are eagerly pre-resolved. We can use a floating load.
KlassPointer picHub = LoadConstantIndirectlyNode.loadKlass(hub);
! return allocateArrayImpl(picHub, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents, threadRegister, maybeUnroll, typeContext, false, options, counters);
}
@Snippet
public static Object allocateArrayPIC(KlassPointer hub, int length, Word prototypeMarkWord, @ConstantParameter int headerSize, @ConstantParameter int log2ElementSize,
@ConstantParameter boolean fillContents, @ConstantParameter Register threadRegister, @ConstantParameter boolean maybeUnroll, @ConstantParameter String typeContext,
! @ConstantParameter OptionValues options, @ConstantParameter Counters counters) {
// Array type would be resolved by dominating resolution.
KlassPointer picHub = LoadConstantIndirectlyFixedNode.loadKlass(hub);
! return allocateArrayImpl(picHub, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents, threadRegister, maybeUnroll, typeContext, false, options, counters);
}
@Snippet
public static Object allocateArray(KlassPointer hub,
int length,
--- 311,333 ----
public static final int MAX_ARRAY_FAST_PATH_ALLOCATION_LENGTH = 0x00FFFFFF;
@Snippet
public static Object allocatePrimitiveArrayPIC(KlassPointer hub, int length, Word prototypeMarkWord, @ConstantParameter int headerSize, @ConstantParameter int log2ElementSize,
@ConstantParameter boolean fillContents, @ConstantParameter Register threadRegister, @ConstantParameter boolean maybeUnroll, @ConstantParameter String typeContext,
! @ConstantParameter Counters counters) {
// Primitive array types are eagerly pre-resolved. We can use a floating load.
KlassPointer picHub = LoadConstantIndirectlyNode.loadKlass(hub);
! return allocateArrayImpl(picHub, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents, threadRegister, maybeUnroll, typeContext, false, counters);
}
@Snippet
public static Object allocateArrayPIC(KlassPointer hub, int length, Word prototypeMarkWord, @ConstantParameter int headerSize, @ConstantParameter int log2ElementSize,
@ConstantParameter boolean fillContents, @ConstantParameter Register threadRegister, @ConstantParameter boolean maybeUnroll, @ConstantParameter String typeContext,
! @ConstantParameter Counters counters) {
// Array type would be resolved by dominating resolution.
KlassPointer picHub = LoadConstantIndirectlyFixedNode.loadKlass(hub);
! return allocateArrayImpl(picHub, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents, threadRegister, maybeUnroll, typeContext, false, counters);
}
@Snippet
public static Object allocateArray(KlassPointer hub,
int length,
*** 331,343 ****
@ConstantParameter int log2ElementSize,
@ConstantParameter boolean fillContents,
@ConstantParameter Register threadRegister,
@ConstantParameter boolean maybeUnroll,
@ConstantParameter String typeContext,
- @ConstantParameter OptionValues options,
@ConstantParameter Counters counters) {
! Object result = allocateArrayImpl(hub, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents, threadRegister, maybeUnroll, typeContext, false, options, counters);
return piArrayCastToSnippetReplaceeStamp(verifyOop(result), length);
}
/**
* When allocating on the slow path, determines whether to use a version of the runtime call
--- 336,358 ----
@ConstantParameter int log2ElementSize,
@ConstantParameter boolean fillContents,
@ConstantParameter Register threadRegister,
@ConstantParameter boolean maybeUnroll,
@ConstantParameter String typeContext,
@ConstantParameter Counters counters) {
! Object result = allocateArrayImpl(hub,
! length,
! prototypeMarkWord,
! headerSize,
! log2ElementSize,
! fillContents,
! threadRegister,
! maybeUnroll,
! typeContext,
! false,
!
! counters);
return piArrayCastToSnippetReplaceeStamp(verifyOop(result), length);
}
/**
* When allocating on the slow path, determines whether to use a version of the runtime call
*** 347,375 ****
static boolean useNullAllocationStubs(@InjectedParameter GraalHotSpotVMConfig config) {
return config.areNullAllocationStubsAvailable();
}
private static Object allocateArrayImpl(KlassPointer hub, int length, Word prototypeMarkWord, int headerSize, int log2ElementSize, boolean fillContents, Register threadRegister,
! boolean maybeUnroll, String typeContext, boolean skipNegativeCheck, OptionValues options, Counters counters) {
Object result;
int allocationSize = arrayAllocationSize(length, headerSize, log2ElementSize);
Word thread = registerAsWord(threadRegister);
Word top = readTlabTop(thread);
Word end = readTlabEnd(thread);
Word newTop = top.add(allocationSize);
if (probability(FREQUENT_PROBABILITY, skipNegativeCheck || belowThan(length, MAX_ARRAY_FAST_PATH_ALLOCATION_LENGTH)) && useTLAB(INJECTED_VMCONFIG) &&
probability(FAST_PATH_PROBABILITY, newTop.belowOrEqual(end))) {
writeTlabTop(thread, newTop);
emitPrefetchAllocate(newTop, true);
! if (counters != null && counters.arrayLoopInit != null) {
! counters.arrayLoopInit.inc();
}
result = formatArray(hub, allocationSize, length, headerSize, top, prototypeMarkWord, fillContents, maybeUnroll, counters);
} else {
result = newArrayStub(hub, length);
}
! profileAllocation("array", allocationSize, typeContext, options);
return result;
}
public static Object newArrayStub(KlassPointer hub, int length) {
if (useNullAllocationStubs(INJECTED_VMCONFIG)) {
--- 362,391 ----
static boolean useNullAllocationStubs(@InjectedParameter GraalHotSpotVMConfig config) {
return config.areNullAllocationStubsAvailable();
}
private static Object allocateArrayImpl(KlassPointer hub, int length, Word prototypeMarkWord, int headerSize, int log2ElementSize, boolean fillContents, Register threadRegister,
! boolean maybeUnroll, String typeContext, boolean skipNegativeCheck, Counters counters) {
Object result;
int allocationSize = arrayAllocationSize(length, headerSize, log2ElementSize);
Word thread = registerAsWord(threadRegister);
Word top = readTlabTop(thread);
Word end = readTlabEnd(thread);
Word newTop = top.add(allocationSize);
if (probability(FREQUENT_PROBABILITY, skipNegativeCheck || belowThan(length, MAX_ARRAY_FAST_PATH_ALLOCATION_LENGTH)) && useTLAB(INJECTED_VMCONFIG) &&
probability(FAST_PATH_PROBABILITY, newTop.belowOrEqual(end))) {
writeTlabTop(thread, newTop);
emitPrefetchAllocate(newTop, true);
! Counters theCounters = counters;
! if (theCounters != null && theCounters.arrayLoopInit != null) {
! theCounters.arrayLoopInit.inc();
}
result = formatArray(hub, allocationSize, length, headerSize, top, prototypeMarkWord, fillContents, maybeUnroll, counters);
} else {
result = newArrayStub(hub, length);
}
! profileAllocation("array", allocationSize, typeContext);
return result;
}
public static Object newArrayStub(KlassPointer hub, int length) {
if (useNullAllocationStubs(INJECTED_VMCONFIG)) {
*** 419,436 ****
@NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = false)
public static native Object dynamicNewInstanceOrNull(@ConstantNodeParameter ForeignCallDescriptor descriptor, Class<?> elementType);
@Snippet
public static Object allocateArrayDynamic(Class<?> elementType, Class<?> voidClass, int length, @ConstantParameter boolean fillContents, @ConstantParameter Register threadRegister,
! @ConstantParameter JavaKind knownElementKind, @ConstantParameter int knownLayoutHelper, Word prototypeMarkWord, @ConstantParameter OptionValues options,
@ConstantParameter Counters counters) {
! Object result = allocateArrayDynamicImpl(elementType, voidClass, length, fillContents, threadRegister, knownElementKind, knownLayoutHelper, prototypeMarkWord, options, counters);
return result;
}
private static Object allocateArrayDynamicImpl(Class<?> elementType, Class<?> voidClass, int length, boolean fillContents, Register threadRegister, JavaKind knownElementKind,
! int knownLayoutHelper, Word prototypeMarkWord, OptionValues options, Counters counters) {
/*
* We only need the dynamic check for void when we have no static information from
* knownElementKind.
*/
staticAssert(knownElementKind != JavaKind.Void, "unsupported knownElementKind");
--- 435,452 ----
@NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = false)
public static native Object dynamicNewInstanceOrNull(@ConstantNodeParameter ForeignCallDescriptor descriptor, Class<?> elementType);
@Snippet
public static Object allocateArrayDynamic(Class<?> elementType, Class<?> voidClass, int length, @ConstantParameter boolean fillContents, @ConstantParameter Register threadRegister,
! @ConstantParameter JavaKind knownElementKind, @ConstantParameter int knownLayoutHelper, Word prototypeMarkWord,
@ConstantParameter Counters counters) {
! Object result = allocateArrayDynamicImpl(elementType, voidClass, length, fillContents, threadRegister, knownElementKind, knownLayoutHelper, prototypeMarkWord, counters);
return result;
}
private static Object allocateArrayDynamicImpl(Class<?> elementType, Class<?> voidClass, int length, boolean fillContents, Register threadRegister, JavaKind knownElementKind,
! int knownLayoutHelper, Word prototypeMarkWord, Counters counters) {
/*
* We only need the dynamic check for void when we have no static information from
* knownElementKind.
*/
staticAssert(knownElementKind != JavaKind.Void, "unsupported knownElementKind");
*** 468,478 ****
//@formatter:on
int headerSize = (layoutHelper >> layoutHelperHeaderSizeShift(INJECTED_VMCONFIG)) & layoutHelperHeaderSizeMask(INJECTED_VMCONFIG);
int log2ElementSize = (layoutHelper >> layoutHelperLog2ElementSizeShift(INJECTED_VMCONFIG)) & layoutHelperLog2ElementSizeMask(INJECTED_VMCONFIG);
! Object result = allocateArrayImpl(nonNullKlass, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents, threadRegister, false, "dynamic type", true, options, counters);
return piArrayCastToSnippetReplaceeStamp(verifyOop(result), length);
}
/**
* Calls the runtime stub for implementing MULTIANEWARRAY.
--- 484,494 ----
//@formatter:on
int headerSize = (layoutHelper >> layoutHelperHeaderSizeShift(INJECTED_VMCONFIG)) & layoutHelperHeaderSizeMask(INJECTED_VMCONFIG);
int log2ElementSize = (layoutHelper >> layoutHelperLog2ElementSizeShift(INJECTED_VMCONFIG)) & layoutHelperLog2ElementSizeMask(INJECTED_VMCONFIG);
! Object result = allocateArrayImpl(nonNullKlass, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents, threadRegister, false, "dynamic type", true, counters);
return piArrayCastToSnippetReplaceeStamp(verifyOop(result), length);
}
/**
* Calls the runtime stub for implementing MULTIANEWARRAY.
*** 534,550 ****
if ((offset & 0x7) != 0) {
memory.writeInt(offset, (int) value, LocationIdentity.init());
offset += 4;
}
ReplacementsUtil.runtimeAssert((offset & 0x7) == 0, "unaligned offset");
if (manualUnroll && ((size - offset) / 8) <= MAX_UNROLLED_OBJECT_ZEROING_STORES) {
ReplacementsUtil.staticAssert(!constantSize, "size shouldn't be constant at instantiation time");
// This case handles arrays of constant length. Instead of having a snippet variant for
// each length, generate a chain of stores of maximum length. Once it's inlined the
// break statement will trim excess stores.
! if (counters != null && counters.instanceSeqInit != null) {
! counters.instanceSeqInit.inc();
}
explodeLoop();
for (int i = 0; i < MAX_UNROLLED_OBJECT_ZEROING_STORES; i++, offset += 8) {
if (offset == size) {
--- 550,567 ----
if ((offset & 0x7) != 0) {
memory.writeInt(offset, (int) value, LocationIdentity.init());
offset += 4;
}
ReplacementsUtil.runtimeAssert((offset & 0x7) == 0, "unaligned offset");
+ Counters theCounters = counters;
if (manualUnroll && ((size - offset) / 8) <= MAX_UNROLLED_OBJECT_ZEROING_STORES) {
ReplacementsUtil.staticAssert(!constantSize, "size shouldn't be constant at instantiation time");
// This case handles arrays of constant length. Instead of having a snippet variant for
// each length, generate a chain of stores of maximum length. Once it's inlined the
// break statement will trim excess stores.
! if (theCounters != null && theCounters.instanceSeqInit != null) {
! theCounters.instanceSeqInit.inc();
}
explodeLoop();
for (int i = 0; i < MAX_UNROLLED_OBJECT_ZEROING_STORES; i++, offset += 8) {
if (offset == size) {
*** 554,570 ****
}
} else {
// Use Word instead of int to avoid extension to long in generated code
Word off = WordFactory.signed(offset);
if (constantSize && ((size - offset) / 8) <= MAX_UNROLLED_OBJECT_ZEROING_STORES) {
! if (counters != null && counters.instanceSeqInit != null) {
! counters.instanceSeqInit.inc();
}
explodeLoop();
} else {
! if (counters != null && counters.instanceLoopInit != null) {
! counters.instanceLoopInit.inc();
}
}
for (; off.rawValue() < size; off = off.add(8)) {
memory.initializeLong(off, value, LocationIdentity.init());
}
--- 571,587 ----
}
} else {
// Use Word instead of int to avoid extension to long in generated code
Word off = WordFactory.signed(offset);
if (constantSize && ((size - offset) / 8) <= MAX_UNROLLED_OBJECT_ZEROING_STORES) {
! if (theCounters != null && theCounters.instanceSeqInit != null) {
! theCounters.instanceSeqInit.inc();
}
explodeLoop();
} else {
! if (theCounters != null && theCounters.instanceLoopInit != null) {
! theCounters.instanceLoopInit.inc();
}
}
for (; off.rawValue() < size; off = off.add(8)) {
memory.initializeLong(off, value, LocationIdentity.init());
}
*** 692,702 ****
args.add("prototypeMarkWord", type.prototypeMarkWord());
args.addConst("fillContents", newInstanceNode.fillContents());
args.addConst("threadRegister", registers.getThreadRegister());
args.addConst("constantSize", true);
args.addConst("typeContext", ProfileAllocations.getValue(localOptions) ? type.toJavaName(false) : "");
- args.addConst("options", localOptions);
args.addConst("counters", counters);
SnippetTemplate template = template(newInstanceNode, args);
graph.getDebug().log("Lowering allocateInstance in %s: node=%s, template=%s, arguments=%s", graph, newInstanceNode, template, args);
template.instantiate(providers.getMetaAccess(), newInstanceNode, DEFAULT_REPLACER, args);
--- 709,718 ----
*** 736,771 ****
args.addConst("log2ElementSize", log2ElementSize);
args.addConst("fillContents", newArrayNode.fillContents());
args.addConst("threadRegister", registers.getThreadRegister());
args.addConst("maybeUnroll", length.isConstant());
args.addConst("typeContext", ProfileAllocations.getValue(localOptions) ? arrayType.toJavaName(false) : "");
- args.addConst("options", localOptions);
args.addConst("counters", counters);
SnippetTemplate template = template(newArrayNode, args);
graph.getDebug().log("Lowering allocateArray in %s: node=%s, template=%s, arguments=%s", graph, newArrayNode, template, args);
template.instantiate(providers.getMetaAccess(), newArrayNode, DEFAULT_REPLACER, args);
}
public void lower(DynamicNewInstanceNode newInstanceNode, HotSpotRegistersProvider registers, LoweringTool tool) {
Arguments args = new Arguments(allocateInstanceDynamic, newInstanceNode.graph().getGuardsStage(), tool.getLoweringStage());
- OptionValues localOptions = newInstanceNode.getOptions();
args.add("type", newInstanceNode.getInstanceType());
ValueNode classClass = newInstanceNode.getClassClass();
assert classClass != null;
args.add("classClass", classClass);
args.addConst("fillContents", newInstanceNode.fillContents());
args.addConst("threadRegister", registers.getThreadRegister());
- args.addConst("options", localOptions);
args.addConst("counters", counters);
SnippetTemplate template = template(newInstanceNode, args);
template.instantiate(providers.getMetaAccess(), newInstanceNode, DEFAULT_REPLACER, args);
}
public void lower(DynamicNewArrayNode newArrayNode, HotSpotRegistersProvider registers, LoweringTool tool) {
StructuredGraph graph = newArrayNode.graph();
- OptionValues localOptions = graph.getOptions();
Arguments args = new Arguments(allocateArrayDynamic, newArrayNode.graph().getGuardsStage(), tool.getLoweringStage());
args.add("elementType", newArrayNode.getElementType());
ValueNode voidClass = newArrayNode.getVoidClass();
assert voidClass != null;
args.add("voidClass", voidClass);
--- 752,783 ----
*** 782,792 ****
args.addConst("knownLayoutHelper", lookupArrayClass(tool, newArrayNode.getKnownElementKind()).layoutHelper());
} else {
args.addConst("knownLayoutHelper", 0);
}
args.add("prototypeMarkWord", lookupArrayClass(tool, JavaKind.Object).prototypeMarkWord());
- args.addConst("options", localOptions);
args.addConst("counters", counters);
SnippetTemplate template = template(newArrayNode, args);
template.instantiate(providers.getMetaAccess(), newArrayNode, DEFAULT_REPLACER, args);
}
--- 794,803 ----
< prev index next >