< prev index next >
src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/NewObjectSnippets.java
Print this page
*** 207,229 ****
}
}
}
@Snippet
! public static Object allocateInstance(@ConstantParameter int size, KlassPointer hub, Word prototypeMarkWord, @ConstantParameter boolean fillContents,
@ConstantParameter Register threadRegister, @ConstantParameter boolean constantSize, @ConstantParameter String typeContext,
@ConstantParameter Counters counters) {
return piCastToSnippetReplaceeStamp(allocateInstanceHelper(size, hub, prototypeMarkWord, fillContents, threadRegister, constantSize, typeContext, counters));
}
! public static Object allocateInstanceHelper(int size, KlassPointer hub, Word prototypeMarkWord, boolean fillContents,
Register threadRegister, boolean constantSize, String typeContext, Counters counters) {
Object result;
Word thread = registerAsWord(threadRegister);
Word top = readTlabTop(thread);
Word end = readTlabEnd(thread);
! Word newTop = top.add(size);
if (useTLAB(INJECTED_VMCONFIG) && probability(FAST_PATH_PROBABILITY, newTop.belowOrEqual(end))) {
writeTlabTop(thread, newTop);
emitPrefetchAllocate(newTop, false);
result = formatObject(hub, size, top, prototypeMarkWord, fillContents, constantSize, counters);
} else {
--- 207,229 ----
}
}
}
@Snippet
! public static Object allocateInstance(@ConstantParameter long size, KlassPointer hub, Word prototypeMarkWord, @ConstantParameter boolean fillContents,
@ConstantParameter Register threadRegister, @ConstantParameter boolean constantSize, @ConstantParameter String typeContext,
@ConstantParameter Counters counters) {
return piCastToSnippetReplaceeStamp(allocateInstanceHelper(size, hub, prototypeMarkWord, fillContents, threadRegister, constantSize, typeContext, counters));
}
! public static Object allocateInstanceHelper(long size, KlassPointer hub, Word prototypeMarkWord, boolean fillContents,
Register threadRegister, boolean constantSize, String typeContext, Counters counters) {
Object result;
Word thread = registerAsWord(threadRegister);
Word top = readTlabTop(thread);
Word end = readTlabEnd(thread);
! Word newTop = top.add(WordFactory.unsigned(size));
if (useTLAB(INJECTED_VMCONFIG) && probability(FAST_PATH_PROBABILITY, newTop.belowOrEqual(end))) {
writeTlabTop(thread, newTop);
emitPrefetchAllocate(newTop, false);
result = formatObject(hub, size, top, prototypeMarkWord, fillContents, constantSize, counters);
} else {
*** 250,260 ****
@NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = false)
private static native Object newInstanceOrNull(@ConstantNodeParameter ForeignCallDescriptor descriptor, KlassPointer hub);
@Snippet
! public static Object allocateInstancePIC(@ConstantParameter int size, KlassPointer hub, Word prototypeMarkWord, @ConstantParameter boolean fillContents,
@ConstantParameter Register threadRegister, @ConstantParameter boolean constantSize, @ConstantParameter String typeContext,
@ConstantParameter Counters counters) {
// Klass must be initialized by the time the first instance is allocated, therefore we can
// just load it from the corresponding cell and avoid the resolution check. We have to use a
// fixed load though, to prevent it from floating above the initialization.
--- 250,260 ----
@NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = false)
private static native Object newInstanceOrNull(@ConstantNodeParameter ForeignCallDescriptor descriptor, KlassPointer hub);
@Snippet
! public static Object allocateInstancePIC(@ConstantParameter long size, KlassPointer hub, Word prototypeMarkWord, @ConstantParameter boolean fillContents,
@ConstantParameter Register threadRegister, @ConstantParameter boolean constantSize, @ConstantParameter String typeContext,
@ConstantParameter Counters counters) {
// Klass must be initialized by the time the first instance is allocated, therefore we can
// just load it from the corresponding cell and avoid the resolution check. We have to use a
// fixed load though, to prevent it from floating above the initialization.
*** 314,333 ****
public static Object allocatePrimitiveArrayPIC(KlassPointer hub, int length, Word prototypeMarkWord, @ConstantParameter int headerSize, @ConstantParameter int log2ElementSize,
@ConstantParameter boolean fillContents, @ConstantParameter Register threadRegister, @ConstantParameter boolean maybeUnroll, @ConstantParameter String typeContext,
@ConstantParameter Counters counters) {
// Primitive array types are eagerly pre-resolved. We can use a floating load.
KlassPointer picHub = LoadConstantIndirectlyNode.loadKlass(hub);
! return allocateArrayImpl(picHub, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents, threadRegister, maybeUnroll, typeContext, false, counters);
}
@Snippet
public static Object allocateArrayPIC(KlassPointer hub, int length, Word prototypeMarkWord, @ConstantParameter int headerSize, @ConstantParameter int log2ElementSize,
@ConstantParameter boolean fillContents, @ConstantParameter Register threadRegister, @ConstantParameter boolean maybeUnroll, @ConstantParameter String typeContext,
@ConstantParameter Counters counters) {
// Array type would be resolved by dominating resolution.
KlassPointer picHub = LoadConstantIndirectlyFixedNode.loadKlass(hub);
! return allocateArrayImpl(picHub, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents, threadRegister, maybeUnroll, typeContext, false, counters);
}
@Snippet
public static Object allocateArray(KlassPointer hub,
int length,
--- 314,333 ----
public static Object allocatePrimitiveArrayPIC(KlassPointer hub, int length, Word prototypeMarkWord, @ConstantParameter int headerSize, @ConstantParameter int log2ElementSize,
@ConstantParameter boolean fillContents, @ConstantParameter Register threadRegister, @ConstantParameter boolean maybeUnroll, @ConstantParameter String typeContext,
@ConstantParameter Counters counters) {
// Primitive array types are eagerly pre-resolved. We can use a floating load.
KlassPointer picHub = LoadConstantIndirectlyNode.loadKlass(hub);
! return allocateArrayImpl(picHub, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents, threadRegister, maybeUnroll, typeContext, counters);
}
@Snippet
public static Object allocateArrayPIC(KlassPointer hub, int length, Word prototypeMarkWord, @ConstantParameter int headerSize, @ConstantParameter int log2ElementSize,
@ConstantParameter boolean fillContents, @ConstantParameter Register threadRegister, @ConstantParameter boolean maybeUnroll, @ConstantParameter String typeContext,
@ConstantParameter Counters counters) {
// Array type would be resolved by dominating resolution.
KlassPointer picHub = LoadConstantIndirectlyFixedNode.loadKlass(hub);
! return allocateArrayImpl(picHub, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents, threadRegister, maybeUnroll, typeContext, counters);
}
@Snippet
public static Object allocateArray(KlassPointer hub,
int length,
*** 346,356 ****
log2ElementSize,
fillContents,
threadRegister,
maybeUnroll,
typeContext,
- false,
counters);
return piArrayCastToSnippetReplaceeStamp(verifyOop(result), length);
}
--- 346,355 ----
*** 362,379 ****
static boolean useNullAllocationStubs(@InjectedParameter GraalHotSpotVMConfig config) {
return config.areNullAllocationStubsAvailable();
}
private static Object allocateArrayImpl(KlassPointer hub, int length, Word prototypeMarkWord, int headerSize, int log2ElementSize, boolean fillContents, Register threadRegister,
! boolean maybeUnroll, String typeContext, boolean skipNegativeCheck, Counters counters) {
Object result;
! int allocationSize = arrayAllocationSize(length, headerSize, log2ElementSize);
Word thread = registerAsWord(threadRegister);
Word top = readTlabTop(thread);
Word end = readTlabEnd(thread);
! Word newTop = top.add(allocationSize);
! if (probability(FREQUENT_PROBABILITY, skipNegativeCheck || belowThan(length, MAX_ARRAY_FAST_PATH_ALLOCATION_LENGTH)) && useTLAB(INJECTED_VMCONFIG) &&
probability(FAST_PATH_PROBABILITY, newTop.belowOrEqual(end))) {
writeTlabTop(thread, newTop);
emitPrefetchAllocate(newTop, true);
Counters theCounters = counters;
if (theCounters != null && theCounters.arrayLoopInit != null) {
--- 361,378 ----
static boolean useNullAllocationStubs(@InjectedParameter GraalHotSpotVMConfig config) {
return config.areNullAllocationStubsAvailable();
}
private static Object allocateArrayImpl(KlassPointer hub, int length, Word prototypeMarkWord, int headerSize, int log2ElementSize, boolean fillContents, Register threadRegister,
! boolean maybeUnroll, String typeContext, Counters counters) {
Object result;
! long allocationSize = arrayAllocationSize(length, headerSize, log2ElementSize);
Word thread = registerAsWord(threadRegister);
Word top = readTlabTop(thread);
Word end = readTlabEnd(thread);
! Word newTop = top.add(WordFactory.unsigned(allocationSize));
! if (probability(FREQUENT_PROBABILITY, belowThan(length, MAX_ARRAY_FAST_PATH_ALLOCATION_LENGTH)) && useTLAB(INJECTED_VMCONFIG) &&
probability(FAST_PATH_PROBABILITY, newTop.belowOrEqual(end))) {
writeTlabTop(thread, newTop);
emitPrefetchAllocate(newTop, true);
Counters theCounters = counters;
if (theCounters != null && theCounters.arrayLoopInit != null) {
*** 484,494 ****
//@formatter:on
int headerSize = (layoutHelper >> layoutHelperHeaderSizeShift(INJECTED_VMCONFIG)) & layoutHelperHeaderSizeMask(INJECTED_VMCONFIG);
int log2ElementSize = (layoutHelper >> layoutHelperLog2ElementSizeShift(INJECTED_VMCONFIG)) & layoutHelperLog2ElementSizeMask(INJECTED_VMCONFIG);
! Object result = allocateArrayImpl(nonNullKlass, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents, threadRegister, false, "dynamic type", true, counters);
return piArrayCastToSnippetReplaceeStamp(verifyOop(result), length);
}
/**
* Calls the runtime stub for implementing MULTIANEWARRAY.
--- 483,493 ----
//@formatter:on
int headerSize = (layoutHelper >> layoutHelperHeaderSizeShift(INJECTED_VMCONFIG)) & layoutHelperHeaderSizeMask(INJECTED_VMCONFIG);
int log2ElementSize = (layoutHelper >> layoutHelperLog2ElementSizeShift(INJECTED_VMCONFIG)) & layoutHelperLog2ElementSizeMask(INJECTED_VMCONFIG);
! Object result = allocateArrayImpl(nonNullKlass, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents, threadRegister, false, "dynamic type", counters);
return piArrayCastToSnippetReplaceeStamp(verifyOop(result), length);
}
/**
* Calls the runtime stub for implementing MULTIANEWARRAY.
*** 538,552 ****
* @param memory beginning of object which is being zeroed
* @param constantSize is {@code size} known to be constant in the snippet
* @param startOffset offset to begin zeroing. May not be word aligned.
* @param manualUnroll maximally unroll zeroing
*/
! private static void zeroMemory(int size, Word memory, boolean constantSize, int startOffset, boolean manualUnroll, Counters counters) {
fillMemory(0, size, memory, constantSize, startOffset, manualUnroll, counters);
}
! private static void fillMemory(long value, int size, Word memory, boolean constantSize, int startOffset, boolean manualUnroll, Counters counters) {
ReplacementsUtil.runtimeAssert((size & 0x7) == 0, "unaligned object size");
int offset = startOffset;
if ((offset & 0x7) != 0) {
memory.writeInt(offset, (int) value, LocationIdentity.init());
offset += 4;
--- 537,551 ----
* @param memory beginning of object which is being zeroed
* @param constantSize is {@code size} known to be constant in the snippet
* @param startOffset offset to begin zeroing. May not be word aligned.
* @param manualUnroll maximally unroll zeroing
*/
! private static void zeroMemory(long size, Word memory, boolean constantSize, int startOffset, boolean manualUnroll, Counters counters) {
fillMemory(0, size, memory, constantSize, startOffset, manualUnroll, counters);
}
! private static void fillMemory(long value, long size, Word memory, boolean constantSize, int startOffset, boolean manualUnroll, Counters counters) {
ReplacementsUtil.runtimeAssert((size & 0x7) == 0, "unaligned object size");
int offset = startOffset;
if ((offset & 0x7) != 0) {
memory.writeInt(offset, (int) value, LocationIdentity.init());
offset += 4;
*** 596,613 ****
* @param memory beginning of object which is being zeroed
* @param constantSize is {@code size} known to be constant in the snippet
* @param startOffset offset to begin zeroing. May not be word aligned.
* @param manualUnroll maximally unroll zeroing
*/
! private static void fillWithGarbage(int size, Word memory, boolean constantSize, int startOffset, boolean manualUnroll, Counters counters) {
fillMemory(0xfefefefefefefefeL, size, memory, constantSize, startOffset, manualUnroll, counters);
}
/**
* Formats some allocated memory with an object header and zeroes out the rest.
*/
! private static Object formatObject(KlassPointer hub, int size, Word memory, Word compileTimePrototypeMarkWord, boolean fillContents, boolean constantSize, Counters counters) {
Word prototypeMarkWord = useBiasedLocking(INJECTED_VMCONFIG) ? hub.readWord(prototypeMarkWordOffset(INJECTED_VMCONFIG), PROTOTYPE_MARK_WORD_LOCATION) : compileTimePrototypeMarkWord;
initializeObjectHeader(memory, prototypeMarkWord, hub);
if (fillContents) {
zeroMemory(size, memory, constantSize, instanceHeaderSize(INJECTED_VMCONFIG), false, counters);
} else if (REPLACEMENTS_ASSERTIONS_ENABLED) {
--- 595,612 ----
* @param memory beginning of object which is being zeroed
* @param constantSize is {@code size} known to be constant in the snippet
* @param startOffset offset to begin zeroing. May not be word aligned.
* @param manualUnroll maximally unroll zeroing
*/
! private static void fillWithGarbage(long size, Word memory, boolean constantSize, int startOffset, boolean manualUnroll, Counters counters) {
fillMemory(0xfefefefefefefefeL, size, memory, constantSize, startOffset, manualUnroll, counters);
}
/**
* Formats some allocated memory with an object header and zeroes out the rest.
*/
! private static Object formatObject(KlassPointer hub, long size, Word memory, Word compileTimePrototypeMarkWord, boolean fillContents, boolean constantSize, Counters counters) {
Word prototypeMarkWord = useBiasedLocking(INJECTED_VMCONFIG) ? hub.readWord(prototypeMarkWordOffset(INJECTED_VMCONFIG), PROTOTYPE_MARK_WORD_LOCATION) : compileTimePrototypeMarkWord;
initializeObjectHeader(memory, prototypeMarkWord, hub);
if (fillContents) {
zeroMemory(size, memory, constantSize, instanceHeaderSize(INJECTED_VMCONFIG), false, counters);
} else if (REPLACEMENTS_ASSERTIONS_ENABLED) {
*** 630,640 ****
}
/**
* Formats some allocated memory with an object header and zeroes out the rest.
*/
! private static Object formatArray(KlassPointer hub, int allocationSize, int length, int headerSize, Word memory, Word prototypeMarkWord, boolean fillContents, boolean maybeUnroll,
Counters counters) {
memory.writeInt(arrayLengthOffset(INJECTED_VMCONFIG), length, LocationIdentity.init());
/*
* store hub last as the concurrent garbage collectors assume length is valid if hub field
* is not null
--- 629,639 ----
}
/**
* Formats some allocated memory with an object header and zeroes out the rest.
*/
! private static Object formatArray(KlassPointer hub, long allocationSize, int length, int headerSize, Word memory, Word prototypeMarkWord, boolean fillContents, boolean maybeUnroll,
Counters counters) {
memory.writeInt(arrayLengthOffset(INJECTED_VMCONFIG), length, LocationIdentity.init());
/*
* store hub last as the concurrent garbage collectors assume length is valid if hub field
* is not null
*** 697,707 ****
public void lower(NewInstanceNode newInstanceNode, HotSpotRegistersProvider registers, LoweringTool tool) {
StructuredGraph graph = newInstanceNode.graph();
HotSpotResolvedObjectType type = (HotSpotResolvedObjectType) newInstanceNode.instanceClass();
assert !type.isArray();
ConstantNode hub = ConstantNode.forConstant(KlassPointerStamp.klassNonNull(), type.klass(), providers.getMetaAccess(), graph);
! int size = instanceSize(type);
OptionValues localOptions = graph.getOptions();
SnippetInfo snippet = GeneratePIC.getValue(localOptions) ? allocateInstancePIC : allocateInstance;
Arguments args = new Arguments(snippet, graph.getGuardsStage(), tool.getLoweringStage());
args.addConst("size", size);
--- 696,706 ----
public void lower(NewInstanceNode newInstanceNode, HotSpotRegistersProvider registers, LoweringTool tool) {
StructuredGraph graph = newInstanceNode.graph();
HotSpotResolvedObjectType type = (HotSpotResolvedObjectType) newInstanceNode.instanceClass();
assert !type.isArray();
ConstantNode hub = ConstantNode.forConstant(KlassPointerStamp.klassNonNull(), type.klass(), providers.getMetaAccess(), graph);
! long size = instanceSize(type);
OptionValues localOptions = graph.getOptions();
SnippetInfo snippet = GeneratePIC.getValue(localOptions) ? allocateInstancePIC : allocateInstance;
Arguments args = new Arguments(snippet, graph.getGuardsStage(), tool.getLoweringStage());
args.addConst("size", size);
*** 822,833 ****
args.addConst("rank", rank);
args.addVarargs("dimensions", int.class, StampFactory.forKind(JavaKind.Int), dims);
template(newmultiarrayNode, args).instantiate(providers.getMetaAccess(), newmultiarrayNode, DEFAULT_REPLACER, args);
}
! private static int instanceSize(HotSpotResolvedObjectType type) {
! int size = type.instanceSize();
assert size >= 0;
return size;
}
public void lower(VerifyHeapNode verifyHeapNode, HotSpotRegistersProvider registers, LoweringTool tool) {
--- 821,832 ----
args.addConst("rank", rank);
args.addVarargs("dimensions", int.class, StampFactory.forKind(JavaKind.Int), dims);
template(newmultiarrayNode, args).instantiate(providers.getMetaAccess(), newmultiarrayNode, DEFAULT_REPLACER, args);
}
! private static long instanceSize(HotSpotResolvedObjectType type) {
! long size = type.instanceSize();
assert size >= 0;
return size;
}
public void lower(VerifyHeapNode verifyHeapNode, HotSpotRegistersProvider registers, LoweringTool tool) {
< prev index next >