< prev index next >
src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/NewObjectSnippets.java
Print this page
@@ -1,7 +1,7 @@
/*
- * Copyright (c) 2012, 2018, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2012, 2019, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation.
@@ -25,10 +25,11 @@
package org.graalvm.compiler.hotspot.replacements;
import static jdk.vm.ci.meta.DeoptimizationAction.None;
import static jdk.vm.ci.meta.DeoptimizationReason.RuntimeConstraint;
import static org.graalvm.compiler.core.common.GraalOptions.GeneratePIC;
+import static org.graalvm.compiler.core.common.GraalOptions.MinimalBulkZeroingSize;
import static org.graalvm.compiler.core.common.calc.UnsignedMath.belowThan;
import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfigBase.INJECTED_OPTIONVALUES;
import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfigBase.INJECTED_VMCONFIG;
import static org.graalvm.compiler.hotspot.HotSpotBackend.NEW_ARRAY;
import static org.graalvm.compiler.hotspot.HotSpotBackend.NEW_ARRAY_OR_NULL;
@@ -128,10 +129,11 @@
import org.graalvm.compiler.replacements.SnippetTemplate.AbstractTemplates;
import org.graalvm.compiler.replacements.SnippetTemplate.Arguments;
import org.graalvm.compiler.replacements.SnippetTemplate.SnippetInfo;
import org.graalvm.compiler.replacements.Snippets;
import org.graalvm.compiler.replacements.nodes.ExplodeLoopNode;
+import org.graalvm.compiler.replacements.nodes.ZeroMemoryNode;
import org.graalvm.compiler.word.Word;
import jdk.internal.vm.compiler.word.LocationIdentity;
import jdk.internal.vm.compiler.word.WordFactory;
import jdk.vm.ci.code.CodeUtil;
@@ -309,25 +311,43 @@
* Maximum array length for which fast path allocation is used.
*/
public static final int MAX_ARRAY_FAST_PATH_ALLOCATION_LENGTH = 0x00FFFFFF;
@Snippet
- public static Object allocatePrimitiveArrayPIC(KlassPointer hub, int length, Word prototypeMarkWord, @ConstantParameter int headerSize, @ConstantParameter int log2ElementSize,
- @ConstantParameter boolean fillContents, @ConstantParameter Register threadRegister, @ConstantParameter boolean maybeUnroll, @ConstantParameter String typeContext,
+ public static Object allocatePrimitiveArrayPIC(KlassPointer hub,
+ int length,
+ Word prototypeMarkWord,
+ @ConstantParameter int headerSize,
+ @ConstantParameter int log2ElementSize,
+ @ConstantParameter boolean fillContents,
+ @ConstantParameter Register threadRegister,
+ @ConstantParameter boolean maybeUnroll,
+ @ConstantParameter String typeContext,
+ @ConstantParameter boolean useBulkZeroing,
@ConstantParameter Counters counters) {
// Primitive array types are eagerly pre-resolved. We can use a floating load.
KlassPointer picHub = LoadConstantIndirectlyNode.loadKlass(hub);
- return allocateArrayImpl(picHub, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents, threadRegister, maybeUnroll, typeContext, counters);
+ return allocateArrayImpl(picHub, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents,
+ threadRegister, maybeUnroll, typeContext, useBulkZeroing, counters);
}
@Snippet
- public static Object allocateArrayPIC(KlassPointer hub, int length, Word prototypeMarkWord, @ConstantParameter int headerSize, @ConstantParameter int log2ElementSize,
- @ConstantParameter boolean fillContents, @ConstantParameter Register threadRegister, @ConstantParameter boolean maybeUnroll, @ConstantParameter String typeContext,
+ public static Object allocateArrayPIC(KlassPointer hub,
+ int length,
+ Word prototypeMarkWord,
+ @ConstantParameter int headerSize,
+ @ConstantParameter int log2ElementSize,
+ @ConstantParameter boolean fillContents,
+ @ConstantParameter Register threadRegister,
+ @ConstantParameter boolean maybeUnroll,
+ @ConstantParameter String typeContext,
+ @ConstantParameter boolean useBulkZeroing,
@ConstantParameter Counters counters) {
// Array type would be resolved by dominating resolution.
KlassPointer picHub = LoadConstantIndirectlyFixedNode.loadKlass(hub);
- return allocateArrayImpl(picHub, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents, threadRegister, maybeUnroll, typeContext, counters);
+ return allocateArrayImpl(picHub, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents,
+ threadRegister, maybeUnroll, typeContext, useBulkZeroing, counters);
}
@Snippet
public static Object allocateArray(KlassPointer hub,
int length,
@@ -336,21 +356,22 @@
@ConstantParameter int log2ElementSize,
@ConstantParameter boolean fillContents,
@ConstantParameter Register threadRegister,
@ConstantParameter boolean maybeUnroll,
@ConstantParameter String typeContext,
+ @ConstantParameter boolean useBulkZeroing,
@ConstantParameter Counters counters) {
Object result = allocateArrayImpl(hub,
length,
prototypeMarkWord,
headerSize,
log2ElementSize,
fillContents,
threadRegister,
maybeUnroll,
typeContext,
-
+ useBulkZeroing,
counters);
return piArrayCastToSnippetReplaceeStamp(verifyOop(result), length);
}
/**
@@ -361,11 +382,11 @@
static boolean useNullAllocationStubs(@InjectedParameter GraalHotSpotVMConfig config) {
return config.areNullAllocationStubsAvailable();
}
private static Object allocateArrayImpl(KlassPointer hub, int length, Word prototypeMarkWord, int headerSize, int log2ElementSize, boolean fillContents, Register threadRegister,
- boolean maybeUnroll, String typeContext, Counters counters) {
+ boolean maybeUnroll, String typeContext, boolean useBulkZeroing, Counters counters) {
Object result;
long allocationSize = arrayAllocationSize(length, headerSize, log2ElementSize);
Word thread = registerAsWord(threadRegister);
Word top = readTlabTop(thread);
Word end = readTlabEnd(thread);
@@ -376,11 +397,11 @@
emitPrefetchAllocate(newTop, true);
Counters theCounters = counters;
if (theCounters != null && theCounters.arrayLoopInit != null) {
theCounters.arrayLoopInit.inc();
}
- result = formatArray(hub, allocationSize, length, headerSize, top, prototypeMarkWord, fillContents, maybeUnroll, counters);
+ result = formatArray(hub, allocationSize, length, headerSize, top, prototypeMarkWord, fillContents, maybeUnroll, useBulkZeroing, counters);
} else {
result = newArrayStub(hub, length);
}
profileAllocation("array", allocationSize, typeContext);
return result;
@@ -433,19 +454,27 @@
@NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = false)
public static native Object dynamicNewInstanceOrNull(@ConstantNodeParameter ForeignCallDescriptor descriptor, Class<?> elementType);
@Snippet
- public static Object allocateArrayDynamic(Class<?> elementType, Class<?> voidClass, int length, @ConstantParameter boolean fillContents, @ConstantParameter Register threadRegister,
- @ConstantParameter JavaKind knownElementKind, @ConstantParameter int knownLayoutHelper, Word prototypeMarkWord,
+ public static Object allocateArrayDynamic(Class<?> elementType,
+ Class<?> voidClass,
+ int length,
+ @ConstantParameter boolean fillContents,
+ @ConstantParameter Register threadRegister,
+ @ConstantParameter JavaKind knownElementKind,
+ @ConstantParameter int knownLayoutHelper,
+ @ConstantParameter boolean useBulkZeroing,
+ Word prototypeMarkWord,
@ConstantParameter Counters counters) {
- Object result = allocateArrayDynamicImpl(elementType, voidClass, length, fillContents, threadRegister, knownElementKind, knownLayoutHelper, prototypeMarkWord, counters);
+ Object result = allocateArrayDynamicImpl(elementType, voidClass, length, fillContents, threadRegister, knownElementKind,
+ knownLayoutHelper, useBulkZeroing, prototypeMarkWord, counters);
return result;
}
private static Object allocateArrayDynamicImpl(Class<?> elementType, Class<?> voidClass, int length, boolean fillContents, Register threadRegister, JavaKind knownElementKind,
- int knownLayoutHelper, Word prototypeMarkWord, Counters counters) {
+ int knownLayoutHelper, boolean useBulkZeroing, Word prototypeMarkWord, Counters counters) {
/*
* We only need the dynamic check for void when we have no static information from
* knownElementKind.
*/
staticAssert(knownElementKind != JavaKind.Void, "unsupported knownElementKind");
@@ -483,11 +512,12 @@
//@formatter:on
int headerSize = (layoutHelper >> layoutHelperHeaderSizeShift(INJECTED_VMCONFIG)) & layoutHelperHeaderSizeMask(INJECTED_VMCONFIG);
int log2ElementSize = (layoutHelper >> layoutHelperLog2ElementSizeShift(INJECTED_VMCONFIG)) & layoutHelperLog2ElementSizeMask(INJECTED_VMCONFIG);
- Object result = allocateArrayImpl(nonNullKlass, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents, threadRegister, false, "dynamic type", counters);
+ Object result = allocateArrayImpl(nonNullKlass, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents,
+ threadRegister, false, "dynamic type", useBulkZeroing, counters);
return piArrayCastToSnippetReplaceeStamp(verifyOop(result), length);
}
/**
* Calls the runtime stub for implementing MULTIANEWARRAY.
@@ -536,16 +566,19 @@
* @param size number of bytes to zero
* @param memory beginning of object which is being zeroed
* @param constantSize is {@code size} known to be constant in the snippet
* @param startOffset offset to begin zeroing. May not be word aligned.
* @param manualUnroll maximally unroll zeroing
+ * @param useBulkZeroing apply bulk zeroing
*/
- private static void zeroMemory(long size, Word memory, boolean constantSize, int startOffset, boolean manualUnroll, Counters counters) {
- fillMemory(0, size, memory, constantSize, startOffset, manualUnroll, counters);
+ private static void zeroMemory(long size, Word memory, boolean constantSize, int startOffset, boolean manualUnroll,
+ boolean useBulkZeroing, Counters counters) {
+ fillMemory(0, size, memory, constantSize, startOffset, manualUnroll, useBulkZeroing, counters);
}
- private static void fillMemory(long value, long size, Word memory, boolean constantSize, int startOffset, boolean manualUnroll, Counters counters) {
+ private static void fillMemory(long value, long size, Word memory, boolean constantSize, int startOffset, boolean manualUnroll,
+ boolean useBulkZeroing, Counters counters) {
ReplacementsUtil.runtimeAssert((size & 0x7) == 0, "unaligned object size");
int offset = startOffset;
if ((offset & 0x7) != 0) {
memory.writeInt(offset, (int) value, LocationIdentity.init());
offset += 4;
@@ -569,26 +602,39 @@
memory.initializeLong(offset, value, LocationIdentity.init());
}
} else {
// Use Word instead of int to avoid extension to long in generated code
Word off = WordFactory.signed(offset);
- if (constantSize && ((size - offset) / 8) <= MAX_UNROLLED_OBJECT_ZEROING_STORES) {
- if (theCounters != null && theCounters.instanceSeqInit != null) {
- theCounters.instanceSeqInit.inc();
+ if (useBulkZeroing && probability(SLOW_PATH_PROBABILITY, size >= getMinimalBulkZeroingSize(INJECTED_OPTIONVALUES))) {
+ if (theCounters != null && theCounters.instanceBulkInit != null) {
+ theCounters.instanceBulkInit.inc();
}
- explodeLoop();
+ ZeroMemoryNode.zero(memory.add(off), size - offset, LocationIdentity.init());
} else {
- if (theCounters != null && theCounters.instanceLoopInit != null) {
- theCounters.instanceLoopInit.inc();
+ if (constantSize && ((size - offset) / 8) <= MAX_UNROLLED_OBJECT_ZEROING_STORES) {
+ if (theCounters != null && theCounters.instanceSeqInit != null) {
+ theCounters.instanceSeqInit.inc();
+ }
+ explodeLoop();
+ } else {
+ if (theCounters != null && theCounters.instanceLoopInit != null) {
+ theCounters.instanceLoopInit.inc();
+ }
}
- }
- for (; off.rawValue() < size; off = off.add(8)) {
- memory.initializeLong(off, value, LocationIdentity.init());
+ for (; off.rawValue() < size; off = off.add(8)) {
+ memory.initializeLong(off, value, LocationIdentity.init());
+ }
+
}
}
}
+ @Fold
+ static int getMinimalBulkZeroingSize(@InjectedParameter OptionValues optionValues) {
+ return MinimalBulkZeroingSize.getValue(optionValues);
+ }
+
/**
* Fill uninitialized memory with garbage value in a newly allocated object, unrolling as
* necessary and ensuring that stores are aligned.
*
* @param size number of bytes to zero
@@ -596,21 +642,21 @@
* @param constantSize is {@code size} known to be constant in the snippet
* @param startOffset offset to begin zeroing. May not be word aligned.
* @param manualUnroll maximally unroll zeroing
*/
private static void fillWithGarbage(long size, Word memory, boolean constantSize, int startOffset, boolean manualUnroll, Counters counters) {
- fillMemory(0xfefefefefefefefeL, size, memory, constantSize, startOffset, manualUnroll, counters);
+ fillMemory(0xfefefefefefefefeL, size, memory, constantSize, startOffset, manualUnroll, false, counters);
}
/**
* Formats some allocated memory with an object header and zeroes out the rest.
*/
private static Object formatObject(KlassPointer hub, long size, Word memory, Word compileTimePrototypeMarkWord, boolean fillContents, boolean constantSize, Counters counters) {
Word prototypeMarkWord = useBiasedLocking(INJECTED_VMCONFIG) ? hub.readWord(prototypeMarkWordOffset(INJECTED_VMCONFIG), PROTOTYPE_MARK_WORD_LOCATION) : compileTimePrototypeMarkWord;
initializeObjectHeader(memory, prototypeMarkWord, hub);
if (fillContents) {
- zeroMemory(size, memory, constantSize, instanceHeaderSize(INJECTED_VMCONFIG), false, counters);
+ zeroMemory(size, memory, constantSize, instanceHeaderSize(INJECTED_VMCONFIG), false, false, counters);
} else if (REPLACEMENTS_ASSERTIONS_ENABLED) {
fillWithGarbage(size, memory, constantSize, instanceHeaderSize(INJECTED_VMCONFIG), false, counters);
}
MembarNode.memoryBarrier(MemoryBarriers.STORE_STORE, LocationIdentity.init());
return memory.toObjectNonNull();
@@ -630,19 +676,19 @@
/**
* Formats some allocated memory with an object header and zeroes out the rest.
*/
private static Object formatArray(KlassPointer hub, long allocationSize, int length, int headerSize, Word memory, Word prototypeMarkWord, boolean fillContents, boolean maybeUnroll,
- Counters counters) {
+ boolean useBulkZeroing, Counters counters) {
memory.writeInt(arrayLengthOffset(INJECTED_VMCONFIG), length, LocationIdentity.init());
/*
* store hub last as the concurrent garbage collectors assume length is valid if hub field
* is not null
*/
initializeObjectHeader(memory, prototypeMarkWord, hub);
if (fillContents) {
- zeroMemory(allocationSize, memory, false, headerSize, maybeUnroll, counters);
+ zeroMemory(allocationSize, memory, false, headerSize, maybeUnroll, useBulkZeroing, counters);
} else if (REPLACEMENTS_ASSERTIONS_ENABLED) {
fillWithGarbage(allocationSize, memory, false, headerSize, maybeUnroll, counters);
}
MembarNode.memoryBarrier(MemoryBarriers.STORE_STORE, LocationIdentity.init());
return memory.toObjectNonNull();
@@ -652,16 +698,18 @@
Counters(SnippetCounter.Group.Factory factory) {
Group newInstance = factory.createSnippetCounterGroup("NewInstance");
Group newArray = factory.createSnippetCounterGroup("NewArray");
instanceSeqInit = new SnippetCounter(newInstance, "tlabSeqInit", "TLAB alloc with unrolled zeroing");
instanceLoopInit = new SnippetCounter(newInstance, "tlabLoopInit", "TLAB alloc with zeroing in a loop");
+ instanceBulkInit = new SnippetCounter(newArray, "tlabBulkInit", "TLAB alloc with bulk zeroing");
arrayLoopInit = new SnippetCounter(newArray, "tlabLoopInit", "TLAB alloc with zeroing in a loop");
stub = new SnippetCounter(newInstance, "stub", "alloc and zeroing via stub");
}
final SnippetCounter instanceSeqInit;
final SnippetCounter instanceLoopInit;
+ final SnippetCounter instanceBulkInit;
final SnippetCounter arrayLoopInit;
final SnippetCounter stub;
}
public static class Templates extends AbstractTemplates {
@@ -751,10 +799,11 @@
args.addConst("log2ElementSize", log2ElementSize);
args.addConst("fillContents", newArrayNode.fillContents());
args.addConst("threadRegister", registers.getThreadRegister());
args.addConst("maybeUnroll", length.isConstant());
args.addConst("typeContext", ProfileAllocations.getValue(localOptions) ? arrayType.toJavaName(false) : "");
+ args.addConst("useBulkZeroing", tool.getLowerer().supportBulkZeroing());
args.addConst("counters", counters);
SnippetTemplate template = template(newArrayNode, args);
graph.getDebug().log("Lowering allocateArray in %s: node=%s, template=%s, arguments=%s", graph, newArrayNode, template, args);
template.instantiate(providers.getMetaAccess(), newArrayNode, DEFAULT_REPLACER, args);
}
@@ -773,11 +822,11 @@
template.instantiate(providers.getMetaAccess(), newInstanceNode, DEFAULT_REPLACER, args);
}
public void lower(DynamicNewArrayNode newArrayNode, HotSpotRegistersProvider registers, LoweringTool tool) {
StructuredGraph graph = newArrayNode.graph();
- Arguments args = new Arguments(allocateArrayDynamic, newArrayNode.graph().getGuardsStage(), tool.getLoweringStage());
+ Arguments args = new Arguments(allocateArrayDynamic, graph.getGuardsStage(), tool.getLoweringStage());
args.add("elementType", newArrayNode.getElementType());
ValueNode voidClass = newArrayNode.getVoidClass();
assert voidClass != null;
args.add("voidClass", voidClass);
ValueNode length = newArrayNode.length();
@@ -792,10 +841,11 @@
if (newArrayNode.getKnownElementKind() != null) {
args.addConst("knownLayoutHelper", lookupArrayClass(tool, newArrayNode.getKnownElementKind()).layoutHelper());
} else {
args.addConst("knownLayoutHelper", 0);
}
+ args.addConst("useBulkZeroing", tool.getLowerer().supportBulkZeroing());
args.add("prototypeMarkWord", lookupArrayClass(tool, JavaKind.Object).prototypeMarkWord());
args.addConst("counters", counters);
SnippetTemplate template = template(newArrayNode, args);
template.instantiate(providers.getMetaAccess(), newArrayNode, DEFAULT_REPLACER, args);
}
< prev index next >